From 709b0f26e7ccbb313569e0a832b3dd5824b672d0 Mon Sep 17 00:00:00 2001 From: harehare Date: Thu, 8 Oct 2026 13:05:00 +0900 Subject: [PATCH 01/88] =?UTF-8?q?=E2=9C=A8=20feat(markdown):=20add=20nativ?= =?UTF-8?q?e=20Markdown=20parser?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a built-in two-phase parser (block, then inline) that builds Node directly, behind the `native-parser` feature (enabled by default). Covers CommonMark, GFM, frontmatter, math and MDX. Output, including positions, is differential-tested against markdown-rs and the CommonMark/GFM spec examples. Disable the feature to fall back to markdown-rs. --- crates/mq-markdown/Cargo.toml | 2 + crates/mq-markdown/src/lib.rs | 6 + crates/mq-markdown/src/markdown.rs | 185 +- crates/mq-markdown/src/node.rs | 36 +- crates/mq-markdown/src/parser.rs | 769 ++++++ crates/mq-markdown/src/parser/block.rs | 1197 +++++++++ crates/mq-markdown/src/parser/code.rs | 154 ++ crates/mq-markdown/src/parser/definition.rs | 146 ++ crates/mq-markdown/src/parser/html_flow.rs | 240 ++ crates/mq-markdown/src/parser/inline.rs | 612 +++++ .../mq-markdown/src/parser/inline/emphasis.rs | 126 + .../mq-markdown/src/parser/inline/entities.rs | 2132 +++++++++++++++++ .../mq-markdown/src/parser/inline/entity.rs | 115 + crates/mq-markdown/src/parser/inline/html.rs | 197 ++ crates/mq-markdown/src/parser/inline/link.rs | 223 ++ .../mq-markdown/src/parser/inline/literal.rs | 323 +++ .../src/parser/inline/punctuation.rs | 375 +++ crates/mq-markdown/src/parser/inline/tail.rs | 190 ++ crates/mq-markdown/src/parser/line.rs | 208 ++ crates/mq-markdown/src/parser/mdx.rs | 371 +++ crates/mq-markdown/src/parser/mdx_flow.rs | 208 ++ crates/mq-markdown/src/parser/resolve.rs | 226 ++ crates/mq-markdown/src/parser/table.rs | 206 ++ crates/mq-markdown/src/parser/tree.rs | 143 ++ 24 files changed, 8280 insertions(+), 110 deletions(-) create mode 100644 crates/mq-markdown/src/parser.rs create mode 100644 crates/mq-markdown/src/parser/block.rs create mode 100644 crates/mq-markdown/src/parser/code.rs create mode 100644 crates/mq-markdown/src/parser/definition.rs create mode 100644 crates/mq-markdown/src/parser/html_flow.rs create mode 100644 crates/mq-markdown/src/parser/inline.rs create mode 100644 crates/mq-markdown/src/parser/inline/emphasis.rs create mode 100644 crates/mq-markdown/src/parser/inline/entities.rs create mode 100644 crates/mq-markdown/src/parser/inline/entity.rs create mode 100644 crates/mq-markdown/src/parser/inline/html.rs create mode 100644 crates/mq-markdown/src/parser/inline/link.rs create mode 100644 crates/mq-markdown/src/parser/inline/literal.rs create mode 100644 crates/mq-markdown/src/parser/inline/punctuation.rs create mode 100644 crates/mq-markdown/src/parser/inline/tail.rs create mode 100644 crates/mq-markdown/src/parser/line.rs create mode 100644 crates/mq-markdown/src/parser/mdx.rs create mode 100644 crates/mq-markdown/src/parser/mdx_flow.rs create mode 100644 crates/mq-markdown/src/parser/resolve.rs create mode 100644 crates/mq-markdown/src/parser/table.rs create mode 100644 crates/mq-markdown/src/parser/tree.rs diff --git a/crates/mq-markdown/Cargo.toml b/crates/mq-markdown/Cargo.toml index f85a42850..26bd61f30 100644 --- a/crates/mq-markdown/Cargo.toml +++ b/crates/mq-markdown/Cargo.toml @@ -38,8 +38,10 @@ name = "benchmark" [features] callout = [] color = [] +default = ["native-parser"] embed = [] html-to-markdown = ["dep:scraper", "dep:ego-tree", "dep:serde_json", "dep:yaml-rust2", "dep:url"] json = ["dep:serde", "dep:serde_json", "smol_str/serde"] +native-parser = [] obsidian = ["wikilink", "callout", "embed"] wikilink = [] diff --git a/crates/mq-markdown/src/lib.rs b/crates/mq-markdown/src/lib.rs index cf2963cfe..a40de0cef 100644 --- a/crates/mq-markdown/src/lib.rs +++ b/crates/mq-markdown/src/lib.rs @@ -11,6 +11,10 @@ //! - **MDX Support**: Parse and manipulate MDX (Markdown + JSX) content //! - **JSON Export**: Serialize markdown AST to JSON (with `json` feature) //! - **Configurable Rendering**: Customize output formatting and styles +//! - **Built-in parser**: The `native-parser` feature, on by default, parses CommonMark, GFM (tables, +//! strikethrough, autolink literals, task lists, footnotes), frontmatter, math and MDX with a parser +//! of this crate. Without it, `markdown-rs` does the parsing, with the same results for all but a few +//! rare inputs. `Markdown::to_html` always uses `markdown-rs`. //! //! ## Quick Start //! @@ -82,6 +86,8 @@ mod html_to_markdown; mod markdown; mod node; mod node_kind; +#[cfg(any(feature = "native-parser", test))] +mod parser; pub use markdown::{Markdown, to_html}; pub use node::{ Blockquote, Break, Code, CodeInline, ColorTheme, Definition, Delete, Emphasis, Footnote, FootnoteRef, Fragment, diff --git a/crates/mq-markdown/src/markdown.rs b/crates/mq-markdown/src/markdown.rs index 8ef692b52..ccb77d6a6 100644 --- a/crates/mq-markdown/src/markdown.rs +++ b/crates/mq-markdown/src/markdown.rs @@ -7,6 +7,7 @@ use crate::node::{ render_values, }; use markdown::{CompileOptions, Constructs, Options, ParseOptions}; +#[cfg(any(not(feature = "native-parser"), test, feature = "json", feature = "html-to-markdown"))] use miette::miette; use std::{fmt, str::FromStr}; use table_layout::{TableLayout, write_padded_cell}; @@ -233,11 +234,8 @@ impl Markdown { } pub fn from_mdx_str(content: &str) -> miette::Result { - let root = markdown::to_mdast(content, &markdown::ParseOptions::mdx()).map_err(|e| miette!(e.reason))?; - let nodes = Node::from_mdast_node(root); - Ok(Self { - nodes, + nodes: parse_mdx_nodes(content)?, options: RenderOptions::default(), }) } @@ -279,53 +277,7 @@ impl Markdown { } pub fn from_markdown_str(content: &str) -> miette::Result { - let root = markdown::to_mdast( - content, - &markdown::ParseOptions { - gfm_strikethrough_single_tilde: true, - math_text_single_dollar: true, - mdx_expression_parse: None, - mdx_esm_parse: None, - constructs: Constructs { - attention: true, - autolink: true, - block_quote: true, - character_escape: true, - character_reference: true, - code_indented: true, - code_fenced: true, - code_text: true, - definition: true, - frontmatter: true, - gfm_autolink_literal: true, - gfm_label_start_footnote: true, - gfm_footnote_definition: true, - gfm_strikethrough: true, - gfm_table: true, - gfm_task_list_item: true, - hard_break_escape: true, - hard_break_trailing: true, - heading_atx: true, - heading_setext: true, - html_flow: true, - html_text: true, - label_start_image: true, - label_start_link: true, - label_end: true, - list_item: true, - math_flow: true, - math_text: true, - mdx_esm: false, - mdx_expression_flow: false, - mdx_expression_text: false, - mdx_jsx_flow: false, - mdx_jsx_text: false, - thematic_break: true, - }, - }, - ) - .map_err(|e| miette!(e.reason))?; - let nodes = Node::from_mdast_node(root); + let nodes = parse_nodes(content)?; #[cfg(all(feature = "embed", feature = "wikilink"))] let nodes = if content.contains("[[") { @@ -358,54 +310,91 @@ impl Markdown { /// isolate expand cost from the mdast parse cost. #[cfg(any(feature = "wikilink", feature = "embed"))] pub fn from_markdown_str_no_expand(content: &str) -> miette::Result> { - let root = markdown::to_mdast( - content, - &markdown::ParseOptions { - gfm_strikethrough_single_tilde: true, - math_text_single_dollar: true, - mdx_expression_parse: None, - mdx_esm_parse: None, - constructs: markdown::Constructs { - attention: true, - autolink: true, - block_quote: true, - character_escape: true, - character_reference: true, - code_indented: true, - code_fenced: true, - code_text: true, - definition: true, - frontmatter: true, - gfm_autolink_literal: true, - gfm_label_start_footnote: true, - gfm_footnote_definition: true, - gfm_strikethrough: true, - gfm_table: true, - gfm_task_list_item: true, - hard_break_escape: true, - hard_break_trailing: true, - heading_atx: true, - heading_setext: true, - html_flow: true, - html_text: true, - label_start_image: true, - label_start_link: true, - label_end: true, - list_item: true, - math_flow: true, - math_text: true, - mdx_esm: false, - mdx_expression_flow: false, - mdx_expression_text: false, - mdx_jsx_flow: false, - mdx_jsx_text: false, - thematic_break: true, - }, - }, - ) - .map_err(|e| miette::miette!(e.reason))?; - Ok(Node::from_mdast_node(root)) + parse_nodes(content) + } +} + +/// Parses `content` with the parser selected by the `native-parser` feature. +fn parse_nodes(content: &str) -> miette::Result> { + #[cfg(feature = "native-parser")] + { + crate::parser::parse(content) } + #[cfg(not(feature = "native-parser"))] + { + parse_with_markdown_rs(content) + } +} + +/// Parses `content` as MDX with the parser selected by the `native-parser` feature. +fn parse_mdx_nodes(content: &str) -> miette::Result> { + #[cfg(feature = "native-parser")] + { + crate::parser::parse_mdx(content) + } + #[cfg(not(feature = "native-parser"))] + { + parse_mdx_with_markdown_rs(content) + } +} + +/// Parses `content` as MDX with `markdown-rs` and converts the `mdast` tree into nodes. +#[cfg(any(not(feature = "native-parser"), test))] +pub(crate) fn parse_mdx_with_markdown_rs(content: &str) -> miette::Result> { + let root = markdown::to_mdast(content, &markdown::ParseOptions::mdx()).map_err(|e| miette!(e.reason))?; + Ok(Node::from_mdast_node(root)) +} + +/// Parses `content` with `markdown-rs` and converts the `mdast` tree into nodes. +#[cfg(any(not(feature = "native-parser"), test))] +pub(crate) fn parse_with_markdown_rs(content: &str) -> miette::Result> { + let root = markdown::to_mdast( + content, + &markdown::ParseOptions { + gfm_strikethrough_single_tilde: true, + math_text_single_dollar: true, + mdx_expression_parse: None, + mdx_esm_parse: None, + constructs: Constructs { + attention: true, + autolink: true, + block_quote: true, + character_escape: true, + character_reference: true, + code_indented: true, + code_fenced: true, + code_text: true, + definition: true, + frontmatter: true, + gfm_autolink_literal: true, + gfm_label_start_footnote: true, + gfm_footnote_definition: true, + gfm_strikethrough: true, + gfm_table: true, + gfm_task_list_item: true, + hard_break_escape: true, + hard_break_trailing: true, + heading_atx: true, + heading_setext: true, + html_flow: true, + html_text: true, + label_start_image: true, + label_start_link: true, + label_end: true, + list_item: true, + math_flow: true, + math_text: true, + mdx_esm: false, + mdx_expression_flow: false, + mdx_expression_text: false, + mdx_jsx_flow: false, + mdx_jsx_text: false, + thematic_break: true, + }, + }, + ) + .map_err(|e| miette!(e.reason))?; + Ok(Node::from_mdast_node(root)) } /// Returns the shared `Options` used for both `Markdown::to_html` and the diff --git a/crates/mq-markdown/src/node.rs b/crates/mq-markdown/src/node.rs index 48871b594..eb42d8be1 100644 --- a/crates/mq-markdown/src/node.rs +++ b/crates/mq-markdown/src/node.rs @@ -3,6 +3,7 @@ use crate::node::attr_value::{ attr_keys::{self, CHILDREN}, }; use itertools::Itertools; +#[cfg(any(not(feature = "native-parser"), test))] use markdown::mdast::{self}; use smol_str::SmolStr; use std::{ @@ -145,8 +146,9 @@ impl ColorTheme<'_> { } } -type Level = u8; +pub(crate) type Level = u8; +#[cfg(any(not(feature = "native-parser"), test))] pub const EMPTY_NODE: Node = Node::Text(Text { value: String::new(), position: None, @@ -183,7 +185,7 @@ impl Display for ListStyle { derive(serde::Serialize, serde::Deserialize), serde(rename_all = "camelCase") )] -pub struct Url(String); +pub struct Url(pub(crate) String); #[derive(Debug, Clone, PartialEq, Default)] pub enum UrlSurroundStyle { @@ -220,7 +222,7 @@ pub enum TitleSurroundStyle { derive(serde::Serialize, serde::Deserialize), serde(rename_all = "camelCase") )] -pub struct Title(String); +pub struct Title(pub(crate) String); impl Display for Title { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { @@ -255,6 +257,7 @@ pub enum TableAlignKind { None, } +#[cfg(any(not(feature = "native-parser"), test))] impl From for TableAlignKind { fn from(value: mdast::AlignKind) -> Self { match value { @@ -880,6 +883,7 @@ pub struct Point { pub column: usize, } +#[cfg(any(not(feature = "native-parser"), test))] impl From for Position { fn from(value: markdown::unist::Position) -> Self { Self { @@ -2689,6 +2693,18 @@ impl Node { } } + /// Builds a blockquote node, or a callout node when the `callout` feature is enabled and the content is one. + pub(crate) fn from_blockquote(values: Vec, position: Option) -> Node { + #[cfg(feature = "callout")] + { + Self::try_parse_callout(values, position) + } + #[cfg(not(feature = "callout"))] + { + Self::Blockquote(Blockquote { values, position }) + } + } + /// Tries to parse a `Blockquote`'s converted nodes as an Obsidian callout. /// /// Returns a `Callout` node when the first text starts with `[!TYPE]`, otherwise @@ -3090,6 +3106,7 @@ impl Node { } } + #[cfg(any(not(feature = "native-parser"), test))] pub(crate) fn from_mdast_node(node: mdast::Node) -> Vec { match node { mdast::Node::Root(root) => root @@ -3184,15 +3201,7 @@ impl Node { }, mdast::Node::Blockquote(mdast::Blockquote { ref position, .. }) => { let pos = position.clone().map(|p| p.into()); - let values = Self::mdast_children_to_node(node); - #[cfg(feature = "callout")] - { - vec![Self::try_parse_callout(values, pos)] - } - #[cfg(not(feature = "callout"))] - { - vec![Self::Blockquote(Blockquote { values, position: pos })] - } + vec![Self::from_blockquote(Self::mdast_children_to_node(node), pos)] } mdast::Node::Definition(mdast::Definition { url, @@ -3483,6 +3492,7 @@ impl Node { } /// Converts the children of `node`, moving them out to avoid deep-cloning the subtree at every level. + #[cfg(any(not(feature = "native-parser"), test))] fn mdast_children_to_node(mut node: mdast::Node) -> Vec { match node.children_mut() { Some(children) => std::mem::take(children) @@ -3493,6 +3503,7 @@ impl Node { } } + #[cfg(any(not(feature = "native-parser"), test))] fn mdast_list_items(list: &mdast::List, level: Level) -> Vec { let mut result = Vec::new(); for node in &list.children { @@ -3537,6 +3548,7 @@ impl Node { /// /// The enclosing [`Self::mdast_list_items`] call emits those children at their own level, /// so skipping them here prevents constructing and discarding the same subtree first. + #[cfg(any(not(feature = "native-parser"), test))] fn mdast_list_item_node(list_item: &mdast::ListItem, list: &mdast::List, level: Level) -> Node { let values = list_item .children diff --git a/crates/mq-markdown/src/parser.rs b/crates/mq-markdown/src/parser.rs new file mode 100644 index 000000000..e174880e8 --- /dev/null +++ b/crates/mq-markdown/src/parser.rs @@ -0,0 +1,769 @@ +//! Native Markdown parser that builds [`Node`] values directly, without going through `mdast`. +//! +//! Parsing runs in phases: [`block`] resolves the block structure line by line into a tree, +//! [`resolve`] collects the definitions and turns the tree into nodes, and [`inline`] parses the raw +//! text of paragraphs, headings and table cells along the way. +//! +//! The output, positions included, is the same as that of `markdown-rs` for CommonMark, GFM, frontmatter, +//! math and MDX (without a JavaScript parser, so expressions only need balanced braces). This holds for +//! every example of the CommonMark and GFM specifications, and for randomly generated documents, except: +//! +//! - lazy continuation lines after definitions, footnotes or thematic breaks in containers, in containers +//! nested more than two deep, or with several container markers on a line, and unclosed fences in +//! containers whose end depends on the line that follows +//! - documents that start with a `---` or `+++` line that never closes, and inputs on which +//! `markdown-rs` panics +//! - a few tab quirks of `markdown-rs`: a tab that a container only partly consumes keeps its rest in +//! text values +//! - character references in an image destination that contains an email address +//! - MDX text expressions and tags that span lines interrupted by container markers +mod block; +mod code; +mod definition; +mod html_flow; +mod inline; +mod line; +mod mdx; +mod mdx_flow; +mod resolve; +mod table; +mod tree; + +use crate::node::Node; + +/// Parses `content` into a flat list of nodes. +pub(crate) fn parse(content: &str) -> miette::Result> { + resolve::resolve(block::parse(content, false), false).map_err(|message| miette::miette!(message)) +} + +/// Parses `content` as MDX: no indented code, HTML, autolinks or GFM, but expressions and JSX. +pub(crate) fn parse_mdx(content: &str) -> miette::Result> { + resolve::resolve(block::parse(content, true), true).map_err(|message| miette::miette!(message)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::markdown::{parse_mdx_with_markdown_rs, parse_with_markdown_rs}; + use proptest::prelude::*; + use proptest::test_runner::{RngAlgorithm, TestRng, TestRunner}; + use rstest::rstest; + + /// `Node`'s `PartialEq` compares rendered output, so compare `Debug` output to include positions. + fn assert_same_as_markdown_rs(input: &str) { + let expected = parse_with_markdown_rs(input).unwrap(); + let actual = parse(input).unwrap(); + assert_eq!(format!("{actual:#?}"), format!("{expected:#?}"), "input: {input:?}"); + } + + /// Compares the MDX parsers. Invalid MDX has to fail in both. + fn assert_same_mdx(input: &str) { + let expected = parse_mdx_with_markdown_rs(input); + let actual = parse_mdx(input); + match (expected, actual) { + (Ok(expected), Ok(actual)) => { + assert_eq!(format!("{actual:#?}"), format!("{expected:#?}"), "input: {input:?}"); + } + (Err(_), Err(_)) => {} + (expected, actual) => panic!("input: {input:?}\n markdown-rs: {expected:?}\n native: {actual:?}"), + } + } + + #[rstest] + #[case::jsx_self("")] + #[case::jsx_attrs("")] + #[case::jsx_member("")] + #[case::jsx_namespace("")] + #[case::jsx_fragment("<>")] + #[case::jsx_text_pair("x")] + #[case::jsx_flow_pair("\n\nx\n\n")] + #[case::jsx_flow_indented_child("\n x\n")] + #[case::jsx_inline("a c d")] + #[case::jsx_inline_self("a d")] + #[case::expr_flow("{a}")] + #[case::expr_nested("{a {b} c}")] + #[case::expr_multiline("{a\nb}")] + #[case::expr_multiline_indented("{a\n b\n c}")] + #[case::expr_text("x {a} y")] + #[case::expr_then_tag("{a} ")] + #[case::tag_then_expr(" {a}")] + #[case::tag_then_text(" x")] + #[case::expr_then_text("{a}x")] + #[case::jsx_in_expr_child("{b}")] + #[case::jsx_unclosed_text("b")] + #[case::jsx_mismatch("")] + #[case::lt_space("a < b")] + #[case::lt_digit("a <3")] + #[case::attr_no_value("")] + #[case::jsx_nested_flow("\n\n\n")] + #[case::jsx_in_quote("> \n> x\n> ")] + #[case::jsx_in_list("- \n x\n ")] + #[case::attr_quote_inside("")] + #[case::two_tags("")] + #[case::two_text_elements("xy")] + #[case::interrupt_paragraph_tag("a\n")] + #[case::interrupt_paragraph_expr("a\n{b}")] + #[case::two_expressions("{a}\n{b}")] + #[case::expr_in_flow_element("\n\n{b}\n\n")] + #[case::indented_tag(" ")] + #[case::very_indented_tag(" ")] + #[case::heading_jsx("# h b")] + #[case::emphasis_crossing("*x*")] + #[case::emphasis_crossing_bad("*x*")] + #[case::link_jsx("[x](y)")] + #[case::code_not_jsx("`` {b}")] + #[case::escaped("\\ \\{b}")] + #[case::attr_entities("")] + #[case::attr_multiline_literal("")] + #[case::closing_spaces("x")] + #[case::lt_space_name("< a>")] + #[case::self_closing_space("")] + #[case::attr_on_next_line("")] + #[case::dashed_name("")] + #[case::dashed_attr("")] + #[case::bad_name_char("")] + #[case::expr_unclosed("{a")] + #[case::expr_empty("{}")] + #[case::expr_space("{ }")] + #[case::empty_flow_element("\n")] + #[case::text_element_unclosed_line("x\n")] + #[case::stray_close("")] + #[case::stray_close_text("x ")] + #[case::element_with_emphasis("*b*")] + #[case::flow_element_with_emphasis("\n*b*\n")] + #[case::flow_element_with_list("\n- b\n")] + #[case::list_items_unbalanced("- \n- ")] + #[case::spaced_equals("")] + #[case::heading("# h\n\ntext *a* [b](c) `d`\n")] + #[case::indented_is_text(" not code\n")] + #[case::indented_heading(" # h\n")] + #[case::no_autolink("")] + #[case::no_gfm("| a |\n|-|\n\n~a~ www.a.b [^a]\n\n- [ ] a\n")] + #[case::no_math("$a$\n\n$$\na\n$$\n")] + #[case::no_frontmatter("---\na\n---\n")] + #[case::esm_is_text("import a from 'b'\n\nexport const c = 1\n")] + #[case::definition("[a]: /u\n\n[a]\n")] + #[case::list_deep_indent("- a\n")] + #[case::fence("```rust\ncode\n```\n")] + fn mdx_matches_markdown_rs(#[case] input: &str) { + assert_same_mdx(input); + } + + #[rstest] + #[case::empty("")] + #[case::blank_lines("\n\n \n")] + #[case::paragraph("hello\n")] + #[case::paragraph_no_eol("hello")] + #[case::paragraph_multiline("a\nb\nc\n")] + #[case::paragraph_indented_continuation("a\n b\n")] + #[case::paragraph_trailing_space("あい\n うえ \n")] + #[case::two_paragraphs("a\n\nb\n")] + #[case::crlf("a\r\nb\r\n\r\nc\r\n")] + #[case::crlf_fence("```\r\na\r\nb\r\n```\r\n")] + #[case::cr_only("a\rb\r")] + #[case::atx_h1("# title\n")] + #[case::atx_h6("###### title\n")] + #[case::atx_seven("####### title\n")] + #[case::atx_no_space("#title\n")] + #[case::atx_empty("#\n")] + #[case::atx_closing("## title ##\n")] + #[case::atx_closing_no_space("## title##\n")] + #[case::atx_indent(" ## title ## \n")] + #[case::atx_multibyte("# あ h #\n")] + #[case::atx_interrupts_paragraph("a\n# b\n")] + #[case::setext_h1("Title\n===\n")] + #[case::setext_h2("a\n ---\n")] + #[case::setext_multiline("a\nb\n---\n")] + #[case::thematic_star("***\n")] + #[case::thematic_dash_spaced("- - -\n")] + #[case::thematic_underscore(" ___ \n")] + #[case::thematic_two("**\n")] + #[case::fence_lang_meta("```rust title\nlet a;\n```\n")] + #[case::fence_indented(" ```\n a\n b\n ```\n")] + #[case::fence_tilde("~~~\na\n~~~\n")] + #[case::fence_unclosed("```\na\n")] + #[case::fence_empty("para\n\n\n```\n```\n")] + #[case::fence_longer_close("```\na\n`````\n")] + #[case::fence_shorter_close("````\na\n```\nb\n````\n")] + #[case::fence_blank_inside("```\na\n\nb\n```\n")] + #[case::fence_backtick_info("``` a`b\nc\n")] + #[case::fence_interrupts_paragraph("a\n```\nb\n```\n")] + #[case::indented_code(" code\n\n more\n\nx\n")] + #[case::quote("> a\n> b\n")] + #[case::quote_indented_lazy(" > a\nb\n")] + #[case::quote_heading("> # h\n>\n> c\n")] + #[case::quote_no_space(">a\n")] + #[case::quote_empty(">\n")] + #[case::quote_nested("> > a\n> b\n")] + #[case::quote_lazy_nested("> > a\nb\n")] + #[case::quote_blank_ends("> a\n\n> b\n")] + #[case::quote_fence("> ```\n> a\n> ```\n")] + #[case::quote_no_lazy_after_fence("> ```\n> a\nb\n")] + #[case::quote_interrupts_paragraph("a\n> b\n")] + #[case::quote_thematic_not_lazy("> a\n---\n")] + #[case::quote_multibyte("> あ\n> い\n")] + #[case::bullet("- a\n- b\n")] + #[case::bullet_loose("- a\n\n- b\n")] + #[case::bullet_indent(" - a\n")] + #[case::bullet_mixed_markers("* a\n+ b\n")] + #[case::ordered("1. a\n2. b\n")] + #[case::ordered_paren("5) a\n")] + #[case::ordered_zero("0. a\n")] + #[case::ordered_too_long("1234567890. a\n")] + #[case::ordered_mixed_delimiters("1. a\n2) b\n")] + #[case::task("- [ ] x\n- [x] y\n- [X] z\n")] + #[case::task_empty("- [ ]\n")] + #[case::task_no_space("- [ ]x\n")] + #[case::nested("- a\n - b\n c\n")] + #[case::nested_ordered("1. a\n 1. b\n 2. c\n2. d\n")] + #[case::nested_dedent("- a\n - b\n- c\n")] + #[case::two_paragraphs("- a\n\n b\n")] + #[case::two_paragraphs_then_item("- a\n\n b\n- c\n")] + #[case::empty_item("-\n")] + #[case::empty_item_content("-\n foo\n")] + #[case::empty_item_blank_then_text("-\n\n foo\n")] + #[case::empty_items("-\n-\n")] + #[case::wide_marker_gap("- a\n b\n")] + #[case::code_in_item("- a\n")] + #[case::lazy("- a\nb\n")] + #[case::lazy_after_blank("- a\n\nb\n")] + #[case::list_interrupts_paragraph("a\n- b\n")] + #[case::ordered_two_no_interrupt("a\n2. b\n")] + #[case::ordered_one_interrupts("a\n1. b\n")] + #[case::empty_item_no_interrupt("a\n-\n")] + #[case::thematic_over_list("- - -\n")] + #[case::list_in_quote("> - a\n> - b\n")] + #[case::quote_in_list("- > a\n > b\n")] + #[case::quote_in_list_level("- a\n > - b\n")] + #[case::heading_in_item("- # h\n text\n")] + #[case::fence_in_item("- ```\n a\n ```\n")] + #[case::multibyte_item("- あ\n- い\n")] + #[case::list_after_paragraph_blank("a\n\n- b\n")] + #[case::two_lists_split("- a\n\n\n- b\n")] + #[case::deep_quote(">>>>>> a\n")] + #[case::fence_unclosed_no_eol("```\na")] + #[case::fence_unclosed_in_item("- ```\n a\n")] + #[case::fence_unclosed_in_item_no_eol("- ```\n a")] + #[case::fence_unclosed_in_quote_no_eol("> ```\n> a")] + #[case::fence_unclosed_in_quote_eof("> ```\n> a\n")] + #[case::fence_unclosed_trailing_blank("```\na\n\n")] + #[case::fence_unclosed_item_then_para("- ```\n a\nb\n")] + #[case::item_code_then_unindented_text("- ===\n\n a\n===")] + #[case::quote_setext_dashes("> a\n> ---\n> ---\n> a")] + #[case::table_basic("| a | b |\n|---|:-:|\n| 1 | 2 |\n")] + #[case::table_no_edge_pipes("a|b\n-|-\n1|2|3\n")] + #[case::table_short_row("|a|b|\n|-|-|\n|1|\n")] + #[case::table_escaped_pipe("| a \\| b | c |\n|--|--|\n")] + #[case::table_indented(" | a |\n |---|\n | b |\n")] + #[case::table_text_after("| a |\n|---|\ntext after\n")] + #[case::table_in_quote("> | a |\n> |---|\n> | b |\n")] + #[case::table_after_paragraph("x\n| a |\n|---|\n")] + #[case::table_column_mismatch("| a | b |\n|---|\n")] + #[case::table_bad_delimiter("|a|\n|:|\n")] + #[case::table_then_blank_para("|a|\n|-|\n\nnext\n")] + #[case::table_then_quote("|a|\n|-|\n> q\n")] + #[case::table_then_heading("|a|\n|-|\n# h\n")] + #[case::table_then_fence("|a|\n|-|\n```\nx\n```\n")] + #[case::table_then_list("|a|\n|-|\n- x\n")] + #[case::table_two("|a|\n|-|\n\n|b|\n|-|\n")] + #[case::table_align_spaces("|a|b|\n|:-|-:|\n| x | y|\n")] + #[case::table_indented_code_row("|a|\n|-|\n|b|\n |c|\n")] + #[case::table_no_body("| a |\n| - |\n")] + #[case::table_empty_cells("||\n|-|\n")] + #[case::table_lone_pipe_row("|a|\n|-|\n|\n")] + #[case::table_no_trailing_pipe("| a | b\n|-|-\n| c | d\n")] + #[case::table_multibyte("| あ | い |\n|---|---|\n| う | え |\n")] + #[case::not_table_setext("Title\n---\n")] + #[case::not_table_no_pipe_delim("a|b\n---\n")] + #[case::table_header_no_pipe("a\n|-|\n")] + #[case::table_one_col_dash("|a|\n-\n")] + #[case::inline_escape("a\\*b")] + #[case::inline_escape_nonpunct("a\\qb")] + #[case::inline_entities("a&b © # A &unknown; &")] + #[case::inline_emphasis_all("a *b* **c** _d_ __e__ ***f***")] + #[case::inline_code("`code` `` a`b `` ` a `")] + #[case::inline_code_unclosed("`a ``b`")] + #[case::inline_hard_break_spaces("a \nb")] + #[case::inline_hard_break_backslash("a\\\nb")] + #[case::inline_soft_break("a\nb")] + #[case::inline_soft_break_trailing_space("a \nb")] + #[case::inline_emphasis_multiline("*a\nb*")] + #[case::inline_strike("a ~b~ ~~c~~ ~~~d~~~")] + #[case::inline_math("$x$ $$y$$ $ a")] + #[case::inline_autolinks("
")] + #[case::inline_links("[a](http://x \"t\") [b][c] [d][] [e] ![f](g) ![h][i]")] + #[case::inline_link_forms("[a]( 'x') [d]( e ) [f](g (h)) [i](j \"k\\\"l\") [m](n&o)")] + #[case::inline_link_multiline("[a](b\n\"t\") [c](d")] + #[case::inline_image_alt("![a *b* `c`](d) ![e ![f](g)](h)")] + #[case::inline_link_in_link("[a [b](c) d](e)")] + #[case::inline_emphasis_link("*[a](b)* **[c][a]**")] + #[case::inline_html("
t ")] + #[case::inline_html_misc(" ")] + #[case::inline_nested_emphasis("*a **b** c* **a *b* c**")] + #[case::inline_intraword("a_b_c a*b*c _a_b")] + #[case::inline_unmatched("*a **b _c ~d [e ![f")] + #[case::inline_rule_of_three("*foo**bar**baz* ***a** b*")] + #[case::inline_heading("# a *b* `c`\n")] + #[case::inline_setext_heading("a *b*\nc\n===\n")] + #[case::inline_list_item("- a *b*\n c **d**\n")] + #[case::inline_table_cell("| a *b* | `c\\|d` |\n|-|-|\n")] + #[case::inline_multibyte("あ*い*う `え` [お](か)")] + #[case::inline_crlf_break("a \r\nb\r\n")] + #[case::math_multiline("x$\nx$")] + #[case::decl_email("x")] + #[case::code_multiline("x`\nx`")] + #[case::def_basic("[a]: b")] + #[case::def_title("[a]: 'T'")] + #[case::def_next_lines("[a]:\nb\n\"t\"")] + #[case::def_title_junk_next_line("[a]: b\n\"t\" x")] + #[case::def_title_junk_same_line("[a]: b \"t\" x")] + #[case::def_then_paragraph("[a]: b\nc")] + #[case::def_two_then_text("[a]: b\n[c]: d\ntext")] + #[case::def_not_at_start("text\n[a]: b")] + #[case::def_then_setext_equals("[a]: b\n=== ")] + #[case::def_then_setext_dashes("[a]: b\n---")] + #[case::def_indent1(" [a]: b")] + #[case::def_indent3(" [a]: b")] + #[case::def_indent4(" [a]: b")] + #[case::def_in_quote("> [a]: b")] + #[case::def_in_list("- [a]: b")] + #[case::def_no_dest("[a]:")] + #[case::def_empty_label("[]: b")] + #[case::def_blank_label("[ ]: b")] + #[case::def_escaped_label("[a\\]b]: c")] + #[case::def_multiline_title("[a]: b \"t\nt\"")] + #[case::def_title_indented("[a]: b\n \"t\"")] + #[case::def_no_space("[a]:b")] + #[case::def_dest_junk("[a]: b c")] + #[case::def_duplicate_labels("[A b]: c\n[a b]: d")] + #[case::def_empty_angle("[a]: <>")] + #[case::def_quote_in_dest("[a]: 'x")] + #[case::def_use_shortcut("[a]: /u\n\n[a] [a][] [b][a] [a][b] [A b]")] + #[case::def_use_normalized("[ab cd]: /u\n\n[AB CD] [ab\ncd]")] + #[case::def_use_before("[a] and [b][a]\n\n[a]: /u \"T\"")] + #[case::def_use_in_link_text("[a]: b\n\n[a](x) [a]")] + #[case::def_use_image("[x]: /u\n\n![x] ![y][x] ![x][]")] + #[case::def_link_in_emphasis("[a]: b\n\n*[a]* **[c][a]**")] + #[case::fn_basic("[^a]: b")] + #[case::fn_lazy("[^a]: b\nc")] + #[case::fn_indented_continuation("[^a]: b\n c")] + #[case::fn_two_paragraphs("[^a]: b\n\n c")] + #[case::fn_two_paragraphs_then_text("[^a]: b\n\n c\n\nd")] + #[case::fn_two_space_continuation("[^a]: b\n c")] + #[case::fn_empty("[^a]:")] + #[case::fn_empty_space("[^a]: ")] + #[case::fn_content_next_line("[^a]:\n b")] + #[case::fn_two_adjacent("[^a]: b\n[^c]: d")] + #[case::fn_two_separated("[^a]: b\n\n[^c]: d")] + #[case::fn_label_space("[^ a]: b")] + #[case::fn_label_inner_space("[^a b]: b")] + #[case::fn_label_empty("[^]: b")] + #[case::fn_indent1(" [^a]: b")] + #[case::fn_indent4(" [^a]: b")] + #[case::fn_in_quote("> [^a]: b")] + #[case::fn_in_list("- [^a]: b")] + #[case::fn_then_quote("[^a]: b\n> q")] + #[case::fn_then_list("[^a]: b\n- x")] + #[case::fn_heading("[^a]: # h")] + #[case::fn_list("[^a]: - x\n - y")] + #[case::fn_fence("[^a]: b\n ```\n x\n ```")] + #[case::fn_ref_case("[^A]: b\n\nx[^a]")] + #[case::fn_no_space("[^a]:b")] + #[case::fn_interrupts_paragraph("text\n[^a]: b")] + #[case::fn_then_dashes("[^a]: b\n---")] + #[case::fn_then_equals("[^a]: b\n===")] + #[case::fn_ref_basic("[^a]\n\n[^a]: note")] + #[case::fn_ref_no_def("[^b] no def")] + #[case::fn_ref_inline("x[^a] and [^a]!\n\n[^a]: note")] + #[case::fn_ref_in_link_text("[a[^a]](b)\n\n[^a]: note")] + #[case::html_div("
\nx\n
\n\ny")] + #[case::html_indented("
\n x\n")] + #[case::html_oneline("
x
")] + #[case::html_script("\nz")] + #[case::html_pre("
\na\n\nb
\nc")] + #[case::html_style_inline_end(" y\nz")] + #[case::html_comment("\nx")] + #[case::html_instruction("\ny")] + #[case::html_declaration("\nx")] + #[case::html_cdata("\ny")] + #[case::html_complete_tag("\ny")] + #[case::html_tag_then_text("y")] + #[case::html_interrupt_basic("text\n\nz")] + #[case::html_multiline_open("\nz")] + #[case::html_in_quote(">
\n> x")] + #[case::html_in_list("-
\n x")] + #[case::html_uppercase("
\nx")] + #[case::html_self_closing("
\nx")] + #[case::html_ins("\nx")] + #[case::html_br("
\nx")] + #[case::html_textarea("")] + #[case::html_table_two_blocks("\n\n\n
")] + #[case::html_img_self_close("\nz")] + #[case::html_img_then_text(" y")] + #[case::html_multiline_tag_inline("\nz")] + #[case::html_indent4("
")] + #[case::html_comment_then_text("y\nz")] + #[case::html_then_code_line("
\n code")] + #[case::html_comment_short("\nx")] + #[case::html_instruction_short("\nx")] + #[case::html_lazy_quote(">
\nx")] + #[case::html_attr_forms("\nx")] + #[case::html_bad_attr("\nx")] + #[case::fm_yaml("---\na: b\n---\ntext")] + #[case::fm_yaml_only("---\na: b\n---")] + #[case::fm_empty("---\n---")] + #[case::fm_unclosed("---\na\n")] + #[case::fm_blank_content("---\n\n---\nx")] + #[case::fm_toml("+++\na = 1\n+++\nx")] + #[case::fm_indented(" ---\na\n---")] + #[case::fm_close_trailing_space("---\na\n--- \nx")] + #[case::fm_close_longer("---\na\n----\nx")] + #[case::fm_open_trailing_space("--- \na\n---")] + #[case::fm_open_junk("---x\na\n---")] + #[case::fm_not_first("text\n---\na\n---")] + #[case::fm_blank_before("\n---\na\n---")] + #[case::fm_multiline_content("---\na: 1\n\nb: 2\n---\n# h")] + #[case::fm_hr_after("---\na\n---\n\n---")] + #[case::math_basic("$$\na\n$$")] + #[case::math_then_text("$$\na\n$$\nx")] + #[case::math_meta("$$ meta\na\n$$")] + #[case::math_unclosed("$$\na")] + #[case::math_longer_open("$$$\na\n$$")] + #[case::math_longer_close("$$\na\n$$$$")] + #[case::math_indented(" $$\n a\n b\n $$")] + #[case::math_empty("$$\n$$")] + #[case::math_inline_not_block("$$a$$")] + #[case::math_info_no_close("$$a\n$$")] + #[case::math_in_quote("> $$\n> a\n> $$")] + #[case::math_in_list("- $$\n a\n $$")] + #[case::math_close_junk("$$\na\n$$ x")] + #[case::math_interrupts_paragraph("text\n$$\na\n$$")] + #[case::math_blank_inside("$$\n\na\n$$")] + #[case::math_single_dollar_line("$ $\nx")] + #[case::crlf_frontmatter("---\r\na\r\n---\r\nx")] + #[case::break_end_indented("a \n b")] + #[case::text_end_indented_continuation("*a*\n ")] + #[case::soft_end_before_node("a\n `b`")] + #[case::break_in_quote("> a \n> b")] + #[case::text_eol_in_footnote("[^a]: x \n [a]")] + #[case::math_whitespace_content(" $$\n \n $$")] + #[case::math_whitespace_content2(" $$\n \n $$")] + #[case::table_indented_no_pipe(" a|b\n|-|:-:|")] + #[case::tab_nested_list("- a\n\t- b\n\t\t- c\n")] + #[case::tab_nested_ordered("1. a\n\t1. b\n\t\t1. c\n")] + #[case::tab_list_code("- a\n\n\t\tcode\n")] + #[case::tab_in_text("a\tb\t*c*\n")] + #[case::tab_code_block("\tcode\n\t\tmore\n")] + #[case::tab_after_marker("-\ta\n")] + #[case::atx_leading_sequences("# # a")] + #[case::atx_leading_sequences2("## # a")] + #[case::atx_only_sequences("# # #")] + #[case::atx_no_space_sequence("# #a")] + #[case::atx_double_sequence("# ## a")] + #[case::atx_tab_sequence("#\t# a")] + #[case::atx_closing_and_leading("# # a #")] + #[case::atx_inner_hash("# a # b")] + #[case::quote_footnote_trailing_space("- a\n > [^a]: x\n ")] + #[case::indented_code_trailing_blank(" code\n\n\n")] + fn matches_markdown_rs(#[case] input: &str) { + assert_same_as_markdown_rs(input); + } + + /// Lines built from container prefixes and block-level bodies only, so inline syntax (not + /// implemented yet) never appears. Constructs whose markdown-rs positions depend on quirks of the + /// following lines are left to the hand-written cases: empty items and quotes, unclosed fences, + /// indented code next to containers, ordered lists that do not start at 1, and lazy continuation + /// inside nested containers, more than one container marker on a line, and runs of setext + /// underlines (`===` is only used by the hand-written cases). + fn block_input() -> impl Strategy { + // At most one container marker per line; deeper nesting comes from indentation. + let indent = prop::sample::select(vec!["", " ", " ", " "]); + let marker = prop::sample::select(vec!["", "> ", ">", "- ", "* ", "+ ", "1. ", "1) "]); + let body = prop::sample::select(vec![ + "a", + "b", + "# a", + "#", + "## b ##", + "---", + "***", + "| a | b |", + "|-|:-:|", + "|1|2|3|", + "a|b", + "[a]: /u", + "[a]: /u 'T'", + "[^a]: x", + "", + "x [a] [b][a] [^a]", + "*a* `b`", + "a ", + "a\\", + "+++", + ]); + let block = (indent, marker, body).prop_map(|(indent, marker, body)| format!("{indent}{marker}{body}")); + let blank = prop::sample::select(vec!["", " "]).prop_map(str::to_string); + // Fences are always closed here. + let fence = ( + prop::sample::select(vec!["", " ", " ", " "]), + prop::sample::select(vec![ + ("```", "```"), + ("```rust x", "````"), + ("~~~", "~~~"), + ("$$", "$$"), + ("$$ m", "$$$"), + ]), + prop::sample::select(vec!["a", "", "> b", "- c"]), + ) + .prop_map(|(indent, (open, close), content)| format!("{indent}{open}\n{indent}{content}\n{indent}{close}")); + let line = prop_oneof![8 => block, 2 => blank, 1 => fence]; + (prop::collection::vec(line, 0..8), any::()).prop_map(|(lines, trailing_eol)| { + let mut input = lines.join("\n"); + if trailing_eol { + input.push('\n'); + } + input + }) + } + + fn env_number(name: &str) -> Option { + std::env::var(name).ok()?.parse().ok() + } + + /// Runs `strategy` with a fixed seed so the outcome never varies between runs. `PROPTEST_CASES` + /// and `PARSER_TEST_SEED` widen the exploration, which finds more differences in the cases listed + /// on [`block_input`] that are not covered yet. + fn check(strategy: impl Strategy) { + check_with(strategy, parse_with_markdown_rs, assert_same_as_markdown_rs); + } + + fn check_with( + strategy: impl Strategy, + reference: fn(&str) -> miette::Result>, + assert_same: fn(&str) -> R, + ) { + let config = ProptestConfig { + cases: env_number("PROPTEST_CASES").unwrap_or(3000), + failure_persistence: None, + ..ProptestConfig::default() + }; + let seed = env_number("PARSER_TEST_SEED").unwrap_or(7); + let mut runner = TestRunner::new_with_rng(config, TestRng::from_seed(RngAlgorithm::ChaCha, &[seed; 32])); + let result = runner.run(&strategy, |input| { + // markdown-rs misparses what follows an opening `---` or `+++` that never closes, and even + // panics on some inputs. Those cannot be compared. + let mut lines = input.lines().map(str::trim_end); + if let Some(first @ ("---" | "+++")) = lines.next() { + prop_assume!(lines.any(|line| line == first)); + } + let outcome = std::panic::catch_unwind(|| reference(&input)); + prop_assume!(outcome.is_ok()); + assert_same(&input); + Ok(()) + }); + if let Err(error) = result { + panic!("{error}"); + } + } + + #[test] + fn block_structure_matches_markdown_rs() { + check(block_input()); + } + + /// Lines of inline syntax. Each line starts with `x` so that no line is a block on its own. + fn inline_input() -> impl Strategy { + let token = prop::sample::select(vec![ + "a", + "b", + "foo", + " ", + " ", + "*", + "**", + "***", + "_", + "__", + "~", + "~~", + "`", + "``", + "[", + "]", + "(", + ")", + "![", + "<", + ">", + "\\", + "&", + "#", + "$", + "$$", + "http://a.b", + "www.a.b", + "a@b.co", + "\"", + "'", + ".", + ",", + "!", + "-", + "/", + "é", + "あ", + "\n", + "](", + "](x)", + "][", + "]:", + "", + "", + "", + "", + ]); + prop::collection::vec(token, 1..14).prop_map(|tokens| { + let mut input = String::from("x"); + for token in tokens { + input.push_str(token); + if token == "\n" { + input.push('x'); + } + } + input + }) + } + + /// Lines of MDX: tags, expressions and the Markdown around them. + fn mdx_input() -> impl Strategy { + let token = prop::sample::select(vec![ + "<", + ">", + "/", + "{", + "}", + "a", + "b", + "c", + " ", + " ", + "=", + "\"", + "'", + ".", + ":", + "-", + "*", + "_", + "`", + "[", + "]", + "(", + ")", + "&", + "", + "", + "", + "", + "<>", + "", + "{x}", + "{x {y}}", + "# ", + "- ", + "> ", + "\\", + "!", + "|", + "\n", + "\n\n", + "\n ", + "", + "", + "", + "{...c}", + "é", + "1. ", + ]); + prop::collection::vec(token, 1..16).prop_map(|tokens| tokens.concat()) + } + + #[test] + fn mdx_syntax_matches_markdown_rs() { + check_with(mdx_input(), parse_mdx_with_markdown_rs, assert_same_mdx); + } + + #[test] + fn inline_syntax_matches_markdown_rs() { + check(inline_input()); + } + + const SPEC_URL: &str = + "https://raw.githubusercontent.com/github/cmark-gfm/828322d1ee4facdab56f0d3edccb13e9af90dcd2/test/spec.txt"; + + /// The markdown of each example of the `CommonMark` and GFM `spec.txt`, with its number. + fn spec_examples(text: &str) -> Vec<(usize, String)> { + let fence = "`".repeat(32); + let open = format!("{fence} example"); + let mut examples = Vec::new(); + let mut lines = text.lines(); + + while let Some(line) = lines.next() { + if line != open { + continue; + } + let markdown = lines.by_ref().take_while(|line| *line != ".").collect::>(); + for line in lines.by_ref() { + if line == fence { + break; + } + } + // `→` stands for a tab in the spec. + examples.push((examples.len() + 1, markdown.join("\n").replace('\u{2192}', "\t") + "\n")); + } + examples + } + + /// Examples of the spec on which the native parser differs from `markdown-rs`, by number. + const KNOWN_DIFFERENCES: &[usize] = &[]; + + /// Compares every example of the spec, from `SPEC_FILE` or fetched over the network. + #[test] + #[ignore = "fetches spec.txt over the network; set SPEC_FILE to use a local copy"] + fn spec_examples_match_markdown_rs() { + let text = match std::env::var("SPEC_FILE") { + Ok(path) => std::fs::read_to_string(path).unwrap(), + Err(_) => ureq::get(SPEC_URL) + .call() + .unwrap() + .into_body() + .read_to_string() + .unwrap(), + }; + let examples = spec_examples(&text); + assert!(examples.len() > 600, "only {} examples were found", examples.len()); + + if let Ok(number) = std::env::var("SPEC_SHOW") { + let (_, markdown) = &examples[number.parse::().unwrap() - 1]; + println!("{markdown:?}"); + println!("native: {:?}", parse(markdown).unwrap()); + println!("markdown-rs: {:?}", parse_with_markdown_rs(markdown).unwrap()); + } + + let differing = examples + .iter() + .filter(|(_, markdown)| { + let Ok(expected) = std::panic::catch_unwind(|| parse_with_markdown_rs(markdown)) else { + return false; + }; + format!("{:#?}", parse(markdown).unwrap()) != format!("{:#?}", expected.unwrap()) + }) + .map(|(number, _)| *number) + .collect::>(); + + assert_eq!(differing, KNOWN_DIFFERENCES, "examples that differ from markdown-rs"); + } +} diff --git a/crates/mq-markdown/src/parser/block.rs b/crates/mq-markdown/src/parser/block.rs new file mode 100644 index 000000000..bd5b9030f --- /dev/null +++ b/crates/mq-markdown/src/parser/block.rs @@ -0,0 +1,1197 @@ +//! Block structure parsing: resolves each line into leaf blocks. +//! +//! Containers (blockquote, list) are resolved by stripping their prefix from each line and parsing +//! the remaining lines again, up to [`MAX_DEPTH`] levels. The remaining leaf blocks (HTML, table, +//! definition, frontmatter, math) are not handled yet, so their lines currently fall into paragraphs. + +use super::code::{Fence, fenced_code, indented_code}; +use super::definition; +use super::html_flow::{self, Kind as HtmlKind}; +use super::inline; +use super::line::{Line, split_lines}; +use super::mdx_flow::{ + FlowOutcome, absorbed_until, blocks_lazy_continuation, looks_like_mdx_flow, mdx_flow, probe_mdx_flow, +}; +use super::table; +use super::tree::{Block, FootnoteBlock, InlineBlock, InlineKind, InlineSource, Item, ListBlock, QuoteBlock}; +use crate::node::{HorizontalRule, Html, Node, Point, Position, Toml, Yaml}; + +/// Lines indented by this many columns or more are code, not other blocks. +pub(super) const CODE_INDENT: usize = 4; + +/// Containers nested deeper than this are parsed as plain paragraph text, which bounds recursion. +const MAX_DEPTH: usize = 128; + +/// Which list markers end a paragraph on its second line although they could not interrupt it. +#[derive(Clone, Copy, PartialEq)] +enum Release { + None, + /// Markers with content. + Markers, + /// Every marker. + AllMarkers, +} + +/// How far a paragraph of dashes right after a setext heading extends, in markdown-rs. +#[derive(Clone, Copy, PartialEq)] +enum Dashes { + /// Not such a paragraph. + No, + /// Up to the next indented code line, at the top level. + UntilCode, + /// A single line, inside containers. + OneLine, +} + +/// What the first line of a container cannot start because the container interrupted a paragraph. +#[derive(Clone, Copy, Default)] +struct Interrupt { + /// Indented code cannot interrupt a paragraph. + code: bool, + /// Empty items and ordered items that do not start at 1 cannot interrupt a paragraph. + list: bool, +} + +pub(super) fn parse(src: &str, mdx: bool) -> Vec { + let lines = split_lines(src, mdx); + match frontmatter(&lines).filter(|_| !mdx) { + Some((node, next)) => { + let mut blocks = vec![Block::Node(node)]; + blocks.extend(parse_blocks(&lines[next..], 0, Interrupt::default())); + blocks + } + None => parse_blocks(&lines, 0, Interrupt::default()), + } +} + +/// Frontmatter at the very start of the document: YAML between `---` lines, or TOML between `+++` +/// lines. Without a closing line it is not frontmatter. +fn frontmatter(lines: &[Line<'_>]) -> Option<(Node, usize)> { + let is_fence = |line: &Line<'_>, marker: &str| { + line.text + .strip_prefix(marker) + .is_some_and(|rest| rest.trim_matches([' ', '\t']).is_empty()) + }; + let first = lines.first()?; + let marker = ["---", "+++"].into_iter().find(|marker| is_fence(first, marker))?; + let close = lines.iter().skip(1).position(|line| is_fence(line, marker))? + 1; + + let parts = lines[1..close] + .iter() + .map(|line| (line.text, line.eol)) + .collect::>(); + let value = join_lines(&parts); + let position = Some(Position { + start: first.point(0), + end: lines[close].end(), + }); + let node = if marker == "---" { + Node::Yaml(Yaml { value, position }) + } else { + Node::Toml(Toml { value, position }) + }; + + Some((node, close + 1)) +} + +/// Parses `lines` into blocks. `interrupting` restricts the first line when the container these +/// lines belong to interrupted an open paragraph. +fn parse_blocks(lines: &[Line<'_>], depth: usize, interrupting: Interrupt) -> Vec { + let containers = depth < MAX_DEPTH; + let mut blocks = Vec::new(); + let mut index = 0; + let mut after_paragraph = interrupting; + let mut after_setext = false; + // Set after a container, and whether a blank line or an empty list item preceded the next block. + let mut after_container = false; + let mut after_list = false; + let mut blank_between = false; + + while index < lines.len() { + let interrupting = std::mem::take(&mut after_paragraph); + let follows_setext = std::mem::take(&mut after_setext); + let line = &lines[index]; + if line.is_blank() { + blank_between = true; + index += 1; + continue; + } + let closed_container = std::mem::take(&mut after_container); + let closed_list = std::mem::take(&mut after_list); + let separated = std::mem::take(&mut blank_between); + + let (columns, indent) = line.indent(); + let rest = &line.text[indent..]; + + if columns >= line.code_indent() && !interrupting.code { + index = indented_code(lines, index, closed_list || closed_container && !separated, &mut blocks); + } else if columns >= line.code_indent() { + // Indented code cannot interrupt a paragraph. + let (next, interrupt, setext) = paragraph(lines, index, Release::None, Dashes::No, &mut blocks); + index = next; + after_paragraph = interrupt; + after_setext = setext; + } else if let Some(fence) = Fence::open(rest, !line.mdx) { + // The end of an unclosed fence without content is quirky right after a container. + let own_end = closed_container + && !(separated + && matches!(blocks.last(), Some(Block::List(l)) if l.items.last().is_some_and(|i| i.children.is_empty()))); + index = fenced_code(lines, index, indent, &fence, own_end, &mut blocks); + } else if let Some(depth) = atx_depth(rest) { + blocks.push(atx_heading(line, indent, depth)); + index += 1; + } else if is_thematic_break(rest) && !(follows_setext && is_dash_run(rest)) { + blocks.push(Block::Node(Node::HorizontalRule(HorizontalRule { + position: Some(Position { + start: line.point(0), + end: line.end(), + }), + }))); + index += 1; + } else if containers && blockquote_marker(line).is_some() { + index = blockquote(lines, index, depth, interrupting, &mut blocks); + after_container = true; + } else if let Some(marker) = + ListMarker::parse(line).filter(|marker| containers && (!interrupting.list || marker.interrupts_paragraph())) + { + index = list(lines, index, &marker, depth, interrupting, &mut blocks); + after_container = true; + after_list = true; + } else if let Some(kind) = html_start(line, rest) { + index = html_block(lines, index, kind, &mut blocks); + } else if let Some(marker) = footnote_marker(line).filter(|_| containers) { + index = footnote(lines, index, &marker, depth, interrupting, &mut blocks); + after_container = true; + } else if let Some((items, next)) = table::parse(lines, index, interrupts_paragraph) { + blocks.push(Block::Table(items)); + index = next; + } else if line.mdx + && matches!(rest.as_bytes().first(), Some(b'<' | b'{')) + && let FlowOutcome::Flow(next) = mdx_flow(lines, index, &mut blocks) + { + index = next; + // Flow content that interrupted a paragraph keeps the restrictions that came with it. + after_paragraph = interrupting; + } else { + let release = match (closed_container, separated) { + (false, _) => Release::None, + (true, false) => Release::AllMarkers, + (true, true) => Release::Markers, + }; + let (next, interrupt, setext) = paragraph( + lines, + index, + release, + match (follows_setext && is_dash_run(rest), depth) { + (false, _) => Dashes::No, + (true, 0) => Dashes::UntilCode, + (true, _) => Dashes::OneLine, + }, + &mut blocks, + ); + index = next; + after_paragraph = interrupt; + after_setext = setext; + } + } + + blocks +} + +/// Tracks just enough of the block state of already collected lines to tell whether a paragraph +/// is open, which decides whether a following line can be a lazy continuation. +struct LeafState<'a> { + fence: Option>, + /// An HTML block that has not ended yet. + html: Option, + /// A table that has not ended yet. Its rows cannot be continued lazily. + table: bool, + /// The number of cells of the last line when it was paragraph text with a pipe, which a delimiter + /// row with as many cells can turn into a header. + header: Option, + paragraph: bool, + /// Number of container markers on the line that opened the paragraph. + paragraph_containers: usize, + /// Whether the last line was a setext underline. markdown-rs still lets unindented text continue. + after_setext: bool, + /// Restrictions on the next fed line, the first line of a container that interrupted a paragraph. + restricted: Interrupt, +} + +impl<'a> LeafState<'a> { + fn new(restricted: Interrupt) -> Self { + Self { + fence: None, + html: None, + table: false, + header: None, + paragraph: false, + paragraph_containers: 0, + after_setext: false, + restricted, + } + } + + /// Whether `line` without its container prefix continues the paragraph of the collected lines. + fn continues_with(&self, line: &Line<'_>) -> bool { + (self.paragraph || (self.after_setext && line.indent().0 < line.code_indent())) && is_lazy_continuation(line) + } + + fn feed(&mut self, line: &Line<'a>) { + let mut line = *line; + let restricted = std::mem::take(&mut self.restricted); + let follows_setext = std::mem::take(&mut self.after_setext); + let mut containers = 0; + // Whether a paragraph was open when the line started, for markers deeper on the same line. + let open = self.paragraph; + loop { + let (columns, indent) = line.indent(); + let rest = &line.text[indent..]; + + if let Some(fence) = &self.fence { + if columns < line.code_indent() && fence.is_closed_by(rest) { + self.fence = None; + } + return; + } + + if let Some(kind) = self.html { + let ends = match kind { + HtmlKind::Basic | HtmlKind::Complete => line.is_blank(), + _ => html_flow::ends_in(kind, line.text), + }; + if ends { + self.html = None; + } + self.paragraph = false; + return; + } + if line.is_blank() { + self.paragraph = false; + self.table = false; + self.header = None; + return; + } + let header = std::mem::take(&mut self.header); + if self.table { + if interrupts_paragraph(&line) { + self.table = false; + } else { + self.paragraph = false; + return; + } + } else if self.paragraph + && !line.lazy + && header.is_some() + && table::delimiter_cells(&line) == header + && !interrupts_paragraph(&line) + { + self.table = true; + self.paragraph = false; + return; + } + if columns >= line.code_indent() { + // Continues an open paragraph of this container, otherwise it is indented code (or + // text when the container interrupted a paragraph). + if !(self.paragraph && containers <= self.paragraph_containers) { + self.paragraph = restricted.code; + self.paragraph_containers = containers; + } + return; + } + if let Some(fence) = Fence::open(rest, !line.mdx) { + self.fence = Some(fence); + self.paragraph = false; + return; + } + if self.paragraph && containers == self.paragraph_containers && setext_depth(&line).is_some() { + self.paragraph = false; + self.after_setext = true; + return; + } + // Dashes right after a setext heading start a paragraph, not a thematic break. + if follows_setext && is_dash_run(rest) { + self.paragraph = true; + self.paragraph_containers = containers; + return; + } + if atx_depth(rest).is_some() || is_thematic_break(rest) { + self.paragraph = false; + return; + } + if let Some(kind) = html_start(&line, rest).filter(|&kind| kind != HtmlKind::Complete || !self.paragraph) { + let closed = match kind { + HtmlKind::Basic | HtmlKind::Complete => false, + _ => html_flow::ends_in(kind, &rest[html_flow::first_line_offset(kind)..]), + }; + self.html = (!closed).then_some(kind); + self.paragraph = false; + return; + } + if let Some(stripped) = strip_blockquote(line) { + line = stripped; + containers += 1; + continue; + } + if let Some(marker) = ListMarker::parse(&line).filter(|m| { + // A marker in or inside the container of the open paragraph has to be able to interrupt it. + let continues_paragraph = open && containers >= self.paragraph_containers; + !(restricted.list || continues_paragraph) || m.interrupts_paragraph() + }) { + line = marker.content(line); + containers += 1; + // The rest of the line starts a new item. + self.paragraph = false; + continue; + } + + // MDX flow content is not part of a paragraph. + if looks_like_mdx_flow(&line) { + self.paragraph = false; + return; + } + self.header = rest.contains('|').then(|| table::row_cells(&line)); + // A lazy line belongs to the paragraph that is already open, in its container. + if !(line.lazy && self.paragraph) { + self.paragraph_containers = containers; + } + self.paragraph = true; + return; + } + } +} + +/// Bytes to skip to get past a blockquote marker (`>` and one optional space), if `line` has one. +fn blockquote_marker(line: &Line<'_>) -> Option { + let (columns, indent) = line.indent(); + let rest = &line.text[indent..]; + if columns >= line.code_indent() || !rest.starts_with('>') { + return None; + } + let optional_space = usize::from(rest[1..].starts_with([' ', '\t'])); + Some(indent + 1 + optional_space) +} + +/// The line without its blockquote marker and the one space or column after it, if it has a marker. +fn strip_blockquote(line: Line<'_>) -> Option> { + let (columns, indent) = line.indent(); + if columns >= line.code_indent() || !line.text[indent..].starts_with('>') { + return None; + } + let after = line.skip(indent + 1); + Some(match after.text.as_bytes().first() { + Some(b' ') => after.skip(1), + // A tab is consumed by one column only. + Some(b'\t') => after.skip_columns(1), + _ => after, + }) +} + +fn blockquote( + lines: &[Line<'_>], + start: usize, + depth: usize, + interrupting: Interrupt, + blocks: &mut Vec, +) -> usize { + let mut inner = Vec::new(); + let mut state = LeafState::new(interrupting); + let mut index = start; + + while let Some(line) = lines.get(index) { + let stripped = match strip_blockquote(*line) { + Some(stripped) => stripped, + // Lazy continuation of a paragraph. + None if state.continues_with(line) && !blocks_lazy_continuation(lines, index) => { + Line { lazy: true, ..*line } + } + None => break, + }; + state.feed(&stripped); + inner.push(stripped); + index += 1; + } + + let children = parse_blocks(&inner, depth + 1, interrupting); + let mut end = inner.last().map_or_else(|| lines[start].end(), Line::end); + // An unclosed fence at the end of the document extends past the last line. + if let Some(child_end) = children.last().and_then(last_end) + && (child_end.line, child_end.column) > (end.line, end.column) + { + end = child_end; + } + let position = Position { + start: lines[start].point(0), + end, + }; + blocks.push(Block::Quote(QuoteBlock { children, position })); + + index +} + +struct ListMarker { + ordered: bool, + /// `-`, `+` or `*` for bullets, `.` or `)` for ordered lists. + delimiter: u8, + start: u32, + /// Bytes from the start of the line to the end of the marker. + after_marker: usize, + /// Columns of whitespace between the marker and the content that belong to the marker. + gap: usize, + /// Columns that continuation lines must be indented by to belong to the item. + width: usize, + /// Whether the first line has no content after the marker. + empty: bool, +} + +impl ListMarker { + fn parse(line: &Line<'_>) -> Option { + let (columns, indent) = line.indent(); + if columns >= line.code_indent() { + return None; + } + let rest = &line.text[indent..]; + let bytes = rest.as_bytes(); + if is_thematic_break(rest) { + return None; + } + + let (ordered, start, marker_len) = match *bytes.first()? { + b'-' | b'+' | b'*' => (false, 0, 1), + b'0'..=b'9' => { + let digits = bytes.iter().take_while(|b| b.is_ascii_digit()).count(); + if digits > 9 || !matches!(bytes.get(digits), Some(b'.' | b')')) { + return None; + } + (true, rest[..digits].parse().ok()?, digits + 1) + } + _ => return None, + }; + let delimiter = bytes[marker_len - 1]; + + let after = &rest[marker_len..]; + let (spaces, blanks) = line.skip(indent + marker_len).indent(); + if blanks == 0 && !after.is_empty() { + return None; + } + + let empty = blanks == after.len(); + // Five or more columns means the content is indented code, so only one belongs to the marker. + let gap = if empty || spaces > line.code_indent() { + 1 + } else { + spaces + }; + + Some(Self { + ordered, + delimiter, + start, + after_marker: indent + marker_len, + gap, + width: columns + marker_len + gap, + empty, + }) + } + + /// The line without the marker and the whitespace that belongs to it. + fn content<'a>(&self, line: Line<'a>) -> Line<'a> { + if self.empty { + line.skip(line.text.len()) + } else { + line.skip(self.after_marker).skip_columns(self.gap) + } + } + + fn same_list(&self, other: &Self) -> bool { + self.ordered == other.ordered && self.delimiter == other.delimiter + } + + /// Whether this marker may interrupt a paragraph. + fn interrupts_paragraph(&self) -> bool { + !self.empty && (!self.ordered || self.start == 1) + } +} + +fn list( + lines: &[Line<'_>], + start: usize, + first: &ListMarker, + depth: usize, + interrupting: Interrupt, + blocks: &mut Vec, +) -> usize { + let mut items = Vec::new(); + let mut spread = false; + let mut index = start; + + loop { + let marker = ListMarker::parse(&lines[index]).unwrap_or_else(|| unreachable!("checked by the caller")); + let (item, end) = list_item( + lines, + index, + &marker, + depth, + if items.is_empty() { + interrupting + } else { + Interrupt::default() + }, + ); + items.push(item); + + // Blank lines between items make the list loose. + let next = (end..lines.len()).find(|&i| !lines[i].is_blank()); + match next.filter(|&i| ListMarker::parse(&lines[i]).is_some_and(|m| m.same_list(first))) { + Some(next) => { + spread |= next > end || items.last().is_some_and(ends_in_empty_item); + index = next; + } + None => { + index = end; + break; + } + } + } + + blocks.push(Block::List(ListBlock { + ordered: first.ordered, + start: first.ordered.then_some(first.start), + spread, + items, + })); + + index +} + +/// The end of the last node that `block` produces. +fn last_end(block: &Block) -> Option { + match block { + Block::Node(node) => node.position().map(|position| position.end), + Block::Quote(quote) => Some(quote.position.end.clone()), + Block::List(list) => list.items.last().map(|item| item.position.end.clone()), + Block::Footnote(footnote) => Some(footnote.position.end.clone()), + Block::Inline(_) | Block::Table(_) | Block::Error(_) => None, + Block::Jsx(tag) => Some(tag.position.end.clone()), + } +} + +/// Extends the position of a trailing blockquote, or of the last item of a trailing list and of the +/// lists nested at its end, over the blank lines that follow. +fn extend_last_items(blocks: &mut [Block], end: &Point, quotes: bool) { + match blocks.last_mut() { + Some(Block::List(list)) => { + if let Some(item) = list.items.last_mut() + // An empty innermost item keeps its own end. + && !item.children.is_empty() + { + item.position.end = end.clone(); + extend_last_items(&mut item.children, end, quotes); + } + } + // A blockquote is only extended by a single whitespace-only line that ends the document. + Some(Block::Quote(quote)) if quotes => extend_quote(quote, end), + // A footnote definition also runs over the blank lines that follow. + Some(Block::Footnote(footnote)) => { + footnote.position.end = end.clone(); + extend_last_items(&mut footnote.children, end, quotes); + } + _ => {} + } +} + +/// Whether the content of `block` ends with a paragraph or a setext heading. +fn ends_in_text(block: &Block) -> bool { + match block { + Block::Inline(block) => match &block.kind { + InlineKind::Paragraph => true, + // A setext heading spans its underline line, an ATX heading does not. + InlineKind::Heading { position, .. } => position.start.line != position.end.line, + }, + Block::List(list) => list + .items + .last() + .and_then(|item| item.children.last()) + .is_some_and(ends_in_text), + Block::Quote(quote) => quote.children.last().is_some_and(ends_in_text), + Block::Footnote(footnote) => footnote.children.last().is_some_and(ends_in_text), + // A definition is a paragraph whose content was consumed. + Block::Node(node) => matches!(node, Node::Definition(_)), + Block::Table(_) | Block::Jsx(_) | Block::Error(_) => false, + } +} + +/// Extends a blockquote that ends in a paragraph, and the blockquotes nested at its end. +fn extend_quote(quote: &mut QuoteBlock, end: &Point) { + if !quote.children.last().is_some_and(ends_in_text) { + return; + } + quote.position.end = end.clone(); + match quote.children.last_mut() { + Some(Block::Quote(inner)) => extend_quote(inner, end), + Some(Block::Footnote(footnote)) => footnote.position.end = end.clone(), + _ => {} + } +} + +/// Whether the item ends with a nested list whose last item is empty. markdown-rs counts that +/// like a blank line between the item and the next one. +fn ends_in_empty_item(item: &Item) -> bool { + match item.children.last() { + Some(Block::List(list)) => list + .items + .last() + .is_some_and(|last| last.children.is_empty() || ends_in_empty_item(last)), + _ => false, + } +} + +/// Collects one item starting at `start`, returning it and the index of the first line after it. +fn list_item( + lines: &[Line<'_>], + start: usize, + marker: &ListMarker, + depth: usize, + interrupting: Interrupt, +) -> (Item, usize) { + let mut first = marker.content(lines[start]); + first.own_chunk = first.indent().0 >= first.code_indent(); + let mut inner = vec![first]; + let mut state = LeafState::new(interrupting); + state.feed(&first); + let mut blanks = Vec::new(); + let mut last_blank_has_whitespace = false; + let mut index = start + 1; + + while let Some(line) = lines.get(index) { + if line.is_blank() { + // An item that starts with a blank line cannot continue after another one. + if first.is_blank() && inner.len() == 1 { + break; + } + last_blank_has_whitespace = !line.text.is_empty(); + blanks.push(line.skip(line.text.len())); + } else if line.lazy { + state.feed(line); + inner.push(*line); + } else if line.indent().0 >= marker.width { + for blank in &blanks { + state.feed(blank); + } + inner.append(&mut blanks); + let stripped = line.skip_columns(marker.width); + state.feed(&stripped); + inner.push(stripped); + } else if blanks.is_empty() && state.continues_with(line) && !blocks_lazy_continuation(lines, index) { + let lazy = Line { lazy: true, ..*line }; + state.feed(&lazy); + inner.push(lazy); + } else { + break; + } + index += 1; + } + + // The checkbox is on the first content line, which is the next one when the marker line is empty. + let target = usize::from(first.is_blank()); + let checked = inner + .get(target) + .filter(|_| !ends_with_setext_underline(&inner[target + 1..])) + .and_then(task_checkbox) + .map(|(checked, bytes)| { + inner[target] = inner[target].skip(bytes); + checked + }); + + // Trailing blank lines belong to whatever follows the item, but still extend its position. + let reaches_end = index == lines.len(); + index -= blanks.len(); + + // An unclosed fence in an item includes its line terminator when the next line is blank or starts + // another container, but not when a plain paragraph or thematic break follows. + let next = lines.get(index); + let item_end = next + .is_none_or(|line| line.is_blank() || blockquote_marker(line).is_some() || ListMarker::parse(line).is_some()); + if let Some(last) = inner.last_mut() { + last.item_end = item_end; + } + + let position = Position { + start: lines[start].point(0), + end: blanks + .last() + .or(inner.last()) + .map_or_else(|| lines[start].end(), Line::end), + }; + let mut children = parse_blocks(&inner, depth + 1, interrupting); + if let Some(blank) = blanks.last() { + extend_last_items( + &mut children, + &blank.end(), + last_blank_has_whitespace && reaches_end && blanks.len() == 1 && blank.eol.is_empty(), + ); + } + let item = Item { + checked, + children, + position, + }; + + (item, index) +} + +/// Whether the paragraph that continues on `lines` is turned into a setext heading. +fn ends_with_setext_underline(lines: &[Line<'_>]) -> bool { + for line in lines { + if setext_depth(line).is_some() { + return true; + } + if interrupts_paragraph(line) { + return false; + } + } + false +} + +/// Recognises a GFM task marker (`[ ] ` or `[x] `) at the start of an item's content, returning +/// the checked state and the bytes to skip. +fn task_checkbox(line: &Line<'_>) -> Option<(bool, usize)> { + if line.mdx { + return None; + } + let bytes = line.text.as_bytes(); + let checked = match bytes.get(..4)? { + [b'[', b' ', b']', b' ' | b'\t'] => false, + [b'[', b'x' | b'X', b']', b' ' | b'\t'] => true, + _ => return None, + }; + (!line.text[4..].trim_matches([' ', '\t']).is_empty()).then_some((checked, 4)) +} + +/// Returns the heading depth when `rest` (indent already removed) is an ATX heading line. +fn atx_depth(rest: &str) -> Option { + let hashes = rest.bytes().take_while(|&b| b == b'#').count(); + let delimited = matches!(rest[hashes..].chars().next(), None | Some(' ' | '\t')); + ((1..=6).contains(&hashes) && delimited).then_some(hashes as u8) +} + +fn atx_heading(line: &Line<'_>, indent: usize, depth: u8) -> Block { + let rest = &line.text[indent + depth as usize..]; + let content = rest.trim_matches([' ', '\t']); + // A closing sequence must be preceded by whitespace, unless it is the whole content. + let content = { + let without_closing = content.trim_end_matches('#'); + if without_closing.is_empty() || without_closing.ends_with([' ', '\t']) { + without_closing.trim_end_matches([' ', '\t']) + } else { + content + } + }; + // markdown-rs drops further `#` sequences (and the whitespace between them) at the start of the text. + let content = { + let mut content = content; + while content.starts_with('#') { + content = content.trim_start_matches('#').trim_start_matches([' ', '\t']); + } + content + }; + let start = line.point(offset_in(line.text, content)); + + Block::Inline(InlineBlock { + source: InlineSource::new(std::iter::once((content, "", start))), + kind: InlineKind::Heading { + depth, + position: Position { + start: line.point(0), + end: line.end(), + }, + }, + }) +} + +/// Joins `(text, eol)` pairs, keeping each line's original terminator between lines. +pub(super) fn join_lines(parts: &[(&str, &str)]) -> String { + let mut value = String::with_capacity(parts.iter().map(|(text, eol)| text.len() + eol.len()).sum()); + for (index, (text, eol)) in parts.iter().enumerate() { + value.push_str(text); + if index + 1 < parts.len() { + value.push_str(eol); + } + } + value +} + +/// Byte offset of `child` within `parent`, where `child` is a subslice of `parent`. +fn offset_in(parent: &str, child: &str) -> usize { + child.as_ptr() as usize - parent.as_ptr() as usize +} + +/// Whether `rest` is only dashes, like `---`, which markdown-rs does not read as a thematic break +/// right after a setext heading. +fn is_dash_run(rest: &str) -> bool { + let content = rest.trim_end_matches([' ', '\t']); + content.len() >= 3 && content.bytes().all(|b| b == b'-') +} + +fn is_thematic_break(rest: &str) -> bool { + let Some(marker) = rest.chars().next().filter(|c| matches!(c, '*' | '-' | '_')) else { + return false; + }; + let mut count = 0; + for c in rest.chars() { + if c == marker { + count += 1; + } else if !matches!(c, ' ' | '\t') { + return false; + } + } + count >= 3 +} + +/// Returns the setext heading depth when `line` is a setext underline. +fn setext_depth(line: &Line<'_>) -> Option { + let (columns, indent) = line.indent(); + let content = line.text[indent..].trim_end_matches([' ', '\t']); + let marker = content.chars().next()?; + if line.lazy || columns >= line.code_indent() || !content.chars().all(|c| c == marker) { + return None; + } + match marker { + '=' => Some(1), + '-' => Some(2), + _ => None, + } +} + +/// Whether a line without its container prefix can continue an open paragraph of that container. +/// Any list marker ends the paragraph here, even one that could not interrupt it elsewhere. +fn is_lazy_continuation(line: &Line<'_>) -> bool { + !interrupts_paragraph(line) && ListMarker::parse(line).is_none() && !starts_construct(line) +} + +/// Whether the line is only the start of a construct that a next line could still complete: one or two +/// underscores (a thematic break), one or two backticks or tildes (a fence), a dollar, or an angle +/// bracket. markdown-rs does not continue paragraphs lazily with such a line. +fn starts_construct(line: &Line<'_>) -> bool { + // Only the end of the document leaves such a start undecided. + if !line.eof || !line.eol.is_empty() { + return false; + } + let text = line.text.trim_matches([' ', '\t']); + match text.as_bytes() { + [b'<'] => !line.mdx, + [b'$'] => true, + [first @ (b'`' | b'~'), rest @ ..] => rest.len() < 2 && rest.iter().all(|b| b == first), + _ => { + // Underscores with spaces between, like `_ _`. + let markers = text.bytes().filter(|&b| b == b'_').count(); + (1..3).contains(&markers) && text.bytes().all(|b| matches!(b, b'_' | b' ' | b'\t')) + } + } +} + +/// Whether `line` ends the paragraph that is being collected. +fn interrupts_paragraph(line: &Line<'_>) -> bool { + if line.is_blank() { + return true; + } + // Already accepted as a continuation by an enclosing container. + if line.lazy { + return false; + } + let (columns, indent) = line.indent(); + let rest = &line.text[indent..]; + columns < line.code_indent() + && (Fence::open(rest, !line.mdx).is_some() + || atx_depth(rest).is_some() + || is_thematic_break(rest) + || blockquote_marker(line).is_some() + || footnote_marker(line).is_some() + || html_start(line, rest).is_some_and(|kind| kind != HtmlKind::Complete) + || ListMarker::parse(line).is_some_and(|marker| marker.interrupts_paragraph())) +} + +/// Returns the index after the block and what it restricts on the first line of a following container. +/// +/// A paragraph of `dashes` right after a setext heading is cut short as described by [`Dashes`]. A paragraph with a `release` has its second line ended by list markers that could not interrupt it. +fn paragraph( + lines: &[Line<'_>], + start: usize, + release: Release, + dashes: Dashes, + blocks: &mut Vec, +) -> (usize, Interrupt, bool) { + let mut index = start + 1; + // Lines up to this one are taken by flow content that turned out to be text. + let mut absorbed = match probe_mdx_flow(lines, start) { + Some(FlowOutcome::Nok(Some(last))) => absorbed_until(lines, start, last), + _ => start, + }; + let mut setext = None; + // Whether a marker that could not interrupt the paragraph ended it anyway. + let mut released = false; + + while let Some(line) = lines.get(index) { + if table::starts_at(lines, index, interrupts_paragraph) { + break; + } + // MDX flow content, such as a tag or an expression alone on a line, interrupts a paragraph. + // What only looks like flow content still takes the lines it spans, whatever they start. + match probe_mdx_flow(lines, index) { + Some(FlowOutcome::Flow(_)) => break, + Some(FlowOutcome::Nok(Some(last))) => absorbed = absorbed.max(absorbed_until(lines, index, last)), + _ => {} + } + if index <= absorbed { + index += 1; + continue; + } + let releases = index == start + 1 + && ListMarker::parse(line).is_some_and(|marker| match release { + Release::None => false, + Release::Markers => !marker.empty, + Release::AllMarkers => true, + }); + if releases { + released = true; + break; + } + if let Some(depth) = setext_depth(line) { + setext = Some((depth, index)); + break; + } + if dashes == Dashes::OneLine + || interrupts_paragraph(line) + || (dashes == Dashes::UntilCode && line.indent().0 >= line.code_indent()) + { + break; + } + index += 1; + } + + // Definitions at the start of the paragraph are blocks of their own. + let mut first = start; + if lines[start].text.trim_start_matches([' ', '\t']).starts_with('[') { + let source = paragraph_source(&lines[start..index]); + let starts = lines[start..index] + .iter() + .map(|line| (line.point(0), line.end())) + .collect::>(); + let (definitions, used) = definition::extract(&source, &starts); + blocks.extend(definitions.into_iter().map(Block::Node)); + first += used; + } + if first == index { + // Nothing is left of the paragraph, so a setext underline starts a paragraph of text instead. + return match setext { + Some((_, underline)) => (underline, Interrupt::default(), true), + None => (index, Interrupt::default(), false), + }; + } + let source = paragraph_source(&lines[first..index]); + + match setext { + Some((depth, underline)) => { + blocks.push(Block::Inline(InlineBlock { + source, + kind: InlineKind::Heading { + depth, + position: Position { + start: lines[first].point(0), + end: lines[underline].end(), + }, + }, + })); + (underline + 1, Interrupt::default(), true) + } + None => { + blocks.push(Block::Inline(InlineBlock { + source, + kind: InlineKind::Paragraph, + })); + // A one-line paragraph right after a container does not restrict what follows it. + ( + index, + Interrupt { + code: dashes == Dashes::No, + list: !(dashes != Dashes::No || released || release == Release::AllMarkers && index == start + 1), + }, + false, + ) + } + } +} + +/// The raw inline content of the lines of a paragraph. Trailing spaces only matter before a line +/// ending, where they can make a hard break, so only the last line is trimmed at its end. +fn paragraph_source(lines: &[Line<'_>]) -> InlineSource { + let last = lines.len() - 1; + InlineSource::new(lines.iter().enumerate().map(|(index, line)| { + // The indentation of later lines stays in the content, where code and JSX keep it as it is. + let text = if index == 0 { + line.text.trim_start_matches([' ', '\t']) + } else { + line.text + }; + let text = if index == last { + text.trim_end_matches([' ', '\t']) + } else { + text + }; + (text, line.eol, line.point(offset_in(line.text, text))) + })) +} + +/// The start of a footnote definition: `[^label]:`. +struct FootnoteMarker<'a> { + label: &'a str, + /// Bytes from the start of the line to the content. + content: usize, +} + +fn footnote_marker<'a>(line: &Line<'a>) -> Option> { + if line.mdx { + return None; + } + let (columns, indent) = line.indent(); + let rest = line.text[indent..].strip_prefix("[^")?; + if columns >= CODE_INDENT { + return None; + } + let close = rest.find(']')?; + let label = &rest[..close]; + if label.is_empty() || label.bytes().any(|b| matches!(b, b' ' | b'\t' | b'[' | b'\r' | b'\n')) { + return None; + } + let after = rest[close + 1..].strip_prefix(':')?; + let blanks = after.len() - after.trim_start_matches([' ', '\t']).len(); + Some(FootnoteMarker { + label, + content: indent + 2 + close + 2 + blanks, + }) +} + +/// Collects a footnote definition, which continues on lines indented by four columns. +fn footnote( + lines: &[Line<'_>], + start: usize, + marker: &FootnoteMarker<'_>, + depth: usize, + interrupting: Interrupt, + blocks: &mut Vec, +) -> usize { + let first = lines[start].skip(marker.content); + let mut inner = vec![first]; + let mut state = LeafState::new(interrupting); + state.feed(&first); + let mut blanks = Vec::new(); + let mut index = start + 1; + + while let Some(line) = lines.get(index) { + if line.is_blank() { + if first.is_blank() && inner.len() == 1 { + break; + } + blanks.push(line.skip(line.text.len())); + } else if line.indent().0 >= CODE_INDENT { + for blank in &blanks { + state.feed(blank); + } + inner.append(&mut blanks); + let stripped = line.skip_columns(CODE_INDENT); + state.feed(&stripped); + inner.push(stripped); + } else if blanks.is_empty() && state.continues_with(line) && !blocks_lazy_continuation(lines, index) { + let lazy = Line { lazy: true, ..*line }; + state.feed(&lazy); + inner.push(lazy); + } else { + break; + } + index += 1; + } + + // Trailing blank lines belong to whatever follows, but still extend the position. + index -= blanks.len(); + let position = Position { + start: lines[start].point(0), + end: blanks + .last() + .or(inner.last()) + .map_or_else(|| lines[start].end(), Line::end), + }; + blocks.push(Block::Footnote(FootnoteBlock { + ident: inline::normalize(marker.label), + children: parse_blocks(&inner, depth + 1, interrupting), + position, + })); + + index +} + +/// Collects an HTML block of `kind` starting at `lines[start]`, which is indented less than four columns. +fn html_block(lines: &[Line<'_>], start: usize, kind: HtmlKind, blocks: &mut Vec) -> usize { + let first = &lines[start]; + let mut last = start; + let mut unterminated = false; + + match kind { + HtmlKind::Basic | HtmlKind::Complete => { + // Ends before a blank line, and lines cannot continue a paragraph lazily. + for (index, line) in lines.iter().enumerate().skip(start + 1) { + if line.is_blank() || line.lazy { + break; + } + last = index; + } + } + _ => { + let (_, indent) = first.indent(); + let from = indent + html_flow::first_line_offset(kind); + if !html_flow::ends_in(kind, &first.text[from..]) { + let mut ended = false; + for (index, line) in lines.iter().enumerate().skip(start + 1) { + if line.lazy { + break; + } + last = index; + if html_flow::ends_in(kind, line.text) { + ended = true; + break; + } + } + // A block that never ends runs to the end of the document, line ending included. + unterminated = !ended && lines[last].eof && !lines[last].eol.is_empty(); + } + } + } + + let parts = lines[start..=last] + .iter() + .map(|line| (line.text, line.eol)) + .collect::>(); + let mut value = join_lines(&parts); + let end = if unterminated { + value.push_str(lines[last].eol); + Point { + line: lines[last].number + 1, + column: 1, + } + } else { + lines[last].end() + }; + blocks.push(Block::Node(Node::Html(Html { + value, + position: Some(Position { + start: first.point(0), + end, + }), + }))); + + last + 1 +} + +/// The kind of HTML block that starts at `rest`; MDX has none. +fn html_start(line: &Line<'_>, rest: &str) -> Option { + if line.mdx { None } else { html_flow::start(rest) } +} diff --git a/crates/mq-markdown/src/parser/code.rs b/crates/mq-markdown/src/parser/code.rs new file mode 100644 index 000000000..41158614f --- /dev/null +++ b/crates/mq-markdown/src/parser/code.rs @@ -0,0 +1,154 @@ +//! Fenced and indented code, and math fences. + +use super::block::{CODE_INDENT, join_lines}; +use super::inline; +use super::line::Line; +use super::tree::Block; +use crate::node::{Code, Math, Node, Point, Position}; + +/// An opening code fence, or math fence. +pub(super) struct Fence<'a> { + marker: u8, + length: usize, + /// Info string after the fence, trimmed. + info: &'a str, +} + +impl<'a> Fence<'a> { + /// Parses `rest` (indent already removed) as an opening fence. + pub(super) fn open(rest: &'a str, math: bool) -> Option { + let marker = *rest + .as_bytes() + .first() + .filter(|b| matches!(b, b'`' | b'~') || (math && **b == b'$'))?; + let length = rest.bytes().take_while(|&b| b == marker).count(); + let info = rest[length..].trim_matches([' ', '\t']); + // Math fences are two dollars or more, and the info string cannot hold the marker of math or + // backtick fences. + let long_enough = length >= if marker == b'$' { 2 } else { 3 }; + let info_ok = !(matches!(marker, b'`' | b'$') && info.as_bytes().contains(&marker)); + (long_enough && info_ok).then_some(Self { marker, length, info }) + } + + /// Whether `rest` (indent already removed) closes this fence. + pub(super) fn is_closed_by(&self, rest: &str) -> bool { + let length = rest.bytes().take_while(|&b| b == self.marker).count(); + length >= self.length && rest[length..].trim_matches([' ', '\t']).is_empty() + } +} + +pub(super) fn fenced_code( + lines: &[Line<'_>], + start: usize, + indent: usize, + fence: &Fence<'_>, + own_end: bool, + blocks: &mut Vec, +) -> usize { + let (lang, meta) = match fence.info.split_once([' ', '\t']) { + Some((lang, meta)) => (Some(lang), Some(meta.trim_start_matches([' ', '\t']))), + None => (Some(fence.info).filter(|s| !s.is_empty()), None), + }; + + let mut body = Vec::new(); + let mut index = start + 1; + let mut end = None; + + while let Some(line) = lines.get(index) { + let (columns, line_indent) = line.indent(); + if columns < line.code_indent() && fence.is_closed_by(&line.text[line_indent..]) { + end = Some(line.end()); + index += 1; + break; + } + // Remove up to the opening fence's indentation from each content line. + let strip = line.text.bytes().take(indent).take_while(|&b| b == b' ').count(); + body.push((&line.text[strip..], line.eol)); + index += 1; + } + + // An unclosed fence runs to the end of its container. When that is the end of the document the + // final line terminator is part of it, so the end is the start of the following line. + // Right after a container, a fence without content ends with its own line instead. + let end = end.unwrap_or_else(|| match lines.last() { + Some(last) if !last.eol.is_empty() && reaches_line_end(last, body.is_empty(), own_end) => Point { + line: last.number + 1, + column: 1, + }, + Some(last) => last.end(), + None => lines[start].end(), + }); + + let position = Some(Position { + start: lines[start].point(indent), + end, + }); + let value = join_lines(&body); + let node = if fence.marker == b'$' { + Node::Math(Math { value, position }) + } else { + Node::Code(Code { + value, + lang: lang.map(inline::unescape), + meta: meta.filter(|m| !m.is_empty()).map(inline::unescape), + fence: true, + position, + }) + }; + blocks.push(Block::Node(node)); + + index +} + +/// Whether an unclosed fence whose last line is `last` includes that line's terminator. +fn reaches_line_end(last: &Line<'_>, empty: bool, own_end: bool) -> bool { + if empty { + last.eof && !own_end + } else { + last.eof || last.item_end + } +} + +pub(super) fn indented_code(lines: &[Line<'_>], start: usize, after_container: bool, blocks: &mut Vec) -> usize { + // markdown-rs makes the first line a chunk of its own in these two cases. + let own_chunk = lines[start].own_chunk || after_container; + let mut last_code = start; + let mut index = start; + + while let Some(line) = lines.get(index).filter(|_| !(own_chunk && index > start)) { + // Whitespace-only lines are code when indented enough, otherwise they may just separate chunks. + if line.indent().0 >= CODE_INDENT { + last_code = index; + } else if !line.is_blank() { + break; + } + index += 1; + } + + let mut parts = lines[start..=last_code] + .iter() + .map(|line| (line.skip_columns(CODE_INDENT).text, line.eol)) + .collect::>(); + // Lines that are empty once the indent is removed only extend the position, not the value. + while parts.len() > 1 && parts.last().is_some_and(|(text, _)| text.is_empty()) { + parts.pop(); + } + let value = join_lines(&parts); + + let position = Position { + start: lines[start].point(0), + end: lines[last_code].end(), + }; + // Mirrors the `mdast` conversion: a value whose line count differs from the span is reported as fenced. + let fence = value.lines().count() != position.end.line - position.start.line + 1; + + blocks.push(Block::Node(Node::Code(Code { + value, + lang: None, + meta: None, + fence, + position: Some(position), + }))); + + last_code + 1 +} diff --git a/crates/mq-markdown/src/parser/definition.rs b/crates/mq-markdown/src/parser/definition.rs new file mode 100644 index 000000000..6de759d4b --- /dev/null +++ b/crates/mq-markdown/src/parser/definition.rs @@ -0,0 +1,146 @@ +//! Link reference definitions: `[label]: destination "title"` at the start of a paragraph. + +use super::inline::{destination, normalize, remove_line_indent, title_at, unescape}; +use super::tree::InlineSource; +use crate::node::{Definition, Node, Point, Position, Title, Url}; + +/// Skips spaces and tabs. +fn skip_blanks(bytes: &[u8], mut index: usize) -> usize { + while matches!(bytes.get(index), Some(b' ' | b'\t')) { + index += 1; + } + index +} + +/// The length of the line ending at `index`, or 0. +fn eol_len(bytes: &[u8], index: usize) -> usize { + match bytes.get(index..) { + Some([b'\r', b'\n', ..]) => 2, + Some([b'\r' | b'\n', ..]) => 1, + _ => 0, + } +} + +/// Skips whitespace including at most one line ending. +fn skip_space(bytes: &[u8], index: usize) -> usize { + let index = skip_blanks(bytes, index); + let eol = eol_len(bytes, index); + if eol > 0 { + skip_blanks(bytes, index + eol) + } else { + index + } +} + +/// Parses the label at `pos` (a `[`) followed by `:`, returning the raw label and the offset after. +fn label(text: &str, pos: usize) -> Option<(&str, usize)> { + let bytes = text.as_bytes(); + let mut index = pos + 1; + while index < bytes.len() && index - pos <= 1000 { + match bytes[index] { + b'\\' => index += 1, + b'[' => return None, + b']' => { + let label = &text[pos + 1..index]; + let has_content = label.bytes().any(|b| !matches!(b, b' ' | b'\t' | b'\n' | b'\r')); + return (has_content && bytes.get(index + 1) == Some(&b':')).then_some((label, index + 2)); + } + _ => {} + } + index += 1; + } + None +} + +/// A definition and where it ends. +struct Parsed { + label: String, + ident: String, + url: String, + title: Option, + /// End of the last token of the definition. + end: usize, + /// Offset of the next line, after the line ending. + next: usize, +} + +fn parse_one(text: &str, pos: usize) -> Option { + let bytes = text.as_bytes(); + let (raw_label, after_colon) = label(text, pos)?; + + let dest_start = skip_space(bytes, after_colon); + let (url, after_dest) = destination(text, dest_start)?; + // A bare destination cannot be empty. + if after_dest == dest_start { + return None; + } + + // A title on the same line or on the next one, with nothing after it on its line. + let title = title_start(bytes, after_dest).and_then(|start| { + let (title, after_title) = title_at(text, start)?; + let after_blanks = skip_blanks(bytes, after_title); + let eol = eol_len(bytes, after_blanks); + (eol > 0 || after_blanks == bytes.len()).then_some((title, after_title, after_blanks + eol)) + }); + + let (title, end, next) = match title { + Some((title, end, next)) => (Some(title), end, next), + None => { + let after_blanks = skip_blanks(bytes, after_dest); + let eol = eol_len(bytes, after_blanks); + if eol == 0 && after_blanks != bytes.len() { + return None; + } + (None, after_dest, after_blanks + eol) + } + }; + + Some(Parsed { + label: unescape(&remove_line_indent(raw_label)), + ident: normalize(raw_label), + url, + title, + end, + next, + }) +} + +/// The start of a title after the destination: whitespace is required before it. +fn title_start(bytes: &[u8], after_dest: usize) -> Option { + let start = skip_space(bytes, after_dest); + (start > after_dest).then_some(start) +} + +/// Parses the definitions at the start of a paragraph. `lines` holds the start (including its +/// indentation) and the end of each line. Returns the definitions and how many lines they use. +pub(super) fn extract(source: &InlineSource, lines: &[(Point, Point)]) -> (Vec, usize) { + let text = source.text.as_str(); + let mut nodes = Vec::new(); + let mut pos = 0; + let mut next_line = 0; + + while pos < text.len() && text.as_bytes()[pos] == b'[' { + let Some(parsed) = parse_one(text, pos) else { + break; + }; + let first = source.lines.partition_point(|(start, _)| *start <= pos) - 1; + // The definition ends with its last line, trailing whitespace included. + let last = source.lines.partition_point(|(start, _)| *start <= parsed.end) - 1; + nodes.push(Node::Definition(Definition { + position: Some(Position { + start: lines[first].0.clone(), + end: lines[last].1.clone(), + }), + url: Url(parsed.url), + title: parsed.title.map(Title), + ident: parsed.ident, + label: Some(parsed.label), + })); + next_line = parsed.next; + pos = skip_blanks(text.as_bytes(), parsed.next); + } + + // Every line that starts before the end of the definitions is used. + let used = source.lines.partition_point(|(start, _)| *start < next_line); + (nodes, used) +} diff --git a/crates/mq-markdown/src/parser/html_flow.rs b/crates/mq-markdown/src/parser/html_flow.rs new file mode 100644 index 000000000..a2ee6c3e2 --- /dev/null +++ b/crates/mq-markdown/src/parser/html_flow.rs @@ -0,0 +1,240 @@ +//! HTML blocks: the seven kinds of `CommonMark`, ported from the rules of `markdown-rs`. + +/// The kind of an HTML block, which decides how it ends. +#[derive(Clone, Copy, PartialEq)] +pub(super) enum Kind { + /// ``. + Comment, + /// ``. + Instruction, + /// ``. + Declaration, + /// ``. + Cdata, + /// A known block tag name: ends at a blank line. + Basic, + /// A complete tag alone on its line: ends at a blank line. It cannot interrupt a paragraph. + Complete, +} + +const RAW_NAMES: [&str; 4] = ["pre", "script", "style", "textarea"]; + +const BLOCK_NAMES: [&str; 62] = [ + "address", + "article", + "aside", + "base", + "basefont", + "blockquote", + "body", + "caption", + "center", + "col", + "colgroup", + "dd", + "details", + "dialog", + "dir", + "div", + "dl", + "dt", + "fieldset", + "figcaption", + "figure", + "footer", + "form", + "frame", + "frameset", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "head", + "header", + "hr", + "html", + "iframe", + "legend", + "li", + "link", + "main", + "menu", + "menuitem", + "nav", + "noframes", + "ol", + "optgroup", + "option", + "p", + "param", + "search", + "section", + "summary", + "table", + "tbody", + "td", + "tfoot", + "th", + "thead", + "title", + "tr", + "track", + "ul", +]; + +/// Recognizes the start of an HTML block in `rest`, the line without its indentation. +pub(super) fn start(rest: &str) -> Option { + let bytes = rest.as_bytes(); + if bytes.first() != Some(&b'<') { + return None; + } + + match bytes.get(1)? { + b'!' => match bytes.get(2)? { + b'-' if bytes.get(3) == Some(&b'-') => Some(Kind::Comment), + b'[' if rest[3..].starts_with("CDATA[") => Some(Kind::Cdata), + byte if byte.is_ascii_alphabetic() => Some(Kind::Declaration), + _ => None, + }, + b'?' => Some(Kind::Instruction), + b'/' => tag(rest, 2, true), + byte if byte.is_ascii_alphabetic() => tag(rest, 1, false), + _ => None, + } +} + +/// Recognizes a block tag, a raw tag or a complete tag whose name starts at `from`. +fn tag(rest: &str, from: usize, closing: bool) -> Option { + let bytes = rest.as_bytes(); + if !bytes.get(from).is_some_and(u8::is_ascii_alphabetic) { + return None; + } + let length = bytes[from..] + .iter() + .take_while(|b| b.is_ascii_alphanumeric() || **b == b'-') + .count(); + let end = from + length; + let terminator = bytes.get(end).copied(); + if !matches!(terminator, None | Some(b'\t' | b' ' | b'/' | b'>')) { + return None; + } + let slash = terminator == Some(b'/'); + let name = rest[from..end].to_ascii_lowercase(); + + if !slash && !closing && RAW_NAMES.contains(&name.as_str()) { + Some(Kind::Raw) + } else if BLOCK_NAMES.contains(&name.as_str()) { + // A slash has to be the end of a self-closing tag. + (!slash || bytes.get(end + 1) == Some(&b'>')).then_some(Kind::Basic) + } else { + complete_tag(&bytes[end..], closing).then_some(Kind::Complete) + } +} + +fn skip_blanks(bytes: &[u8], mut index: usize) -> usize { + while matches!(bytes.get(index), Some(b' ' | b'\t')) { + index += 1; + } + index +} + +/// Whether the rest of a tag after its name is complete and alone on the line. +fn complete_tag(bytes: &[u8], closing: bool) -> bool { + let mut index = 0; + + if closing { + index = skip_blanks(bytes, index); + return bytes.get(index) == Some(&b'>') && skip_blanks(bytes, index + 1) == bytes.len(); + } + + loop { + index = skip_blanks(bytes, index); + match bytes.get(index) { + Some(b'/') => { + return bytes.get(index + 1) == Some(&b'>') && skip_blanks(bytes, index + 2) == bytes.len(); + } + Some(b'0'..=b'9' | b':' | b'A'..=b'Z' | b'_' | b'a'..=b'z') => { + index += 1; + while matches!( + bytes.get(index), + Some(b'-' | b'.' | b'0'..=b'9' | b':' | b'A'..=b'Z' | b'_' | b'a'..=b'z') + ) { + index += 1; + } + let after = skip_blanks(bytes, index); + if bytes.get(after) == Some(&b'=') { + let Some(end) = attribute_value(bytes, skip_blanks(bytes, after + 1)) else { + return false; + }; + index = end; + } + } + Some(b'>') => return skip_blanks(bytes, index + 1) == bytes.len(), + _ => return false, + } + } +} + +/// The offset after the attribute value at `index`. +fn attribute_value(bytes: &[u8], index: usize) -> Option { + match bytes.get(index)? { + b'<' | b'=' | b'>' | b'`' => None, + quote @ (b'"' | b'\'') => { + let close = bytes[index + 1..].iter().position(|b| b == quote)?; + let end = index + 1 + close + 1; + matches!(bytes.get(end), Some(b'\t' | b' ' | b'/' | b'>')).then_some(end) + } + _ => { + let length = bytes[index..] + .iter() + .take_while(|b| !matches!(b, b'\t' | b' ' | b'"' | b'\'' | b'/' | b'<' | b'=' | b'>' | b'`')) + .count(); + Some(index + length) + } + } +} + +/// Where in the first line the search for the end of a block starts, so that the opening marker is +/// only reused where `markdown-rs` reuses it (`` and `` are complete). +pub(super) fn first_line_offset(kind: Kind) -> usize { + match kind { + Kind::Comment => 2, + Kind::Instruction => 1, + Kind::Declaration => 3, + Kind::Cdata => 9, + Kind::Raw | Kind::Basic | Kind::Complete => 1, + } +} + +/// Whether `text` holds what ends a block of `kind`. Blocks of the last two kinds end at blank lines. +pub(super) fn ends_in(kind: Kind, text: &str) -> bool { + match kind { + Kind::Comment => text.contains("-->"), + Kind::Instruction => text.contains("?>"), + Kind::Declaration => text.contains('>'), + Kind::Cdata => text.contains("]]>"), + Kind::Raw => has_raw_end(text), + Kind::Basic | Kind::Complete => false, + } +} + +/// Whether `text` has a closing tag of a raw element, such as ``. +fn has_raw_end(text: &str) -> bool { + let bytes = text.as_bytes(); + let mut index = 0; + while let Some(found) = text[index..].find("') + && RAW_NAMES.contains(&text[from..from + letters].to_ascii_lowercase().as_str()) + { + return true; + } + index = from; + } + false +} diff --git a/crates/mq-markdown/src/parser/inline.rs b/crates/mq-markdown/src/parser/inline.rs new file mode 100644 index 000000000..976018d4a --- /dev/null +++ b/crates/mq-markdown/src/parser/inline.rs @@ -0,0 +1,612 @@ +//! Inline parsing of paragraph, heading and table cell content. +//! +//! The content is scanned once, left to right, into a flat list of items: literal text, finished +//! nodes (code spans, autolinks, breaks), delimiter runs and bracket openers. Links close over the +//! items after their opener, and emphasis is resolved with the delimiter stack algorithm of the +//! CommonMark specification, so nesting never recurses over the source. + +mod emphasis; +mod entities; +mod entity; +mod html; +mod link; +mod literal; +mod punctuation; +mod tail; + +pub(super) use entity::{decode_references, remove_line_indent, unescape}; +pub(super) use link::normalize; +pub(super) use tail::{destination, title_at}; + +use super::mdx::{self, Fallback, Parsed, TagKind}; +use super::resolve::References; +use super::tree::InlineSource; +use crate::node::{ + Break, CodeInline, MathInline, MdxAttributeContent, MdxJsxTextElement, MdxTextExpression, Node, Position, Text, +}; +use smol_str::SmolStr; +use std::borrow::Cow; +use std::cell::RefCell; + +/// The value of a text item: a slice of the source, or a decoded string. +pub(super) enum Value { + Slice(usize, usize), + Owned(String), +} + +/// A run of `*`, `_` or `~` that may open or close emphasis. +pub(super) struct Delim { + pub(super) ch: u8, + /// Offset of the first remaining character. + pub(super) start: usize, + pub(super) count: usize, + pub(super) can_open: bool, + pub(super) can_close: bool, +} + +/// A `[` or `![` that may open a link or an image. +pub(super) struct Opener { + pub(super) image: bool, + /// Offset of the `[`, or of the `!` of an image. + pub(super) start: usize, + /// Links cannot contain links, so openers before a link are deactivated. + pub(super) active: bool, +} + +pub(super) enum Item { + Text { + start: usize, + end: usize, + value: Value, + }, + Node(Node), + Delim(Delim), + Open(Opener), + /// An email address in plain text. It is not linked inside link text. + Email { + start: usize, + end: usize, + prefixed: bool, + }, + /// A JSX tag in MDX. Tags are paired when the items become nodes. + Jsx(TextTag), +} + +/// A JSX tag in text, with its offsets in the source. +pub(super) struct TextTag { + name: Option, + attributes: Vec, + kind: TagKind, + start: usize, + end: usize, +} + +/// What the scanner and its helpers share. +#[derive(Clone, Copy)] +pub(super) struct Context<'a> { + pub(super) source: &'a InlineSource, + pub(super) references: &'a References, + /// Whether email addresses in plain text are linked. They are not inside link text. + pub(super) emails: bool, + /// The first error in the content, which only MDX has. + error: &'a RefCell>, +} + +impl Context<'_> { + /// Records an error, keeping the first one. + fn fail(&self, message: String) { + self.error.borrow_mut().get_or_insert(message); + } + + pub(super) fn src(&self) -> &str { + &self.source.text + } + + pub(super) fn position(&self, start: usize, end: usize) -> Position { + Position { + start: self.source.point(start), + end: self.source.end_point(end), + } + } +} + +/// Parses raw inline content into nodes. +pub(super) fn parse(source: &InlineSource, references: &References) -> Result, String> { + if source.text.is_empty() { + return Ok(Vec::new()); + } + + let error = RefCell::new(None); + let context = Context { + source, + references, + emails: !references.mdx, + error: &error, + }; + let mut scanner = Scanner { + context: &context, + items: Vec::new(), + openers: Vec::new(), + pos: 0, + run: 0, + }; + scanner.scan(); + scanner.flush(); + + let Scanner { mut items, .. } = scanner; + emphasis::process(&mut items, 0, &context); + let nodes = to_nodes(items, &context); + + match error.into_inner() { + Some(message) => Err(message), + None => Ok(nodes), + } +} + +/// Converts items to nodes, merging adjacent text. Unused delimiters and openers become text. +pub(super) fn to_nodes(items: Vec, context: &Context<'_>) -> Vec { + let src = context.src(); + let items = if context.emails && src.contains('@') { + literal::link_emails(src, items) + } else { + items + }; + let mut nodes = Vec::new(); + // Elements that are open, with the nodes that came before them. + let mut open: Vec<(TextTag, Vec)> = Vec::new(); + let mut current: Option<(usize, usize, String)> = None; + + let finish = |current: &mut Option<(usize, usize, String)>, nodes: &mut Vec| { + if let Some((start, end, value)) = current.take() { + nodes.push(Node::Text(Text { + value, + position: Some(context.position(start, end)), + })); + } + }; + + for item in items { + let (start, end, text): (usize, usize, Cow<'_, str>) = match item { + // A line ending directly inside a JSX element is not part of its content. + Item::Text { + value: Value::Slice(from, to), + .. + } if !open.is_empty() && from < to && src[from..to].bytes().all(|b| matches!(b, b'\n' | b'\r')) => { + continue; + } + Item::Text { start, end, value } => match value { + // The backslash of an escape is not part of the position of the text it starts. + Value::Slice(from, to) => (from, end, Cow::Borrowed(&src[from..to])), + Value::Owned(value) => (start, end, Cow::Owned(value)), + }, + Item::Delim(delim) => { + let end = delim.start + delim.count; + (delim.start, end, Cow::Borrowed(&src[delim.start..end])) + } + Item::Open(opener) => { + let end = opener.start + if opener.image { 2 } else { 1 }; + (opener.start, end, Cow::Borrowed(&src[opener.start..end])) + } + Item::Node(node) => { + finish(&mut current, &mut nodes); + nodes.push(node); + continue; + } + Item::Jsx(tag) => { + finish(&mut current, &mut nodes); + match tag.kind { + TagKind::Open => open.push((tag, std::mem::take(&mut nodes))), + TagKind::SelfClosing => { + let end = tag.end; + nodes.push(text_element(context, tag, Vec::new(), end)); + } + TagKind::Close => match open.pop() { + None => context.fail("Unexpected closing slash `/` in tag, expected an open tag first".into()), + Some((opening, before)) => { + if opening.name != tag.name { + context.fail(format!( + "Unexpected closing tag ``, expected corresponding closing tag for `<{}>`", + tag.name.as_deref().unwrap_or_default(), + opening.name.as_deref().unwrap_or_default() + )); + } + let children = std::mem::replace(&mut nodes, before); + nodes.push(text_element(context, opening, children, tag.end)); + } + }, + } + continue; + } + Item::Email { start, end, prefixed } => { + finish(&mut current, &mut nodes); + let text = &src[start..end]; + let position = Some(context.position(start, end)); + nodes.push(Node::Link(crate::node::Link { + url: crate::node::Url(if prefixed { + format!("mailto:{text}") + } else { + text.to_string() + }), + title: None, + values: vec![Node::Text(Text { + value: text.to_string(), + position: position.clone(), + })], + position, + })); + continue; + } + }; + match &mut current { + Some((_, current_end, value)) => { + value.push_str(&text); + *current_end = end; + } + None => current = Some((start, end, text.into_owned())), + } + } + finish(&mut current, &mut nodes); + + // Elements that never close are an error, and are closed where the content ends. + while let Some((opening, before)) = open.pop() { + context.fail(format!( + "Expected a closing tag for `<{}>` before the end of the content", + opening.name.as_deref().unwrap_or_default() + )); + let end = opening.end; + let children = std::mem::replace(&mut nodes, before); + nodes.push(text_element(context, opening, children, end)); + } + + nodes +} + +fn text_element(context: &Context<'_>, tag: TextTag, children: Vec, end: usize) -> Node { + Node::MdxJsxTextElement(MdxJsxTextElement { + children, + position: Some(context.position(tag.start, end)), + name: tag.name.map(SmolStr::new), + attributes: tag.attributes, + }) +} + +struct Scanner<'a> { + context: &'a Context<'a>, + items: Vec, + /// Indexes of the `Item::Open` items that are still waiting for a `]`. + openers: Vec, + pos: usize, + /// Start of the plain text that has not been pushed as an item yet. + run: usize, +} + +impl Scanner<'_> { + fn src(&self) -> &str { + self.context.src() + } + + fn scan(&mut self) { + let len = self.src().len(); + while self.pos < len { + let bytes = self.src().as_bytes(); + match bytes[self.pos] { + b'\\' => self.escape(), + b'&' => self.entity(), + b'`' => self.span(b'`'), + b'$' if !self.context.references.mdx => self.span(b'$'), + b'*' | b'_' => self.delimiter(), + b'~' if !self.context.references.mdx => self.delimiter(), + b'[' => self.open(false), + b'!' if bytes.get(self.pos + 1) == Some(&b'[') => self.open(true), + b']' => self.close(), + b'<' if !self.context.references.mdx => self.angle(), + b'<' => self.jsx(), + b'{' if self.context.references.mdx => self.expression(), + b'\n' | b'\r' => self.line_ending(), + b'h' | b'H' | b'w' | b'W' if !self.context.references.mdx => self.literal_url(), + _ => self.pos += 1, + } + } + } + + /// Pushes the pending plain text up to the current position as an item. + fn flush(&mut self) { + if self.run < self.pos { + self.items.push(Item::Text { + start: self.run, + end: self.pos, + value: Value::Slice(self.run, self.pos), + }); + } + self.run = self.pos; + } + + /// Pushes `item` after the pending text and continues scanning at `end`. + fn push(&mut self, item: Item, end: usize) { + self.flush(); + self.items.push(item); + self.pos = end; + self.run = end; + } + + fn push_node(&mut self, node: Node, end: usize) { + self.push(Item::Node(node), end); + } + + fn escape(&mut self) { + let src = self.src(); + let bytes = src.as_bytes(); + let pos = self.pos; + match bytes.get(pos + 1) { + Some(next) if next.is_ascii_punctuation() => { + let item = Item::Text { + start: pos, + end: pos + 2, + value: Value::Slice(pos + 1, pos + 2), + }; + self.push(item, pos + 2); + } + Some(b'\n' | b'\r') => { + let end = pos + 1 + eol_len(&bytes[pos + 1..]); + let node = Node::Break(Break { + position: Some(self.context.position(pos, end)), + }); + self.push_node(node, end); + // The indentation of the next line is not part of the text. + while matches!(self.src().as_bytes().get(self.pos), Some(b' ' | b'\t')) { + self.pos += 1; + } + self.run = self.pos; + } + _ => self.pos += 1, + } + } + + fn entity(&mut self) { + match entity::decode(self.src(), self.pos) { + Some((end, value)) => { + let item = Item::Text { + start: self.pos, + end, + value: Value::Owned(value), + }; + self.push(item, end); + } + None => self.pos += 1, + } + } + + fn line_ending(&mut self) { + let src = self.src(); + let bytes = src.as_bytes(); + let pos = self.pos; + let eol = eol_len(&bytes[pos..]); + + // Trailing whitespace before the line ending is dropped, or makes a hard break. + let mut start = pos; + while start > self.run && matches!(bytes[start - 1], b' ' | b'\t') { + start -= 1; + } + let hard = pos - start >= 2 && bytes[start..pos].iter().all(|&b| b == b' '); + // The indentation of the next line is not part of the text. + let mut after = pos + eol; + while matches!(bytes.get(after), Some(b' ' | b'\t')) { + after += 1; + } + + self.pos = start; + self.flush(); + if hard { + let node = Node::Break(Break { + position: Some(self.context.position(start, pos + eol)), + }); + self.items.push(Item::Node(node)); + } else { + self.items.push(Item::Text { + start: pos, + end: pos + eol, + value: Value::Slice(pos, pos + eol), + }); + } + self.pos = after; + self.run = after; + } + + /// A code span (`` ` ``) or an inline math span (`$`). + fn span(&mut self, ch: u8) { + let src = self.src(); + let bytes = src.as_bytes(); + let start = self.pos; + let size = bytes[start..].iter().take_while(|&&b| b == ch).count(); + + // Find the next run of exactly the same length. + let mut index = start + size; + let mut close = None; + while index < bytes.len() { + if bytes[index] == ch { + let run = bytes[index..].iter().take_while(|&&b| b == ch).count(); + if run == size { + close = Some(index); + break; + } + index += run; + } else { + index += 1; + } + } + + let Some(close) = close else { + self.pos = start + size; + return; + }; + + let end = close + size; + // In a table cell, `\|` stands for `|` inside code as well. + let content = &src[start + size..close]; + let value = if self.context.source.table && content.contains("\\|") { + span_value(&content.replace("\\|", "|")) + } else { + span_value(content) + }; + let position = Some(self.context.position(start, end)); + let node = if ch == b'`' { + Node::CodeInline(CodeInline { value, position }) + } else { + Node::MathInline(MathInline { value, position }) + }; + self.push_node(node, end); + } + + fn delimiter(&mut self) { + let src = self.src(); + let bytes = src.as_bytes(); + let start = self.pos; + let ch = bytes[start]; + let size = bytes[start..].iter().take_while(|&&b| b == ch).count(); + let end = start + size; + + // Strikethrough runs are one or two tildes. + if ch == b'~' && size > 2 { + self.pos = end; + return; + } + + let before = src[..start].chars().next_back(); + let after = src[end..].chars().next(); + let (can_open, can_close) = emphasis::flanking(ch, before, after); + if !can_open && !can_close { + self.pos = end; + return; + } + + let delim = Delim { + ch, + start, + count: size, + can_open, + can_close, + }; + self.push(Item::Delim(delim), end); + } + + fn open(&mut self, image: bool) { + let opener = Opener { + image, + start: self.pos, + active: true, + }; + let end = self.pos + if image { 2 } else { 1 }; + self.flush(); + self.openers.push(self.items.len()); + self.items.push(Item::Open(opener)); + self.pos = end; + self.run = end; + } + + fn close(&mut self) { + self.flush(); + if !link::close(self) { + // Not a link: the bracket stays plain text. + self.pos += 1; + } + } + + /// `<` starts an autolink or raw inline HTML. + fn angle(&mut self) { + let src = self.src(); + let pos = self.pos; + if let Some(autolink) = html::autolink(src, pos) { + let position = |start, end| Some(self.context.position(start, end)); + let node = Node::Link(crate::node::Link { + url: crate::node::Url(autolink.url), + title: None, + values: vec![Node::Text(Text { + value: src[autolink.text.0..autolink.text.1].to_string(), + position: position(autolink.text.0, autolink.text.1), + })], + position: position(pos, autolink.end), + }); + self.push_node(node, autolink.end); + } else if let Some(end) = html::inline_html(src, pos) { + let node = Node::Html(crate::node::Html { + value: src[pos..end].to_string(), + position: Some(self.context.position(pos, end)), + }); + self.push_node(node, end); + } else { + self.pos += 1; + } + } + + /// A JSX tag in MDX text. + fn jsx(&mut self) { + match mdx::tag(self.src(), self.pos) { + Parsed::Ok(tag) => { + let end = tag.end; + let item = Item::Jsx(TextTag { + name: tag.name, + attributes: tag.attributes, + kind: tag.kind, + start: self.pos, + end, + }); + self.push(item, end); + } + Parsed::Error(message) => { + self.context.fail(message); + self.pos += 1; + } + Parsed::More(Fallback::Error(message)) => { + self.context.fail(message); + self.pos += 1; + } + Parsed::Nok | Parsed::More(Fallback::Nok) => self.pos += 1, + } + } + + /// An expression in MDX text. + fn expression(&mut self) { + match mdx::expression(self.src(), self.pos) { + Parsed::Ok((end, value)) => { + let node = Node::MdxTextExpression(MdxTextExpression { + value, + position: Some(self.context.position(self.pos, end)), + }); + self.push_node(node, end); + } + Parsed::Error(message) => { + self.context.fail(message); + self.pos += 1; + } + Parsed::More(Fallback::Error(message)) => { + self.context.fail(message); + self.pos += 1; + } + Parsed::Nok | Parsed::More(Fallback::Nok) => self.pos += 1, + } + } + + fn literal_url(&mut self) { + literal::url(self); + } +} + +/// The length of the line ending at the start of `bytes`. +fn eol_len(bytes: &[u8]) -> usize { + match bytes { + [b'\r', b'\n', ..] => 2, + [b'\r' | b'\n', ..] => 1, + _ => 0, + } +} + +/// The value of a code or math span: the content as it is, except that one space is removed from +/// both ends when both have one and the content is not only spaces. +fn span_value(content: &str) -> SmolStr { + let bytes = content.as_bytes(); + if bytes.len() > 2 && bytes[0] == b' ' && bytes[bytes.len() - 1] == b' ' && bytes.iter().any(|&b| b != b' ') { + SmolStr::new(&content[1..content.len() - 1]) + } else { + SmolStr::new(content) + } +} diff --git a/crates/mq-markdown/src/parser/inline/emphasis.rs b/crates/mq-markdown/src/parser/inline/emphasis.rs new file mode 100644 index 000000000..f1c2ab1ec --- /dev/null +++ b/crates/mq-markdown/src/parser/inline/emphasis.rs @@ -0,0 +1,126 @@ +//! Emphasis, strong emphasis and strikethrough, resolved with a stack of delimiter runs. +//! +//! This follows the attention algorithm of `markdown-rs`, which differs from the reference +//! implementation of `CommonMark` in how runs next to other runs open and close, and in using the +//! remaining length of a run for the rule of three. + +use super::punctuation::is_punctuation; +use super::{Context, Delim, Item, to_nodes}; +use crate::node::{Delete, Emphasis, Node, Strong}; + +#[derive(PartialEq)] +enum Kind { + Whitespace, + Punctuation, + Other, +} + +fn kind(char: Option) -> Kind { + match char { + None => Kind::Whitespace, + Some(char) if char.is_whitespace() => Kind::Whitespace, + Some(char) if is_punctuation(char) => Kind::Punctuation, + Some(_) => Kind::Other, + } +} + +/// Whether a run of `ch` between `before` and `after` can open and can close. +pub(super) fn flanking(ch: u8, before: Option, after: Option) -> (bool, bool) { + let (before_kind, after_kind) = (kind(before), kind(after)); + let marker = |char: Option| matches!(char, Some('*' | '_' | '~')); + + let open = after_kind == Kind::Other + || (after_kind == Kind::Punctuation && before_kind != Kind::Other) + || (ch != b'~' && marker(after)); + let close = before_kind == Kind::Other + || (before_kind == Kind::Punctuation && after_kind != Kind::Other) + || (ch != b'~' && marker(before)); + + if ch == b'_' { + ( + open && (before_kind != Kind::Other || !close), + close && (after_kind != Kind::Other || !open), + ) + } else { + (open, close) + } +} + +/// Whether `opener` and `closer` can match, by character and by the rule of three. +fn matches(opener: &Delim, closer: &Delim) -> bool { + if !opener.can_open || opener.ch != closer.ch { + return false; + } + if (opener.can_close || closer.can_open) + && !closer.count.is_multiple_of(3) + && (opener.count + closer.count).is_multiple_of(3) + { + return false; + } + // Strikethrough matches runs of the same length, one or two tildes. + !(closer.ch == b'~' && (closer.count != opener.count || closer.count > 2)) +} + +/// Matches delimiters in `items[bottom..]`, replacing each pair and what is between them by a node. +pub(super) fn process(items: &mut Vec, bottom: usize, context: &Context<'_>) { + let mut closer = bottom; + + while closer < items.len() { + let Item::Delim(current) = &items[closer] else { + closer += 1; + continue; + }; + let opener = if current.can_close { + (bottom..closer).rev().find(|&index| match &items[index] { + Item::Delim(candidate) => matches(candidate, current), + _ => false, + }) + } else { + None + }; + let Some(opener) = opener else { + closer += 1; + continue; + }; + + let (open, close) = match (&items[opener], &items[closer]) { + (Item::Delim(open), Item::Delim(close)) => (open, close), + _ => unreachable!("both indexes are delimiters"), + }; + let used = if open.count > 1 && close.count > 1 { 2 } else { 1 }; + let ch = close.ch; + let start = open.start + open.count - used; + let end = close.start + used; + + let children = items.drain(opener + 1..closer).collect::>(); + let values = to_nodes(children, context); + let position = Some(context.position(start, end)); + let node = match (ch, used) { + (b'~', _) => Node::Delete(Delete { values, position }), + (_, 2) => Node::Strong(Strong { values, position }), + _ => Node::Emphasis(Emphasis { values, position }), + }; + + // The drained items are gone, so the closer is now right after the new node. + closer = opener + 1; + items.insert(closer, Item::Node(node)); + closer += 1; + + if let Item::Delim(open) = &mut items[opener] { + open.count -= used; + } + if let Item::Delim(close) = &mut items[closer] { + close.start += used; + close.count -= used; + } + + // Delimiters inside the new node cannot open anything else, and those used up are removed. + if matches!(&items[closer], Item::Delim(close) if close.count == 0) { + items.remove(closer); + } + if matches!(&items[opener], Item::Delim(open) if open.count == 0) { + items.remove(opener); + closer -= 1; + } + } +} diff --git a/crates/mq-markdown/src/parser/inline/entities.rs b/crates/mq-markdown/src/parser/inline/entities.rs new file mode 100644 index 000000000..8b6913480 --- /dev/null +++ b/crates/mq-markdown/src/parser/inline/entities.rs @@ -0,0 +1,2132 @@ +//! HTML character reference names and their values. +//! +//! Generated from the table of `markdown-rs` (MIT License, Titus Wormer), which follows the +//! [HTML standard](https://html.spec.whatwg.org/multipage/named-characters.html). Sorted by name. + +pub(super) static ENTITIES: [(&str, &str); 2125] = [ + ("AElig", "Æ"), + ("AMP", "&"), + ("Aacute", "Á"), + ("Abreve", "Ă"), + ("Acirc", "Â"), + ("Acy", "А"), + ("Afr", "𝔄"), + ("Agrave", "À"), + ("Alpha", "Α"), + ("Amacr", "Ā"), + ("And", "⩓"), + ("Aogon", "Ą"), + ("Aopf", "𝔸"), + ("ApplyFunction", "⁡"), + ("Aring", "Å"), + ("Ascr", "𝒜"), + ("Assign", "≔"), + ("Atilde", "Ã"), + ("Auml", "Ä"), + ("Backslash", "∖"), + ("Barv", "⫧"), + ("Barwed", "⌆"), + ("Bcy", "Б"), + ("Because", "∵"), + ("Bernoullis", "ℬ"), + ("Beta", "Β"), + ("Bfr", "𝔅"), + ("Bopf", "𝔹"), + ("Breve", "˘"), + ("Bscr", "ℬ"), + ("Bumpeq", "≎"), + ("CHcy", "Ч"), + ("COPY", "©"), + ("Cacute", "Ć"), + ("Cap", "⋒"), + ("CapitalDifferentialD", "ⅅ"), + ("Cayleys", "ℭ"), + ("Ccaron", "Č"), + ("Ccedil", "Ç"), + ("Ccirc", "Ĉ"), + ("Cconint", "∰"), + ("Cdot", "Ċ"), + ("Cedilla", "¸"), + ("CenterDot", "·"), + ("Cfr", "ℭ"), + ("Chi", "Χ"), + ("CircleDot", "⊙"), + ("CircleMinus", "⊖"), + ("CirclePlus", "⊕"), + ("CircleTimes", "⊗"), + ("ClockwiseContourIntegral", "∲"), + ("CloseCurlyDoubleQuote", "”"), + ("CloseCurlyQuote", "’"), + ("Colon", "∷"), + ("Colone", "⩴"), + ("Congruent", "≡"), + ("Conint", "∯"), + ("ContourIntegral", "∮"), + ("Copf", "ℂ"), + ("Coproduct", "∐"), + ("CounterClockwiseContourIntegral", "∳"), + ("Cross", "⨯"), + ("Cscr", "𝒞"), + ("Cup", "⋓"), + ("CupCap", "≍"), + ("DD", "ⅅ"), + ("DDotrahd", "⤑"), + ("DJcy", "Ђ"), + ("DScy", "Ѕ"), + ("DZcy", "Џ"), + ("Dagger", "‡"), + ("Darr", "↡"), + ("Dashv", "⫤"), + ("Dcaron", "Ď"), + ("Dcy", "Д"), + ("Del", "∇"), + ("Delta", "Δ"), + ("Dfr", "𝔇"), + ("DiacriticalAcute", "´"), + ("DiacriticalDot", "˙"), + ("DiacriticalDoubleAcute", "˝"), + ("DiacriticalGrave", "`"), + ("DiacriticalTilde", "˜"), + ("Diamond", "⋄"), + ("DifferentialD", "ⅆ"), + ("Dopf", "𝔻"), + ("Dot", "¨"), + ("DotDot", "⃜"), + ("DotEqual", "≐"), + ("DoubleContourIntegral", "∯"), + ("DoubleDot", "¨"), + ("DoubleDownArrow", "⇓"), + ("DoubleLeftArrow", "⇐"), + ("DoubleLeftRightArrow", "⇔"), + ("DoubleLeftTee", "⫤"), + ("DoubleLongLeftArrow", "⟸"), + ("DoubleLongLeftRightArrow", "⟺"), + ("DoubleLongRightArrow", "⟹"), + ("DoubleRightArrow", "⇒"), + ("DoubleRightTee", "⊨"), + ("DoubleUpArrow", "⇑"), + ("DoubleUpDownArrow", "⇕"), + ("DoubleVerticalBar", "∥"), + ("DownArrow", "↓"), + ("DownArrowBar", "⤓"), + ("DownArrowUpArrow", "⇵"), + ("DownBreve", "̑"), + ("DownLeftRightVector", "⥐"), + ("DownLeftTeeVector", "⥞"), + ("DownLeftVector", "↽"), + ("DownLeftVectorBar", "⥖"), + ("DownRightTeeVector", "⥟"), + ("DownRightVector", "⇁"), + ("DownRightVectorBar", "⥗"), + ("DownTee", "⊤"), + ("DownTeeArrow", "↧"), + ("Downarrow", "⇓"), + ("Dscr", "𝒟"), + ("Dstrok", "Đ"), + ("ENG", "Ŋ"), + ("ETH", "Ð"), + ("Eacute", "É"), + ("Ecaron", "Ě"), + ("Ecirc", "Ê"), + ("Ecy", "Э"), + ("Edot", "Ė"), + ("Efr", "𝔈"), + ("Egrave", "È"), + ("Element", "∈"), + ("Emacr", "Ē"), + ("EmptySmallSquare", "◻"), + ("EmptyVerySmallSquare", "▫"), + ("Eogon", "Ę"), + ("Eopf", "𝔼"), + ("Epsilon", "Ε"), + ("Equal", "⩵"), + ("EqualTilde", "≂"), + ("Equilibrium", "⇌"), + ("Escr", "ℰ"), + ("Esim", "⩳"), + ("Eta", "Η"), + ("Euml", "Ë"), + ("Exists", "∃"), + ("ExponentialE", "ⅇ"), + ("Fcy", "Ф"), + ("Ffr", "𝔉"), + ("FilledSmallSquare", "◼"), + ("FilledVerySmallSquare", "▪"), + ("Fopf", "𝔽"), + ("ForAll", "∀"), + ("Fouriertrf", "ℱ"), + ("Fscr", "ℱ"), + ("GJcy", "Ѓ"), + ("GT", ">"), + ("Gamma", "Γ"), + ("Gammad", "Ϝ"), + ("Gbreve", "Ğ"), + ("Gcedil", "Ģ"), + ("Gcirc", "Ĝ"), + ("Gcy", "Г"), + ("Gdot", "Ġ"), + ("Gfr", "𝔊"), + ("Gg", "⋙"), + ("Gopf", "𝔾"), + ("GreaterEqual", "≥"), + ("GreaterEqualLess", "⋛"), + ("GreaterFullEqual", "≧"), + ("GreaterGreater", "⪢"), + ("GreaterLess", "≷"), + ("GreaterSlantEqual", "⩾"), + ("GreaterTilde", "≳"), + ("Gscr", "𝒢"), + ("Gt", "≫"), + ("HARDcy", "Ъ"), + ("Hacek", "ˇ"), + ("Hat", "^"), + ("Hcirc", "Ĥ"), + ("Hfr", "ℌ"), + ("HilbertSpace", "ℋ"), + ("Hopf", "ℍ"), + ("HorizontalLine", "─"), + ("Hscr", "ℋ"), + ("Hstrok", "Ħ"), + ("HumpDownHump", "≎"), + ("HumpEqual", "≏"), + ("IEcy", "Е"), + ("IJlig", "IJ"), + ("IOcy", "Ё"), + ("Iacute", "Í"), + ("Icirc", "Î"), + ("Icy", "И"), + ("Idot", "İ"), + ("Ifr", "ℑ"), + ("Igrave", "Ì"), + ("Im", "ℑ"), + ("Imacr", "Ī"), + ("ImaginaryI", "ⅈ"), + ("Implies", "⇒"), + ("Int", "∬"), + ("Integral", "∫"), + ("Intersection", "⋂"), + ("InvisibleComma", "⁣"), + ("InvisibleTimes", "⁢"), + ("Iogon", "Į"), + ("Iopf", "𝕀"), + ("Iota", "Ι"), + ("Iscr", "ℐ"), + ("Itilde", "Ĩ"), + ("Iukcy", "І"), + ("Iuml", "Ï"), + ("Jcirc", "Ĵ"), + ("Jcy", "Й"), + ("Jfr", "𝔍"), + ("Jopf", "𝕁"), + ("Jscr", "𝒥"), + ("Jsercy", "Ј"), + ("Jukcy", "Є"), + ("KHcy", "Х"), + ("KJcy", "Ќ"), + ("Kappa", "Κ"), + ("Kcedil", "Ķ"), + ("Kcy", "К"), + ("Kfr", "𝔎"), + ("Kopf", "𝕂"), + ("Kscr", "𝒦"), + ("LJcy", "Љ"), + ("LT", "<"), + ("Lacute", "Ĺ"), + ("Lambda", "Λ"), + ("Lang", "⟪"), + ("Laplacetrf", "ℒ"), + ("Larr", "↞"), + ("Lcaron", "Ľ"), + ("Lcedil", "Ļ"), + ("Lcy", "Л"), + ("LeftAngleBracket", "⟨"), + ("LeftArrow", "←"), + ("LeftArrowBar", "⇤"), + ("LeftArrowRightArrow", "⇆"), + ("LeftCeiling", "⌈"), + ("LeftDoubleBracket", "⟦"), + ("LeftDownTeeVector", "⥡"), + ("LeftDownVector", "⇃"), + ("LeftDownVectorBar", "⥙"), + ("LeftFloor", "⌊"), + ("LeftRightArrow", "↔"), + ("LeftRightVector", "⥎"), + ("LeftTee", "⊣"), + ("LeftTeeArrow", "↤"), + ("LeftTeeVector", "⥚"), + ("LeftTriangle", "⊲"), + ("LeftTriangleBar", "⧏"), + ("LeftTriangleEqual", "⊴"), + ("LeftUpDownVector", "⥑"), + ("LeftUpTeeVector", "⥠"), + ("LeftUpVector", "↿"), + ("LeftUpVectorBar", "⥘"), + ("LeftVector", "↼"), + ("LeftVectorBar", "⥒"), + ("Leftarrow", "⇐"), + ("Leftrightarrow", "⇔"), + ("LessEqualGreater", "⋚"), + ("LessFullEqual", "≦"), + ("LessGreater", "≶"), + ("LessLess", "⪡"), + ("LessSlantEqual", "⩽"), + ("LessTilde", "≲"), + ("Lfr", "𝔏"), + ("Ll", "⋘"), + ("Lleftarrow", "⇚"), + ("Lmidot", "Ŀ"), + ("LongLeftArrow", "⟵"), + ("LongLeftRightArrow", "⟷"), + ("LongRightArrow", "⟶"), + ("Longleftarrow", "⟸"), + ("Longleftrightarrow", "⟺"), + ("Longrightarrow", "⟹"), + ("Lopf", "𝕃"), + ("LowerLeftArrow", "↙"), + ("LowerRightArrow", "↘"), + ("Lscr", "ℒ"), + ("Lsh", "↰"), + ("Lstrok", "Ł"), + ("Lt", "≪"), + ("Map", "⤅"), + ("Mcy", "М"), + ("MediumSpace", " "), + ("Mellintrf", "ℳ"), + ("Mfr", "𝔐"), + ("MinusPlus", "∓"), + ("Mopf", "𝕄"), + ("Mscr", "ℳ"), + ("Mu", "Μ"), + ("NJcy", "Њ"), + ("Nacute", "Ń"), + ("Ncaron", "Ň"), + ("Ncedil", "Ņ"), + ("Ncy", "Н"), + ("NegativeMediumSpace", "\u{200B}"), + ("NegativeThickSpace", "\u{200B}"), + ("NegativeThinSpace", "\u{200B}"), + ("NegativeVeryThinSpace", "\u{200B}"), + ("NestedGreaterGreater", "≫"), + ("NestedLessLess", "≪"), + ("NewLine", "\n"), + ("Nfr", "𝔑"), + ("NoBreak", "\u{2060}"), + ("NonBreakingSpace", " "), + ("Nopf", "ℕ"), + ("Not", "⫬"), + ("NotCongruent", "≢"), + ("NotCupCap", "≭"), + ("NotDoubleVerticalBar", "∦"), + ("NotElement", "∉"), + ("NotEqual", "≠"), + ("NotEqualTilde", "≂̸"), + ("NotExists", "∄"), + ("NotGreater", "≯"), + ("NotGreaterEqual", "≱"), + ("NotGreaterFullEqual", "≧̸"), + ("NotGreaterGreater", "≫̸"), + ("NotGreaterLess", "≹"), + ("NotGreaterSlantEqual", "⩾̸"), + ("NotGreaterTilde", "≵"), + ("NotHumpDownHump", "≎̸"), + ("NotHumpEqual", "≏̸"), + ("NotLeftTriangle", "⋪"), + ("NotLeftTriangleBar", "⧏̸"), + ("NotLeftTriangleEqual", "⋬"), + ("NotLess", "≮"), + ("NotLessEqual", "≰"), + ("NotLessGreater", "≸"), + ("NotLessLess", "≪̸"), + ("NotLessSlantEqual", "⩽̸"), + ("NotLessTilde", "≴"), + ("NotNestedGreaterGreater", "⪢̸"), + ("NotNestedLessLess", "⪡̸"), + ("NotPrecedes", "⊀"), + ("NotPrecedesEqual", "⪯̸"), + ("NotPrecedesSlantEqual", "⋠"), + ("NotReverseElement", "∌"), + ("NotRightTriangle", "⋫"), + ("NotRightTriangleBar", "⧐̸"), + ("NotRightTriangleEqual", "⋭"), + ("NotSquareSubset", "⊏̸"), + ("NotSquareSubsetEqual", "⋢"), + ("NotSquareSuperset", "⊐̸"), + ("NotSquareSupersetEqual", "⋣"), + ("NotSubset", "⊂⃒"), + ("NotSubsetEqual", "⊈"), + ("NotSucceeds", "⊁"), + ("NotSucceedsEqual", "⪰̸"), + ("NotSucceedsSlantEqual", "⋡"), + ("NotSucceedsTilde", "≿̸"), + ("NotSuperset", "⊃⃒"), + ("NotSupersetEqual", "⊉"), + ("NotTilde", "≁"), + ("NotTildeEqual", "≄"), + ("NotTildeFullEqual", "≇"), + ("NotTildeTilde", "≉"), + ("NotVerticalBar", "∤"), + ("Nscr", "𝒩"), + ("Ntilde", "Ñ"), + ("Nu", "Ν"), + ("OElig", "Œ"), + ("Oacute", "Ó"), + ("Ocirc", "Ô"), + ("Ocy", "О"), + ("Odblac", "Ő"), + ("Ofr", "𝔒"), + ("Ograve", "Ò"), + ("Omacr", "Ō"), + ("Omega", "Ω"), + ("Omicron", "Ο"), + ("Oopf", "𝕆"), + ("OpenCurlyDoubleQuote", "“"), + ("OpenCurlyQuote", "‘"), + ("Or", "⩔"), + ("Oscr", "𝒪"), + ("Oslash", "Ø"), + ("Otilde", "Õ"), + ("Otimes", "⨷"), + ("Ouml", "Ö"), + ("OverBar", "‾"), + ("OverBrace", "⏞"), + ("OverBracket", "⎴"), + ("OverParenthesis", "⏜"), + ("PartialD", "∂"), + ("Pcy", "П"), + ("Pfr", "𝔓"), + ("Phi", "Φ"), + ("Pi", "Π"), + ("PlusMinus", "±"), + ("Poincareplane", "ℌ"), + ("Popf", "ℙ"), + ("Pr", "⪻"), + ("Precedes", "≺"), + ("PrecedesEqual", "⪯"), + ("PrecedesSlantEqual", "≼"), + ("PrecedesTilde", "≾"), + ("Prime", "″"), + ("Product", "∏"), + ("Proportion", "∷"), + ("Proportional", "∝"), + ("Pscr", "𝒫"), + ("Psi", "Ψ"), + ("QUOT", "\""), + ("Qfr", "𝔔"), + ("Qopf", "ℚ"), + ("Qscr", "𝒬"), + ("RBarr", "⤐"), + ("REG", "®"), + ("Racute", "Ŕ"), + ("Rang", "⟫"), + ("Rarr", "↠"), + ("Rarrtl", "⤖"), + ("Rcaron", "Ř"), + ("Rcedil", "Ŗ"), + ("Rcy", "Р"), + ("Re", "ℜ"), + ("ReverseElement", "∋"), + ("ReverseEquilibrium", "⇋"), + ("ReverseUpEquilibrium", "⥯"), + ("Rfr", "ℜ"), + ("Rho", "Ρ"), + ("RightAngleBracket", "⟩"), + ("RightArrow", "→"), + ("RightArrowBar", "⇥"), + ("RightArrowLeftArrow", "⇄"), + ("RightCeiling", "⌉"), + ("RightDoubleBracket", "⟧"), + ("RightDownTeeVector", "⥝"), + ("RightDownVector", "⇂"), + ("RightDownVectorBar", "⥕"), + ("RightFloor", "⌋"), + ("RightTee", "⊢"), + ("RightTeeArrow", "↦"), + ("RightTeeVector", "⥛"), + ("RightTriangle", "⊳"), + ("RightTriangleBar", "⧐"), + ("RightTriangleEqual", "⊵"), + ("RightUpDownVector", "⥏"), + ("RightUpTeeVector", "⥜"), + ("RightUpVector", "↾"), + ("RightUpVectorBar", "⥔"), + ("RightVector", "⇀"), + ("RightVectorBar", "⥓"), + ("Rightarrow", "⇒"), + ("Ropf", "ℝ"), + ("RoundImplies", "⥰"), + ("Rrightarrow", "⇛"), + ("Rscr", "ℛ"), + ("Rsh", "↱"), + ("RuleDelayed", "⧴"), + ("SHCHcy", "Щ"), + ("SHcy", "Ш"), + ("SOFTcy", "Ь"), + ("Sacute", "Ś"), + ("Sc", "⪼"), + ("Scaron", "Š"), + ("Scedil", "Ş"), + ("Scirc", "Ŝ"), + ("Scy", "С"), + ("Sfr", "𝔖"), + ("ShortDownArrow", "↓"), + ("ShortLeftArrow", "←"), + ("ShortRightArrow", "→"), + ("ShortUpArrow", "↑"), + ("Sigma", "Σ"), + ("SmallCircle", "∘"), + ("Sopf", "𝕊"), + ("Sqrt", "√"), + ("Square", "□"), + ("SquareIntersection", "⊓"), + ("SquareSubset", "⊏"), + ("SquareSubsetEqual", "⊑"), + ("SquareSuperset", "⊐"), + ("SquareSupersetEqual", "⊒"), + ("SquareUnion", "⊔"), + ("Sscr", "𝒮"), + ("Star", "⋆"), + ("Sub", "⋐"), + ("Subset", "⋐"), + ("SubsetEqual", "⊆"), + ("Succeeds", "≻"), + ("SucceedsEqual", "⪰"), + ("SucceedsSlantEqual", "≽"), + ("SucceedsTilde", "≿"), + ("SuchThat", "∋"), + ("Sum", "∑"), + ("Sup", "⋑"), + ("Superset", "⊃"), + ("SupersetEqual", "⊇"), + ("Supset", "⋑"), + ("THORN", "Þ"), + ("TRADE", "™"), + ("TSHcy", "Ћ"), + ("TScy", "Ц"), + ("Tab", "\t"), + ("Tau", "Τ"), + ("Tcaron", "Ť"), + ("Tcedil", "Ţ"), + ("Tcy", "Т"), + ("Tfr", "𝔗"), + ("Therefore", "∴"), + ("Theta", "Θ"), + ("ThickSpace", "  "), + ("ThinSpace", " "), + ("Tilde", "∼"), + ("TildeEqual", "≃"), + ("TildeFullEqual", "≅"), + ("TildeTilde", "≈"), + ("Topf", "𝕋"), + ("TripleDot", "⃛"), + ("Tscr", "𝒯"), + ("Tstrok", "Ŧ"), + ("Uacute", "Ú"), + ("Uarr", "↟"), + ("Uarrocir", "⥉"), + ("Ubrcy", "Ў"), + ("Ubreve", "Ŭ"), + ("Ucirc", "Û"), + ("Ucy", "У"), + ("Udblac", "Ű"), + ("Ufr", "𝔘"), + ("Ugrave", "Ù"), + ("Umacr", "Ū"), + ("UnderBar", "_"), + ("UnderBrace", "⏟"), + ("UnderBracket", "⎵"), + ("UnderParenthesis", "⏝"), + ("Union", "⋃"), + ("UnionPlus", "⊎"), + ("Uogon", "Ų"), + ("Uopf", "𝕌"), + ("UpArrow", "↑"), + ("UpArrowBar", "⤒"), + ("UpArrowDownArrow", "⇅"), + ("UpDownArrow", "↕"), + ("UpEquilibrium", "⥮"), + ("UpTee", "⊥"), + ("UpTeeArrow", "↥"), + ("Uparrow", "⇑"), + ("Updownarrow", "⇕"), + ("UpperLeftArrow", "↖"), + ("UpperRightArrow", "↗"), + ("Upsi", "ϒ"), + ("Upsilon", "Υ"), + ("Uring", "Ů"), + ("Uscr", "𝒰"), + ("Utilde", "Ũ"), + ("Uuml", "Ü"), + ("VDash", "⊫"), + ("Vbar", "⫫"), + ("Vcy", "В"), + ("Vdash", "⊩"), + ("Vdashl", "⫦"), + ("Vee", "⋁"), + ("Verbar", "‖"), + ("Vert", "‖"), + ("VerticalBar", "∣"), + ("VerticalLine", "|"), + ("VerticalSeparator", "❘"), + ("VerticalTilde", "≀"), + ("VeryThinSpace", " "), + ("Vfr", "𝔙"), + ("Vopf", "𝕍"), + ("Vscr", "𝒱"), + ("Vvdash", "⊪"), + ("Wcirc", "Ŵ"), + ("Wedge", "⋀"), + ("Wfr", "𝔚"), + ("Wopf", "𝕎"), + ("Wscr", "𝒲"), + ("Xfr", "𝔛"), + ("Xi", "Ξ"), + ("Xopf", "𝕏"), + ("Xscr", "𝒳"), + ("YAcy", "Я"), + ("YIcy", "Ї"), + ("YUcy", "Ю"), + ("Yacute", "Ý"), + ("Ycirc", "Ŷ"), + ("Ycy", "Ы"), + ("Yfr", "𝔜"), + ("Yopf", "𝕐"), + ("Yscr", "𝒴"), + ("Yuml", "Ÿ"), + ("ZHcy", "Ж"), + ("Zacute", "Ź"), + ("Zcaron", "Ž"), + ("Zcy", "З"), + ("Zdot", "Ż"), + ("ZeroWidthSpace", "\u{200B}"), + ("Zeta", "Ζ"), + ("Zfr", "ℨ"), + ("Zopf", "ℤ"), + ("Zscr", "𝒵"), + ("aacute", "á"), + ("abreve", "ă"), + ("ac", "∾"), + ("acE", "∾̳"), + ("acd", "∿"), + ("acirc", "â"), + ("acute", "´"), + ("acy", "а"), + ("aelig", "æ"), + ("af", "⁡"), + ("afr", "𝔞"), + ("agrave", "à"), + ("alefsym", "ℵ"), + ("aleph", "ℵ"), + ("alpha", "α"), + ("amacr", "ā"), + ("amalg", "⨿"), + ("amp", "&"), + ("and", "∧"), + ("andand", "⩕"), + ("andd", "⩜"), + ("andslope", "⩘"), + ("andv", "⩚"), + ("ang", "∠"), + ("ange", "⦤"), + ("angle", "∠"), + ("angmsd", "∡"), + ("angmsdaa", "⦨"), + ("angmsdab", "⦩"), + ("angmsdac", "⦪"), + ("angmsdad", "⦫"), + ("angmsdae", "⦬"), + ("angmsdaf", "⦭"), + ("angmsdag", "⦮"), + ("angmsdah", "⦯"), + ("angrt", "∟"), + ("angrtvb", "⊾"), + ("angrtvbd", "⦝"), + ("angsph", "∢"), + ("angst", "Å"), + ("angzarr", "⍼"), + ("aogon", "ą"), + ("aopf", "𝕒"), + ("ap", "≈"), + ("apE", "⩰"), + ("apacir", "⩯"), + ("ape", "≊"), + ("apid", "≋"), + ("apos", "'"), + ("approx", "≈"), + ("approxeq", "≊"), + ("aring", "å"), + ("ascr", "𝒶"), + ("ast", "*"), + ("asymp", "≈"), + ("asympeq", "≍"), + ("atilde", "ã"), + ("auml", "ä"), + ("awconint", "∳"), + ("awint", "⨑"), + ("bNot", "⫭"), + ("backcong", "≌"), + ("backepsilon", "϶"), + ("backprime", "‵"), + ("backsim", "∽"), + ("backsimeq", "⋍"), + ("barvee", "⊽"), + ("barwed", "⌅"), + ("barwedge", "⌅"), + ("bbrk", "⎵"), + ("bbrktbrk", "⎶"), + ("bcong", "≌"), + ("bcy", "б"), + ("bdquo", "„"), + ("becaus", "∵"), + ("because", "∵"), + ("bemptyv", "⦰"), + ("bepsi", "϶"), + ("bernou", "ℬ"), + ("beta", "β"), + ("beth", "ℶ"), + ("between", "≬"), + ("bfr", "𝔟"), + ("bigcap", "⋂"), + ("bigcirc", "◯"), + ("bigcup", "⋃"), + ("bigodot", "⨀"), + ("bigoplus", "⨁"), + ("bigotimes", "⨂"), + ("bigsqcup", "⨆"), + ("bigstar", "★"), + ("bigtriangledown", "▽"), + ("bigtriangleup", "△"), + ("biguplus", "⨄"), + ("bigvee", "⋁"), + ("bigwedge", "⋀"), + ("bkarow", "⤍"), + ("blacklozenge", "⧫"), + ("blacksquare", "▪"), + ("blacktriangle", "▴"), + ("blacktriangledown", "▾"), + ("blacktriangleleft", "◂"), + ("blacktriangleright", "▸"), + ("blank", "␣"), + ("blk12", "▒"), + ("blk14", "░"), + ("blk34", "▓"), + ("block", "█"), + ("bne", "=⃥"), + ("bnequiv", "≡⃥"), + ("bnot", "⌐"), + ("bopf", "𝕓"), + ("bot", "⊥"), + ("bottom", "⊥"), + ("bowtie", "⋈"), + ("boxDL", "╗"), + ("boxDR", "╔"), + ("boxDl", "╖"), + ("boxDr", "╓"), + ("boxH", "═"), + ("boxHD", "╦"), + ("boxHU", "╩"), + ("boxHd", "╤"), + ("boxHu", "╧"), + ("boxUL", "╝"), + ("boxUR", "╚"), + ("boxUl", "╜"), + ("boxUr", "╙"), + ("boxV", "║"), + ("boxVH", "╬"), + ("boxVL", "╣"), + ("boxVR", "╠"), + ("boxVh", "╫"), + ("boxVl", "╢"), + ("boxVr", "╟"), + ("boxbox", "⧉"), + ("boxdL", "╕"), + ("boxdR", "╒"), + ("boxdl", "┐"), + ("boxdr", "┌"), + ("boxh", "─"), + ("boxhD", "╥"), + ("boxhU", "╨"), + ("boxhd", "┬"), + ("boxhu", "┴"), + ("boxminus", "⊟"), + ("boxplus", "⊞"), + ("boxtimes", "⊠"), + ("boxuL", "╛"), + ("boxuR", "╘"), + ("boxul", "┘"), + ("boxur", "└"), + ("boxv", "│"), + ("boxvH", "╪"), + ("boxvL", "╡"), + ("boxvR", "╞"), + ("boxvh", "┼"), + ("boxvl", "┤"), + ("boxvr", "├"), + ("bprime", "‵"), + ("breve", "˘"), + ("brvbar", "¦"), + ("bscr", "𝒷"), + ("bsemi", "⁏"), + ("bsim", "∽"), + ("bsime", "⋍"), + ("bsol", "\\"), + ("bsolb", "⧅"), + ("bsolhsub", "⟈"), + ("bull", "•"), + ("bullet", "•"), + ("bump", "≎"), + ("bumpE", "⪮"), + ("bumpe", "≏"), + ("bumpeq", "≏"), + ("cacute", "ć"), + ("cap", "∩"), + ("capand", "⩄"), + ("capbrcup", "⩉"), + ("capcap", "⩋"), + ("capcup", "⩇"), + ("capdot", "⩀"), + ("caps", "∩︀"), + ("caret", "⁁"), + ("caron", "ˇ"), + ("ccaps", "⩍"), + ("ccaron", "č"), + ("ccedil", "ç"), + ("ccirc", "ĉ"), + ("ccups", "⩌"), + ("ccupssm", "⩐"), + ("cdot", "ċ"), + ("cedil", "¸"), + ("cemptyv", "⦲"), + ("cent", "¢"), + ("centerdot", "·"), + ("cfr", "𝔠"), + ("chcy", "ч"), + ("check", "✓"), + ("checkmark", "✓"), + ("chi", "χ"), + ("cir", "○"), + ("cirE", "⧃"), + ("circ", "ˆ"), + ("circeq", "≗"), + ("circlearrowleft", "↺"), + ("circlearrowright", "↻"), + ("circledR", "®"), + ("circledS", "Ⓢ"), + ("circledast", "⊛"), + ("circledcirc", "⊚"), + ("circleddash", "⊝"), + ("cire", "≗"), + ("cirfnint", "⨐"), + ("cirmid", "⫯"), + ("cirscir", "⧂"), + ("clubs", "♣"), + ("clubsuit", "♣"), + ("colon", ":"), + ("colone", "≔"), + ("coloneq", "≔"), + ("comma", ","), + ("commat", "@"), + ("comp", "∁"), + ("compfn", "∘"), + ("complement", "∁"), + ("complexes", "ℂ"), + ("cong", "≅"), + ("congdot", "⩭"), + ("conint", "∮"), + ("copf", "𝕔"), + ("coprod", "∐"), + ("copy", "©"), + ("copysr", "℗"), + ("crarr", "↵"), + ("cross", "✗"), + ("cscr", "𝒸"), + ("csub", "⫏"), + ("csube", "⫑"), + ("csup", "⫐"), + ("csupe", "⫒"), + ("ctdot", "⋯"), + ("cudarrl", "⤸"), + ("cudarrr", "⤵"), + ("cuepr", "⋞"), + ("cuesc", "⋟"), + ("cularr", "↶"), + ("cularrp", "⤽"), + ("cup", "∪"), + ("cupbrcap", "⩈"), + ("cupcap", "⩆"), + ("cupcup", "⩊"), + ("cupdot", "⊍"), + ("cupor", "⩅"), + ("cups", "∪︀"), + ("curarr", "↷"), + ("curarrm", "⤼"), + ("curlyeqprec", "⋞"), + ("curlyeqsucc", "⋟"), + ("curlyvee", "⋎"), + ("curlywedge", "⋏"), + ("curren", "¤"), + ("curvearrowleft", "↶"), + ("curvearrowright", "↷"), + ("cuvee", "⋎"), + ("cuwed", "⋏"), + ("cwconint", "∲"), + ("cwint", "∱"), + ("cylcty", "⌭"), + ("dArr", "⇓"), + ("dHar", "⥥"), + ("dagger", "†"), + ("daleth", "ℸ"), + ("darr", "↓"), + ("dash", "‐"), + ("dashv", "⊣"), + ("dbkarow", "⤏"), + ("dblac", "˝"), + ("dcaron", "ď"), + ("dcy", "д"), + ("dd", "ⅆ"), + ("ddagger", "‡"), + ("ddarr", "⇊"), + ("ddotseq", "⩷"), + ("deg", "°"), + ("delta", "δ"), + ("demptyv", "⦱"), + ("dfisht", "⥿"), + ("dfr", "𝔡"), + ("dharl", "⇃"), + ("dharr", "⇂"), + ("diam", "⋄"), + ("diamond", "⋄"), + ("diamondsuit", "♦"), + ("diams", "♦"), + ("die", "¨"), + ("digamma", "ϝ"), + ("disin", "⋲"), + ("div", "÷"), + ("divide", "÷"), + ("divideontimes", "⋇"), + ("divonx", "⋇"), + ("djcy", "ђ"), + ("dlcorn", "⌞"), + ("dlcrop", "⌍"), + ("dollar", "$"), + ("dopf", "𝕕"), + ("dot", "˙"), + ("doteq", "≐"), + ("doteqdot", "≑"), + ("dotminus", "∸"), + ("dotplus", "∔"), + ("dotsquare", "⊡"), + ("doublebarwedge", "⌆"), + ("downarrow", "↓"), + ("downdownarrows", "⇊"), + ("downharpoonleft", "⇃"), + ("downharpoonright", "⇂"), + ("drbkarow", "⤐"), + ("drcorn", "⌟"), + ("drcrop", "⌌"), + ("dscr", "𝒹"), + ("dscy", "ѕ"), + ("dsol", "⧶"), + ("dstrok", "đ"), + ("dtdot", "⋱"), + ("dtri", "▿"), + ("dtrif", "▾"), + ("duarr", "⇵"), + ("duhar", "⥯"), + ("dwangle", "⦦"), + ("dzcy", "џ"), + ("dzigrarr", "⟿"), + ("eDDot", "⩷"), + ("eDot", "≑"), + ("eacute", "é"), + ("easter", "⩮"), + ("ecaron", "ě"), + ("ecir", "≖"), + ("ecirc", "ê"), + ("ecolon", "≕"), + ("ecy", "э"), + ("edot", "ė"), + ("ee", "ⅇ"), + ("efDot", "≒"), + ("efr", "𝔢"), + ("eg", "⪚"), + ("egrave", "è"), + ("egs", "⪖"), + ("egsdot", "⪘"), + ("el", "⪙"), + ("elinters", "⏧"), + ("ell", "ℓ"), + ("els", "⪕"), + ("elsdot", "⪗"), + ("emacr", "ē"), + ("empty", "∅"), + ("emptyset", "∅"), + ("emptyv", "∅"), + ("emsp", " "), + ("emsp13", " "), + ("emsp14", " "), + ("eng", "ŋ"), + ("ensp", " "), + ("eogon", "ę"), + ("eopf", "𝕖"), + ("epar", "⋕"), + ("eparsl", "⧣"), + ("eplus", "⩱"), + ("epsi", "ε"), + ("epsilon", "ε"), + ("epsiv", "ϵ"), + ("eqcirc", "≖"), + ("eqcolon", "≕"), + ("eqsim", "≂"), + ("eqslantgtr", "⪖"), + ("eqslantless", "⪕"), + ("equals", "="), + ("equest", "≟"), + ("equiv", "≡"), + ("equivDD", "⩸"), + ("eqvparsl", "⧥"), + ("erDot", "≓"), + ("erarr", "⥱"), + ("escr", "ℯ"), + ("esdot", "≐"), + ("esim", "≂"), + ("eta", "η"), + ("eth", "ð"), + ("euml", "ë"), + ("euro", "€"), + ("excl", "!"), + ("exist", "∃"), + ("expectation", "ℰ"), + ("exponentiale", "ⅇ"), + ("fallingdotseq", "≒"), + ("fcy", "ф"), + ("female", "♀"), + ("ffilig", "ffi"), + ("fflig", "ff"), + ("ffllig", "ffl"), + ("ffr", "𝔣"), + ("filig", "fi"), + ("fjlig", "fj"), + ("flat", "♭"), + ("fllig", "fl"), + ("fltns", "▱"), + ("fnof", "ƒ"), + ("fopf", "𝕗"), + ("forall", "∀"), + ("fork", "⋔"), + ("forkv", "⫙"), + ("fpartint", "⨍"), + ("frac12", "½"), + ("frac13", "⅓"), + ("frac14", "¼"), + ("frac15", "⅕"), + ("frac16", "⅙"), + ("frac18", "⅛"), + ("frac23", "⅔"), + ("frac25", "⅖"), + ("frac34", "¾"), + ("frac35", "⅗"), + ("frac38", "⅜"), + ("frac45", "⅘"), + ("frac56", "⅚"), + ("frac58", "⅝"), + ("frac78", "⅞"), + ("frasl", "⁄"), + ("frown", "⌢"), + ("fscr", "𝒻"), + ("gE", "≧"), + ("gEl", "⪌"), + ("gacute", "ǵ"), + ("gamma", "γ"), + ("gammad", "ϝ"), + ("gap", "⪆"), + ("gbreve", "ğ"), + ("gcirc", "ĝ"), + ("gcy", "г"), + ("gdot", "ġ"), + ("ge", "≥"), + ("gel", "⋛"), + ("geq", "≥"), + ("geqq", "≧"), + ("geqslant", "⩾"), + ("ges", "⩾"), + ("gescc", "⪩"), + ("gesdot", "⪀"), + ("gesdoto", "⪂"), + ("gesdotol", "⪄"), + ("gesl", "⋛︀"), + ("gesles", "⪔"), + ("gfr", "𝔤"), + ("gg", "≫"), + ("ggg", "⋙"), + ("gimel", "ℷ"), + ("gjcy", "ѓ"), + ("gl", "≷"), + ("glE", "⪒"), + ("gla", "⪥"), + ("glj", "⪤"), + ("gnE", "≩"), + ("gnap", "⪊"), + ("gnapprox", "⪊"), + ("gne", "⪈"), + ("gneq", "⪈"), + ("gneqq", "≩"), + ("gnsim", "⋧"), + ("gopf", "𝕘"), + ("grave", "`"), + ("gscr", "ℊ"), + ("gsim", "≳"), + ("gsime", "⪎"), + ("gsiml", "⪐"), + ("gt", ">"), + ("gtcc", "⪧"), + ("gtcir", "⩺"), + ("gtdot", "⋗"), + ("gtlPar", "⦕"), + ("gtquest", "⩼"), + ("gtrapprox", "⪆"), + ("gtrarr", "⥸"), + ("gtrdot", "⋗"), + ("gtreqless", "⋛"), + ("gtreqqless", "⪌"), + ("gtrless", "≷"), + ("gtrsim", "≳"), + ("gvertneqq", "≩︀"), + ("gvnE", "≩︀"), + ("hArr", "⇔"), + ("hairsp", " "), + ("half", "½"), + ("hamilt", "ℋ"), + ("hardcy", "ъ"), + ("harr", "↔"), + ("harrcir", "⥈"), + ("harrw", "↭"), + ("hbar", "ℏ"), + ("hcirc", "ĥ"), + ("hearts", "♥"), + ("heartsuit", "♥"), + ("hellip", "…"), + ("hercon", "⊹"), + ("hfr", "𝔥"), + ("hksearow", "⤥"), + ("hkswarow", "⤦"), + ("hoarr", "⇿"), + ("homtht", "∻"), + ("hookleftarrow", "↩"), + ("hookrightarrow", "↪"), + ("hopf", "𝕙"), + ("horbar", "―"), + ("hscr", "𝒽"), + ("hslash", "ℏ"), + ("hstrok", "ħ"), + ("hybull", "⁃"), + ("hyphen", "‐"), + ("iacute", "í"), + ("ic", "⁣"), + ("icirc", "î"), + ("icy", "и"), + ("iecy", "е"), + ("iexcl", "¡"), + ("iff", "⇔"), + ("ifr", "𝔦"), + ("igrave", "ì"), + ("ii", "ⅈ"), + ("iiiint", "⨌"), + ("iiint", "∭"), + ("iinfin", "⧜"), + ("iiota", "℩"), + ("ijlig", "ij"), + ("imacr", "ī"), + ("image", "ℑ"), + ("imagline", "ℐ"), + ("imagpart", "ℑ"), + ("imath", "ı"), + ("imof", "⊷"), + ("imped", "Ƶ"), + ("in", "∈"), + ("incare", "℅"), + ("infin", "∞"), + ("infintie", "⧝"), + ("inodot", "ı"), + ("int", "∫"), + ("intcal", "⊺"), + ("integers", "ℤ"), + ("intercal", "⊺"), + ("intlarhk", "⨗"), + ("intprod", "⨼"), + ("iocy", "ё"), + ("iogon", "į"), + ("iopf", "𝕚"), + ("iota", "ι"), + ("iprod", "⨼"), + ("iquest", "¿"), + ("iscr", "𝒾"), + ("isin", "∈"), + ("isinE", "⋹"), + ("isindot", "⋵"), + ("isins", "⋴"), + ("isinsv", "⋳"), + ("isinv", "∈"), + ("it", "⁢"), + ("itilde", "ĩ"), + ("iukcy", "і"), + ("iuml", "ï"), + ("jcirc", "ĵ"), + ("jcy", "й"), + ("jfr", "𝔧"), + ("jmath", "ȷ"), + ("jopf", "𝕛"), + ("jscr", "𝒿"), + ("jsercy", "ј"), + ("jukcy", "є"), + ("kappa", "κ"), + ("kappav", "ϰ"), + ("kcedil", "ķ"), + ("kcy", "к"), + ("kfr", "𝔨"), + ("kgreen", "ĸ"), + ("khcy", "х"), + ("kjcy", "ќ"), + ("kopf", "𝕜"), + ("kscr", "𝓀"), + ("lAarr", "⇚"), + ("lArr", "⇐"), + ("lAtail", "⤛"), + ("lBarr", "⤎"), + ("lE", "≦"), + ("lEg", "⪋"), + ("lHar", "⥢"), + ("lacute", "ĺ"), + ("laemptyv", "⦴"), + ("lagran", "ℒ"), + ("lambda", "λ"), + ("lang", "⟨"), + ("langd", "⦑"), + ("langle", "⟨"), + ("lap", "⪅"), + ("laquo", "«"), + ("larr", "←"), + ("larrb", "⇤"), + ("larrbfs", "⤟"), + ("larrfs", "⤝"), + ("larrhk", "↩"), + ("larrlp", "↫"), + ("larrpl", "⤹"), + ("larrsim", "⥳"), + ("larrtl", "↢"), + ("lat", "⪫"), + ("latail", "⤙"), + ("late", "⪭"), + ("lates", "⪭︀"), + ("lbarr", "⤌"), + ("lbbrk", "❲"), + ("lbrace", "{"), + ("lbrack", "["), + ("lbrke", "⦋"), + ("lbrksld", "⦏"), + ("lbrkslu", "⦍"), + ("lcaron", "ľ"), + ("lcedil", "ļ"), + ("lceil", "⌈"), + ("lcub", "{"), + ("lcy", "л"), + ("ldca", "⤶"), + ("ldquo", "“"), + ("ldquor", "„"), + ("ldrdhar", "⥧"), + ("ldrushar", "⥋"), + ("ldsh", "↲"), + ("le", "≤"), + ("leftarrow", "←"), + ("leftarrowtail", "↢"), + ("leftharpoondown", "↽"), + ("leftharpoonup", "↼"), + ("leftleftarrows", "⇇"), + ("leftrightarrow", "↔"), + ("leftrightarrows", "⇆"), + ("leftrightharpoons", "⇋"), + ("leftrightsquigarrow", "↭"), + ("leftthreetimes", "⋋"), + ("leg", "⋚"), + ("leq", "≤"), + ("leqq", "≦"), + ("leqslant", "⩽"), + ("les", "⩽"), + ("lescc", "⪨"), + ("lesdot", "⩿"), + ("lesdoto", "⪁"), + ("lesdotor", "⪃"), + ("lesg", "⋚︀"), + ("lesges", "⪓"), + ("lessapprox", "⪅"), + ("lessdot", "⋖"), + ("lesseqgtr", "⋚"), + ("lesseqqgtr", "⪋"), + ("lessgtr", "≶"), + ("lesssim", "≲"), + ("lfisht", "⥼"), + ("lfloor", "⌊"), + ("lfr", "𝔩"), + ("lg", "≶"), + ("lgE", "⪑"), + ("lhard", "↽"), + ("lharu", "↼"), + ("lharul", "⥪"), + ("lhblk", "▄"), + ("ljcy", "љ"), + ("ll", "≪"), + ("llarr", "⇇"), + ("llcorner", "⌞"), + ("llhard", "⥫"), + ("lltri", "◺"), + ("lmidot", "ŀ"), + ("lmoust", "⎰"), + ("lmoustache", "⎰"), + ("lnE", "≨"), + ("lnap", "⪉"), + ("lnapprox", "⪉"), + ("lne", "⪇"), + ("lneq", "⪇"), + ("lneqq", "≨"), + ("lnsim", "⋦"), + ("loang", "⟬"), + ("loarr", "⇽"), + ("lobrk", "⟦"), + ("longleftarrow", "⟵"), + ("longleftrightarrow", "⟷"), + ("longmapsto", "⟼"), + ("longrightarrow", "⟶"), + ("looparrowleft", "↫"), + ("looparrowright", "↬"), + ("lopar", "⦅"), + ("lopf", "𝕝"), + ("loplus", "⨭"), + ("lotimes", "⨴"), + ("lowast", "∗"), + ("lowbar", "_"), + ("loz", "◊"), + ("lozenge", "◊"), + ("lozf", "⧫"), + ("lpar", "("), + ("lparlt", "⦓"), + ("lrarr", "⇆"), + ("lrcorner", "⌟"), + ("lrhar", "⇋"), + ("lrhard", "⥭"), + ("lrm", "‎"), + ("lrtri", "⊿"), + ("lsaquo", "‹"), + ("lscr", "𝓁"), + ("lsh", "↰"), + ("lsim", "≲"), + ("lsime", "⪍"), + ("lsimg", "⪏"), + ("lsqb", "["), + ("lsquo", "‘"), + ("lsquor", "‚"), + ("lstrok", "ł"), + ("lt", "<"), + ("ltcc", "⪦"), + ("ltcir", "⩹"), + ("ltdot", "⋖"), + ("lthree", "⋋"), + ("ltimes", "⋉"), + ("ltlarr", "⥶"), + ("ltquest", "⩻"), + ("ltrPar", "⦖"), + ("ltri", "◃"), + ("ltrie", "⊴"), + ("ltrif", "◂"), + ("lurdshar", "⥊"), + ("luruhar", "⥦"), + ("lvertneqq", "≨︀"), + ("lvnE", "≨︀"), + ("mDDot", "∺"), + ("macr", "¯"), + ("male", "♂"), + ("malt", "✠"), + ("maltese", "✠"), + ("map", "↦"), + ("mapsto", "↦"), + ("mapstodown", "↧"), + ("mapstoleft", "↤"), + ("mapstoup", "↥"), + ("marker", "▮"), + ("mcomma", "⨩"), + ("mcy", "м"), + ("mdash", "—"), + ("measuredangle", "∡"), + ("mfr", "𝔪"), + ("mho", "℧"), + ("micro", "µ"), + ("mid", "∣"), + ("midast", "*"), + ("midcir", "⫰"), + ("middot", "·"), + ("minus", "−"), + ("minusb", "⊟"), + ("minusd", "∸"), + ("minusdu", "⨪"), + ("mlcp", "⫛"), + ("mldr", "…"), + ("mnplus", "∓"), + ("models", "⊧"), + ("mopf", "𝕞"), + ("mp", "∓"), + ("mscr", "𝓂"), + ("mstpos", "∾"), + ("mu", "μ"), + ("multimap", "⊸"), + ("mumap", "⊸"), + ("nGg", "⋙̸"), + ("nGt", "≫⃒"), + ("nGtv", "≫̸"), + ("nLeftarrow", "⇍"), + ("nLeftrightarrow", "⇎"), + ("nLl", "⋘̸"), + ("nLt", "≪⃒"), + ("nLtv", "≪̸"), + ("nRightarrow", "⇏"), + ("nVDash", "⊯"), + ("nVdash", "⊮"), + ("nabla", "∇"), + ("nacute", "ń"), + ("nang", "∠⃒"), + ("nap", "≉"), + ("napE", "⩰̸"), + ("napid", "≋̸"), + ("napos", "ʼn"), + ("napprox", "≉"), + ("natur", "♮"), + ("natural", "♮"), + ("naturals", "ℕ"), + ("nbsp", " "), + ("nbump", "≎̸"), + ("nbumpe", "≏̸"), + ("ncap", "⩃"), + ("ncaron", "ň"), + ("ncedil", "ņ"), + ("ncong", "≇"), + ("ncongdot", "⩭̸"), + ("ncup", "⩂"), + ("ncy", "н"), + ("ndash", "–"), + ("ne", "≠"), + ("neArr", "⇗"), + ("nearhk", "⤤"), + ("nearr", "↗"), + ("nearrow", "↗"), + ("nedot", "≐̸"), + ("nequiv", "≢"), + ("nesear", "⤨"), + ("nesim", "≂̸"), + ("nexist", "∄"), + ("nexists", "∄"), + ("nfr", "𝔫"), + ("ngE", "≧̸"), + ("nge", "≱"), + ("ngeq", "≱"), + ("ngeqq", "≧̸"), + ("ngeqslant", "⩾̸"), + ("nges", "⩾̸"), + ("ngsim", "≵"), + ("ngt", "≯"), + ("ngtr", "≯"), + ("nhArr", "⇎"), + ("nharr", "↮"), + ("nhpar", "⫲"), + ("ni", "∋"), + ("nis", "⋼"), + ("nisd", "⋺"), + ("niv", "∋"), + ("njcy", "њ"), + ("nlArr", "⇍"), + ("nlE", "≦̸"), + ("nlarr", "↚"), + ("nldr", "‥"), + ("nle", "≰"), + ("nleftarrow", "↚"), + ("nleftrightarrow", "↮"), + ("nleq", "≰"), + ("nleqq", "≦̸"), + ("nleqslant", "⩽̸"), + ("nles", "⩽̸"), + ("nless", "≮"), + ("nlsim", "≴"), + ("nlt", "≮"), + ("nltri", "⋪"), + ("nltrie", "⋬"), + ("nmid", "∤"), + ("nopf", "𝕟"), + ("not", "¬"), + ("notin", "∉"), + ("notinE", "⋹̸"), + ("notindot", "⋵̸"), + ("notinva", "∉"), + ("notinvb", "⋷"), + ("notinvc", "⋶"), + ("notni", "∌"), + ("notniva", "∌"), + ("notnivb", "⋾"), + ("notnivc", "⋽"), + ("npar", "∦"), + ("nparallel", "∦"), + ("nparsl", "⫽⃥"), + ("npart", "∂̸"), + ("npolint", "⨔"), + ("npr", "⊀"), + ("nprcue", "⋠"), + ("npre", "⪯̸"), + ("nprec", "⊀"), + ("npreceq", "⪯̸"), + ("nrArr", "⇏"), + ("nrarr", "↛"), + ("nrarrc", "⤳̸"), + ("nrarrw", "↝̸"), + ("nrightarrow", "↛"), + ("nrtri", "⋫"), + ("nrtrie", "⋭"), + ("nsc", "⊁"), + ("nsccue", "⋡"), + ("nsce", "⪰̸"), + ("nscr", "𝓃"), + ("nshortmid", "∤"), + ("nshortparallel", "∦"), + ("nsim", "≁"), + ("nsime", "≄"), + ("nsimeq", "≄"), + ("nsmid", "∤"), + ("nspar", "∦"), + ("nsqsube", "⋢"), + ("nsqsupe", "⋣"), + ("nsub", "⊄"), + ("nsubE", "⫅̸"), + ("nsube", "⊈"), + ("nsubset", "⊂⃒"), + ("nsubseteq", "⊈"), + ("nsubseteqq", "⫅̸"), + ("nsucc", "⊁"), + ("nsucceq", "⪰̸"), + ("nsup", "⊅"), + ("nsupE", "⫆̸"), + ("nsupe", "⊉"), + ("nsupset", "⊃⃒"), + ("nsupseteq", "⊉"), + ("nsupseteqq", "⫆̸"), + ("ntgl", "≹"), + ("ntilde", "ñ"), + ("ntlg", "≸"), + ("ntriangleleft", "⋪"), + ("ntrianglelefteq", "⋬"), + ("ntriangleright", "⋫"), + ("ntrianglerighteq", "⋭"), + ("nu", "ν"), + ("num", "#"), + ("numero", "№"), + ("numsp", " "), + ("nvDash", "⊭"), + ("nvHarr", "⤄"), + ("nvap", "≍⃒"), + ("nvdash", "⊬"), + ("nvge", "≥⃒"), + ("nvgt", ">⃒"), + ("nvinfin", "⧞"), + ("nvlArr", "⤂"), + ("nvle", "≤⃒"), + ("nvlt", "<⃒"), + ("nvltrie", "⊴⃒"), + ("nvrArr", "⤃"), + ("nvrtrie", "⊵⃒"), + ("nvsim", "∼⃒"), + ("nwArr", "⇖"), + ("nwarhk", "⤣"), + ("nwarr", "↖"), + ("nwarrow", "↖"), + ("nwnear", "⤧"), + ("oS", "Ⓢ"), + ("oacute", "ó"), + ("oast", "⊛"), + ("ocir", "⊚"), + ("ocirc", "ô"), + ("ocy", "о"), + ("odash", "⊝"), + ("odblac", "ő"), + ("odiv", "⨸"), + ("odot", "⊙"), + ("odsold", "⦼"), + ("oelig", "œ"), + ("ofcir", "⦿"), + ("ofr", "𝔬"), + ("ogon", "˛"), + ("ograve", "ò"), + ("ogt", "⧁"), + ("ohbar", "⦵"), + ("ohm", "Ω"), + ("oint", "∮"), + ("olarr", "↺"), + ("olcir", "⦾"), + ("olcross", "⦻"), + ("oline", "‾"), + ("olt", "⧀"), + ("omacr", "ō"), + ("omega", "ω"), + ("omicron", "ο"), + ("omid", "⦶"), + ("ominus", "⊖"), + ("oopf", "𝕠"), + ("opar", "⦷"), + ("operp", "⦹"), + ("oplus", "⊕"), + ("or", "∨"), + ("orarr", "↻"), + ("ord", "⩝"), + ("order", "ℴ"), + ("orderof", "ℴ"), + ("ordf", "ª"), + ("ordm", "º"), + ("origof", "⊶"), + ("oror", "⩖"), + ("orslope", "⩗"), + ("orv", "⩛"), + ("oscr", "ℴ"), + ("oslash", "ø"), + ("osol", "⊘"), + ("otilde", "õ"), + ("otimes", "⊗"), + ("otimesas", "⨶"), + ("ouml", "ö"), + ("ovbar", "⌽"), + ("par", "∥"), + ("para", "¶"), + ("parallel", "∥"), + ("parsim", "⫳"), + ("parsl", "⫽"), + ("part", "∂"), + ("pcy", "п"), + ("percnt", "%"), + ("period", "."), + ("permil", "‰"), + ("perp", "⊥"), + ("pertenk", "‱"), + ("pfr", "𝔭"), + ("phi", "φ"), + ("phiv", "ϕ"), + ("phmmat", "ℳ"), + ("phone", "☎"), + ("pi", "π"), + ("pitchfork", "⋔"), + ("piv", "ϖ"), + ("planck", "ℏ"), + ("planckh", "ℎ"), + ("plankv", "ℏ"), + ("plus", "+"), + ("plusacir", "⨣"), + ("plusb", "⊞"), + ("pluscir", "⨢"), + ("plusdo", "∔"), + ("plusdu", "⨥"), + ("pluse", "⩲"), + ("plusmn", "±"), + ("plussim", "⨦"), + ("plustwo", "⨧"), + ("pm", "±"), + ("pointint", "⨕"), + ("popf", "𝕡"), + ("pound", "£"), + ("pr", "≺"), + ("prE", "⪳"), + ("prap", "⪷"), + ("prcue", "≼"), + ("pre", "⪯"), + ("prec", "≺"), + ("precapprox", "⪷"), + ("preccurlyeq", "≼"), + ("preceq", "⪯"), + ("precnapprox", "⪹"), + ("precneqq", "⪵"), + ("precnsim", "⋨"), + ("precsim", "≾"), + ("prime", "′"), + ("primes", "ℙ"), + ("prnE", "⪵"), + ("prnap", "⪹"), + ("prnsim", "⋨"), + ("prod", "∏"), + ("profalar", "⌮"), + ("profline", "⌒"), + ("profsurf", "⌓"), + ("prop", "∝"), + ("propto", "∝"), + ("prsim", "≾"), + ("prurel", "⊰"), + ("pscr", "𝓅"), + ("psi", "ψ"), + ("puncsp", " "), + ("qfr", "𝔮"), + ("qint", "⨌"), + ("qopf", "𝕢"), + ("qprime", "⁗"), + ("qscr", "𝓆"), + ("quaternions", "ℍ"), + ("quatint", "⨖"), + ("quest", "?"), + ("questeq", "≟"), + ("quot", "\""), + ("rAarr", "⇛"), + ("rArr", "⇒"), + ("rAtail", "⤜"), + ("rBarr", "⤏"), + ("rHar", "⥤"), + ("race", "∽̱"), + ("racute", "ŕ"), + ("radic", "√"), + ("raemptyv", "⦳"), + ("rang", "⟩"), + ("rangd", "⦒"), + ("range", "⦥"), + ("rangle", "⟩"), + ("raquo", "»"), + ("rarr", "→"), + ("rarrap", "⥵"), + ("rarrb", "⇥"), + ("rarrbfs", "⤠"), + ("rarrc", "⤳"), + ("rarrfs", "⤞"), + ("rarrhk", "↪"), + ("rarrlp", "↬"), + ("rarrpl", "⥅"), + ("rarrsim", "⥴"), + ("rarrtl", "↣"), + ("rarrw", "↝"), + ("ratail", "⤚"), + ("ratio", "∶"), + ("rationals", "ℚ"), + ("rbarr", "⤍"), + ("rbbrk", "❳"), + ("rbrace", "}"), + ("rbrack", "]"), + ("rbrke", "⦌"), + ("rbrksld", "⦎"), + ("rbrkslu", "⦐"), + ("rcaron", "ř"), + ("rcedil", "ŗ"), + ("rceil", "⌉"), + ("rcub", "}"), + ("rcy", "р"), + ("rdca", "⤷"), + ("rdldhar", "⥩"), + ("rdquo", "”"), + ("rdquor", "”"), + ("rdsh", "↳"), + ("real", "ℜ"), + ("realine", "ℛ"), + ("realpart", "ℜ"), + ("reals", "ℝ"), + ("rect", "▭"), + ("reg", "®"), + ("rfisht", "⥽"), + ("rfloor", "⌋"), + ("rfr", "𝔯"), + ("rhard", "⇁"), + ("rharu", "⇀"), + ("rharul", "⥬"), + ("rho", "ρ"), + ("rhov", "ϱ"), + ("rightarrow", "→"), + ("rightarrowtail", "↣"), + ("rightharpoondown", "⇁"), + ("rightharpoonup", "⇀"), + ("rightleftarrows", "⇄"), + ("rightleftharpoons", "⇌"), + ("rightrightarrows", "⇉"), + ("rightsquigarrow", "↝"), + ("rightthreetimes", "⋌"), + ("ring", "˚"), + ("risingdotseq", "≓"), + ("rlarr", "⇄"), + ("rlhar", "⇌"), + ("rlm", "‏"), + ("rmoust", "⎱"), + ("rmoustache", "⎱"), + ("rnmid", "⫮"), + ("roang", "⟭"), + ("roarr", "⇾"), + ("robrk", "⟧"), + ("ropar", "⦆"), + ("ropf", "𝕣"), + ("roplus", "⨮"), + ("rotimes", "⨵"), + ("rpar", ")"), + ("rpargt", "⦔"), + ("rppolint", "⨒"), + ("rrarr", "⇉"), + ("rsaquo", "›"), + ("rscr", "𝓇"), + ("rsh", "↱"), + ("rsqb", "]"), + ("rsquo", "’"), + ("rsquor", "’"), + ("rthree", "⋌"), + ("rtimes", "⋊"), + ("rtri", "▹"), + ("rtrie", "⊵"), + ("rtrif", "▸"), + ("rtriltri", "⧎"), + ("ruluhar", "⥨"), + ("rx", "℞"), + ("sacute", "ś"), + ("sbquo", "‚"), + ("sc", "≻"), + ("scE", "⪴"), + ("scap", "⪸"), + ("scaron", "š"), + ("sccue", "≽"), + ("sce", "⪰"), + ("scedil", "ş"), + ("scirc", "ŝ"), + ("scnE", "⪶"), + ("scnap", "⪺"), + ("scnsim", "⋩"), + ("scpolint", "⨓"), + ("scsim", "≿"), + ("scy", "с"), + ("sdot", "⋅"), + ("sdotb", "⊡"), + ("sdote", "⩦"), + ("seArr", "⇘"), + ("searhk", "⤥"), + ("searr", "↘"), + ("searrow", "↘"), + ("sect", "§"), + ("semi", ";"), + ("seswar", "⤩"), + ("setminus", "∖"), + ("setmn", "∖"), + ("sext", "✶"), + ("sfr", "𝔰"), + ("sfrown", "⌢"), + ("sharp", "♯"), + ("shchcy", "щ"), + ("shcy", "ш"), + ("shortmid", "∣"), + ("shortparallel", "∥"), + ("shy", "\u{AD}"), + ("sigma", "σ"), + ("sigmaf", "ς"), + ("sigmav", "ς"), + ("sim", "∼"), + ("simdot", "⩪"), + ("sime", "≃"), + ("simeq", "≃"), + ("simg", "⪞"), + ("simgE", "⪠"), + ("siml", "⪝"), + ("simlE", "⪟"), + ("simne", "≆"), + ("simplus", "⨤"), + ("simrarr", "⥲"), + ("slarr", "←"), + ("smallsetminus", "∖"), + ("smashp", "⨳"), + ("smeparsl", "⧤"), + ("smid", "∣"), + ("smile", "⌣"), + ("smt", "⪪"), + ("smte", "⪬"), + ("smtes", "⪬︀"), + ("softcy", "ь"), + ("sol", "/"), + ("solb", "⧄"), + ("solbar", "⌿"), + ("sopf", "𝕤"), + ("spades", "♠"), + ("spadesuit", "♠"), + ("spar", "∥"), + ("sqcap", "⊓"), + ("sqcaps", "⊓︀"), + ("sqcup", "⊔"), + ("sqcups", "⊔︀"), + ("sqsub", "⊏"), + ("sqsube", "⊑"), + ("sqsubset", "⊏"), + ("sqsubseteq", "⊑"), + ("sqsup", "⊐"), + ("sqsupe", "⊒"), + ("sqsupset", "⊐"), + ("sqsupseteq", "⊒"), + ("squ", "□"), + ("square", "□"), + ("squarf", "▪"), + ("squf", "▪"), + ("srarr", "→"), + ("sscr", "𝓈"), + ("ssetmn", "∖"), + ("ssmile", "⌣"), + ("sstarf", "⋆"), + ("star", "☆"), + ("starf", "★"), + ("straightepsilon", "ϵ"), + ("straightphi", "ϕ"), + ("strns", "¯"), + ("sub", "⊂"), + ("subE", "⫅"), + ("subdot", "⪽"), + ("sube", "⊆"), + ("subedot", "⫃"), + ("submult", "⫁"), + ("subnE", "⫋"), + ("subne", "⊊"), + ("subplus", "⪿"), + ("subrarr", "⥹"), + ("subset", "⊂"), + ("subseteq", "⊆"), + ("subseteqq", "⫅"), + ("subsetneq", "⊊"), + ("subsetneqq", "⫋"), + ("subsim", "⫇"), + ("subsub", "⫕"), + ("subsup", "⫓"), + ("succ", "≻"), + ("succapprox", "⪸"), + ("succcurlyeq", "≽"), + ("succeq", "⪰"), + ("succnapprox", "⪺"), + ("succneqq", "⪶"), + ("succnsim", "⋩"), + ("succsim", "≿"), + ("sum", "∑"), + ("sung", "♪"), + ("sup", "⊃"), + ("sup1", "¹"), + ("sup2", "²"), + ("sup3", "³"), + ("supE", "⫆"), + ("supdot", "⪾"), + ("supdsub", "⫘"), + ("supe", "⊇"), + ("supedot", "⫄"), + ("suphsol", "⟉"), + ("suphsub", "⫗"), + ("suplarr", "⥻"), + ("supmult", "⫂"), + ("supnE", "⫌"), + ("supne", "⊋"), + ("supplus", "⫀"), + ("supset", "⊃"), + ("supseteq", "⊇"), + ("supseteqq", "⫆"), + ("supsetneq", "⊋"), + ("supsetneqq", "⫌"), + ("supsim", "⫈"), + ("supsub", "⫔"), + ("supsup", "⫖"), + ("swArr", "⇙"), + ("swarhk", "⤦"), + ("swarr", "↙"), + ("swarrow", "↙"), + ("swnwar", "⤪"), + ("szlig", "ß"), + ("target", "⌖"), + ("tau", "τ"), + ("tbrk", "⎴"), + ("tcaron", "ť"), + ("tcedil", "ţ"), + ("tcy", "т"), + ("tdot", "⃛"), + ("telrec", "⌕"), + ("tfr", "𝔱"), + ("there4", "∴"), + ("therefore", "∴"), + ("theta", "θ"), + ("thetasym", "ϑ"), + ("thetav", "ϑ"), + ("thickapprox", "≈"), + ("thicksim", "∼"), + ("thinsp", " "), + ("thkap", "≈"), + ("thksim", "∼"), + ("thorn", "þ"), + ("tilde", "˜"), + ("times", "×"), + ("timesb", "⊠"), + ("timesbar", "⨱"), + ("timesd", "⨰"), + ("tint", "∭"), + ("toea", "⤨"), + ("top", "⊤"), + ("topbot", "⌶"), + ("topcir", "⫱"), + ("topf", "𝕥"), + ("topfork", "⫚"), + ("tosa", "⤩"), + ("tprime", "‴"), + ("trade", "™"), + ("triangle", "▵"), + ("triangledown", "▿"), + ("triangleleft", "◃"), + ("trianglelefteq", "⊴"), + ("triangleq", "≜"), + ("triangleright", "▹"), + ("trianglerighteq", "⊵"), + ("tridot", "◬"), + ("trie", "≜"), + ("triminus", "⨺"), + ("triplus", "⨹"), + ("trisb", "⧍"), + ("tritime", "⨻"), + ("trpezium", "⏢"), + ("tscr", "𝓉"), + ("tscy", "ц"), + ("tshcy", "ћ"), + ("tstrok", "ŧ"), + ("twixt", "≬"), + ("twoheadleftarrow", "↞"), + ("twoheadrightarrow", "↠"), + ("uArr", "⇑"), + ("uHar", "⥣"), + ("uacute", "ú"), + ("uarr", "↑"), + ("ubrcy", "ў"), + ("ubreve", "ŭ"), + ("ucirc", "û"), + ("ucy", "у"), + ("udarr", "⇅"), + ("udblac", "ű"), + ("udhar", "⥮"), + ("ufisht", "⥾"), + ("ufr", "𝔲"), + ("ugrave", "ù"), + ("uharl", "↿"), + ("uharr", "↾"), + ("uhblk", "▀"), + ("ulcorn", "⌜"), + ("ulcorner", "⌜"), + ("ulcrop", "⌏"), + ("ultri", "◸"), + ("umacr", "ū"), + ("uml", "¨"), + ("uogon", "ų"), + ("uopf", "𝕦"), + ("uparrow", "↑"), + ("updownarrow", "↕"), + ("upharpoonleft", "↿"), + ("upharpoonright", "↾"), + ("uplus", "⊎"), + ("upsi", "υ"), + ("upsih", "ϒ"), + ("upsilon", "υ"), + ("upuparrows", "⇈"), + ("urcorn", "⌝"), + ("urcorner", "⌝"), + ("urcrop", "⌎"), + ("uring", "ů"), + ("urtri", "◹"), + ("uscr", "𝓊"), + ("utdot", "⋰"), + ("utilde", "ũ"), + ("utri", "▵"), + ("utrif", "▴"), + ("uuarr", "⇈"), + ("uuml", "ü"), + ("uwangle", "⦧"), + ("vArr", "⇕"), + ("vBar", "⫨"), + ("vBarv", "⫩"), + ("vDash", "⊨"), + ("vangrt", "⦜"), + ("varepsilon", "ϵ"), + ("varkappa", "ϰ"), + ("varnothing", "∅"), + ("varphi", "ϕ"), + ("varpi", "ϖ"), + ("varpropto", "∝"), + ("varr", "↕"), + ("varrho", "ϱ"), + ("varsigma", "ς"), + ("varsubsetneq", "⊊︀"), + ("varsubsetneqq", "⫋︀"), + ("varsupsetneq", "⊋︀"), + ("varsupsetneqq", "⫌︀"), + ("vartheta", "ϑ"), + ("vartriangleleft", "⊲"), + ("vartriangleright", "⊳"), + ("vcy", "в"), + ("vdash", "⊢"), + ("vee", "∨"), + ("veebar", "⊻"), + ("veeeq", "≚"), + ("vellip", "⋮"), + ("verbar", "|"), + ("vert", "|"), + ("vfr", "𝔳"), + ("vltri", "⊲"), + ("vnsub", "⊂⃒"), + ("vnsup", "⊃⃒"), + ("vopf", "𝕧"), + ("vprop", "∝"), + ("vrtri", "⊳"), + ("vscr", "𝓋"), + ("vsubnE", "⫋︀"), + ("vsubne", "⊊︀"), + ("vsupnE", "⫌︀"), + ("vsupne", "⊋︀"), + ("vzigzag", "⦚"), + ("wcirc", "ŵ"), + ("wedbar", "⩟"), + ("wedge", "∧"), + ("wedgeq", "≙"), + ("weierp", "℘"), + ("wfr", "𝔴"), + ("wopf", "𝕨"), + ("wp", "℘"), + ("wr", "≀"), + ("wreath", "≀"), + ("wscr", "𝓌"), + ("xcap", "⋂"), + ("xcirc", "◯"), + ("xcup", "⋃"), + ("xdtri", "▽"), + ("xfr", "𝔵"), + ("xhArr", "⟺"), + ("xharr", "⟷"), + ("xi", "ξ"), + ("xlArr", "⟸"), + ("xlarr", "⟵"), + ("xmap", "⟼"), + ("xnis", "⋻"), + ("xodot", "⨀"), + ("xopf", "𝕩"), + ("xoplus", "⨁"), + ("xotime", "⨂"), + ("xrArr", "⟹"), + ("xrarr", "⟶"), + ("xscr", "𝓍"), + ("xsqcup", "⨆"), + ("xuplus", "⨄"), + ("xutri", "△"), + ("xvee", "⋁"), + ("xwedge", "⋀"), + ("yacute", "ý"), + ("yacy", "я"), + ("ycirc", "ŷ"), + ("ycy", "ы"), + ("yen", "¥"), + ("yfr", "𝔶"), + ("yicy", "ї"), + ("yopf", "𝕪"), + ("yscr", "𝓎"), + ("yucy", "ю"), + ("yuml", "ÿ"), + ("zacute", "ź"), + ("zcaron", "ž"), + ("zcy", "з"), + ("zdot", "ż"), + ("zeetrf", "ℨ"), + ("zeta", "ζ"), + ("zfr", "𝔷"), + ("zhcy", "ж"), + ("zigrarr", "⇝"), + ("zopf", "𝕫"), + ("zscr", "𝓏"), + ("zwj", "‍"), + ("zwnj", "‌"), +]; diff --git a/crates/mq-markdown/src/parser/inline/entity.rs b/crates/mq-markdown/src/parser/inline/entity.rs new file mode 100644 index 000000000..3300f6c80 --- /dev/null +++ b/crates/mq-markdown/src/parser/inline/entity.rs @@ -0,0 +1,115 @@ +//! Character references (`&`, `#`, `#`) and backslash escapes. + +use super::entities::ENTITIES; + +/// Decodes the character reference that starts at `pos`, returning the offset after it and its value. +pub(super) fn decode(src: &str, pos: usize) -> Option<(usize, String)> { + let bytes = src.as_bytes(); + debug_assert_eq!(bytes[pos], b'&'); + let rest = &bytes[pos + 1..]; + + if rest.first() == Some(&b'#') { + let (radix, digits_from, max) = match rest.get(1) { + Some(b'x' | b'X') => (16, 2, 6), + _ => (10, 1, 7), + }; + let digits = rest[digits_from..] + .iter() + .take_while(|b| (radix == 16 && b.is_ascii_hexdigit()) || b.is_ascii_digit()) + .count(); + if digits == 0 || digits > max || rest.get(digits_from + digits) != Some(&b';') { + return None; + } + let number = u32::from_str_radix(&src[pos + 1 + digits_from..pos + 1 + digits_from + digits], radix).ok()?; + let char = match char::from_u32(number) { + Some(char) if number != 0 => char, + _ => char::REPLACEMENT_CHARACTER, + }; + return Some((pos + 1 + digits_from + digits + 1, char.to_string())); + } + + let name = rest.iter().take_while(|b| b.is_ascii_alphanumeric()).count(); + if name == 0 || name > 31 || rest.get(name) != Some(&b';') { + return None; + } + let name = &src[pos + 1..pos + 1 + name]; + let index = ENTITIES + .binary_search_by(|(key, _)| key.as_bytes().cmp(name.as_bytes())) + .ok()?; + Some((pos + 1 + name.len() + 1, ENTITIES[index].1.to_string())) +} + +/// Decodes backslash escapes and character references, as in link destinations and titles. +pub(in crate::parser) fn unescape(value: &str) -> String { + let bytes = value.as_bytes(); + let mut result = String::with_capacity(value.len()); + let mut from = 0; + let mut index = 0; + + while index < bytes.len() { + match bytes[index] { + b'\\' if bytes.get(index + 1).is_some_and(u8::is_ascii_punctuation) => { + result.push_str(&value[from..index]); + result.push(bytes[index + 1] as char); + index += 2; + from = index; + } + b'&' => match decode(value, index) { + Some((end, decoded)) => { + result.push_str(&value[from..index]); + result.push_str(&decoded); + index = end; + from = index; + } + None => index += 1, + }, + _ => index += 1, + } + } + result.push_str(&value[from..]); + + result +} + +/// Decodes character references only, as in the string values of JSX attributes. +pub(in crate::parser) fn decode_references(value: &str) -> String { + let bytes = value.as_bytes(); + let mut result = String::with_capacity(value.len()); + let mut from = 0; + let mut index = 0; + + while index < bytes.len() { + if bytes[index] == b'&' + && let Some((end, decoded)) = decode(value, index) + { + result.push_str(&value[from..index]); + result.push_str(&decoded); + index = end; + from = index; + } else { + index += 1; + } + } + result.push_str(&value[from..]); + + result +} + +/// Removes the indentation that follows line endings, as labels of definitions and references show it. +pub(in crate::parser) fn remove_line_indent(value: &str) -> std::borrow::Cow<'_, str> { + if !value.contains(['\n', '\r']) { + return std::borrow::Cow::Borrowed(value); + } + let mut result = String::with_capacity(value.len()); + let mut after_line_ending = false; + for char in value.chars() { + match char { + ' ' | '\t' if after_line_ending => {} + _ => { + after_line_ending = matches!(char, '\n' | '\r'); + result.push(char); + } + } + } + std::borrow::Cow::Owned(result) +} diff --git a/crates/mq-markdown/src/parser/inline/html.rs b/crates/mq-markdown/src/parser/inline/html.rs new file mode 100644 index 000000000..c25b1e4e8 --- /dev/null +++ b/crates/mq-markdown/src/parser/inline/html.rs @@ -0,0 +1,197 @@ +//! Autolinks (``) and raw inline HTML. + +/// An autolink found at `<`. +pub(super) struct Autolink { + pub(super) url: String, + /// Start and end of the link text, inside the angle brackets. + pub(super) text: (usize, usize), + /// Offset after the closing `>`. + pub(super) end: usize, +} + +/// Parses an autolink at `pos` (a `<`): an absolute URI or an email address. +pub(super) fn autolink(src: &str, pos: usize) -> Option { + let bytes = src.as_bytes(); + let inner = pos + 1; + + let scheme_char = |b: &u8| b.is_ascii_alphanumeric() || matches!(b, b'+' | b'-' | b'.'); + let scheme = bytes + .get(inner) + .is_some_and(u8::is_ascii_alphabetic) + .then(|| { + 1 + bytes[inner + 1..] + .iter() + .take(31) + .take_while(|b| scheme_char(b)) + .count() + }) + .filter(|&length| length >= 2 && bytes.get(inner + length) == Some(&b':')); + + if let Some(length) = scheme { + // Everything up to the closing `>` is the URL. + let from = inner + length + 1; + let close = from + + bytes[from..] + .iter() + .position(|&b| b == b'>' || b <= 0x1F || matches!(b, b' ' | b'<' | 0x7F))?; + return (bytes[close] == b'>').then(|| Autolink { + url: src[inner..close].to_string(), + text: (inner, close), + end: close + 1, + }); + } + + let end = email(bytes, inner)?; + Some(Autolink { + url: format!("mailto:{}", &src[inner..end]), + text: (inner, end), + end: end + 1, + }) +} + +/// Parses an email address for an autolink at `start`, returning the offset of the closing `>`. +/// The characters allowed differ slightly from `CommonMark`, following `markdown-rs`. +fn email(bytes: &[u8], start: usize) -> Option { + let atext = |b: u8| matches!(b, b'#'..=b'\'' | b'*' | b'+' | b'-'..=b'9' | b'=' | b'?' | b'A'..=b'Z' | b'^'..=b'~'); + let mut index = start; + while bytes.get(index).copied().is_some_and(atext) { + index += 1; + } + if index == start || bytes.get(index) != Some(&b'@') { + return None; + } + index += 1; + + // Labels of at most 63 letters, digits and hyphens, starting and ending with a letter or digit. + loop { + let label = index; + if !bytes.get(index).is_some_and(u8::is_ascii_alphanumeric) { + return None; + } + while bytes + .get(index) + .is_some_and(|b| b.is_ascii_alphanumeric() || *b == b'-') + { + index += 1; + } + if index - label > 63 || bytes[index - 1] == b'-' { + return None; + } + match bytes.get(index) { + Some(b'.') => index += 1, + Some(b'>') => return Some(index), + _ => return None, + } + } +} + +/// Parses raw inline HTML at `pos` (a `<`), returning the offset after it. +pub(super) fn inline_html(src: &str, pos: usize) -> Option { + let rest = &src[pos..]; + let bytes = rest.as_bytes(); + + if let Some(after) = rest.strip_prefix("") { + return Some(pos + 5); + } + if rest.starts_with("") { + return Some(pos + 6); + } + return after.find("-->").map(|index| pos + 4 + index + 3); + } + if let Some(after) = rest.strip_prefix("").map(|index| pos + 2 + index + 2); + } + if let Some(after) = rest.strip_prefix("").map(|index| pos + 9 + index + 3); + } + if rest.starts_with("').map(|index| pos + index + 1); + } + if rest.starts_with(" bool { + matches!(byte, b' ' | b'\t' | b'\n' | b'\r') +} + +/// The length of a tag name at the start of `bytes`. +fn tag_name(bytes: &[u8]) -> usize { + if !bytes.first().is_some_and(u8::is_ascii_alphabetic) { + return 0; + } + bytes + .iter() + .take_while(|b| b.is_ascii_alphanumeric() || **b == b'-') + .count() +} + +fn skip_spaces(bytes: &[u8], mut index: usize) -> usize { + while bytes.get(index).copied().is_some_and(is_space) { + index += 1; + } + index +} + +/// The length of a closing tag such as `` at the start of `bytes`. +fn closing_tag(bytes: &[u8]) -> Option { + let name = tag_name(&bytes[2..]); + if name == 0 { + return None; + } + let index = skip_spaces(bytes, 2 + name); + (bytes.get(index) == Some(&b'>')).then_some(index + 1) +} + +/// The length of an open tag such as `` at the start of `bytes`. +fn open_tag(bytes: &[u8]) -> Option { + let name = tag_name(&bytes[1..]); + if name == 0 { + return None; + } + let mut index = 1 + name; + + loop { + let after_space = skip_spaces(bytes, index); + let has_space = after_space > index; + match bytes.get(after_space)? { + b'>' => return Some(after_space + 1), + b'/' => return (bytes.get(after_space + 1) == Some(&b'>')).then_some(after_space + 2), + byte if has_space && (byte.is_ascii_alphabetic() || matches!(byte, b'_' | b':')) => { + index = attribute(bytes, after_space)?; + } + _ => return None, + } + } +} + +/// The offset after the attribute (name and optional value) that starts at `start`. +fn attribute(bytes: &[u8], start: usize) -> Option { + let name = bytes[start..] + .iter() + .take_while(|b| b.is_ascii_alphanumeric() || matches!(**b, b'_' | b'.' | b':' | b'-')) + .count(); + let end = start + name; + + let after_space = skip_spaces(bytes, end); + if bytes.get(after_space) != Some(&b'=') { + return Some(end); + } + let value = skip_spaces(bytes, after_space + 1); + match bytes.get(value)? { + quote @ (b'"' | b'\'') => { + let close = bytes[value + 1..].iter().position(|b| b == quote)?; + Some(value + 1 + close + 1) + } + _ => { + let length = bytes[value..] + .iter() + .take_while(|&&b| !is_space(b) && !matches!(b, b'"' | b'\'' | b'=' | b'<' | b'>' | b'`')) + .count(); + (length > 0).then_some(value + length) + } + } +} diff --git a/crates/mq-markdown/src/parser/inline/link.rs b/crates/mq-markdown/src/parser/inline/link.rs new file mode 100644 index 000000000..63a1f031e --- /dev/null +++ b/crates/mq-markdown/src/parser/inline/link.rs @@ -0,0 +1,223 @@ +//! Links, images, references and footnote references, resolved when a `]` is found. + +use super::entity::{remove_line_indent, unescape}; +use super::tail::{Tail, inline_tail, reference}; +use super::{Context, Item, Scanner, emphasis, to_nodes}; +use crate::node::{FootnoteRef, Image, ImageRef, Link, LinkRef, Node, Position, Title, Url}; + +/// Normalizes a reference label the way `markdown-rs` does, including its handling of whitespace: +/// the first gap between words is dropped. +pub(in crate::parser) fn normalize(label: &str) -> String { + let bytes = label.as_bytes(); + let mut result = String::with_capacity(label.len()); + let mut in_whitespace = true; + let mut start = 0; + + for (index, byte) in bytes.iter().enumerate() { + if matches!(byte, b'\t' | b'\n' | b'\r' | b' ') { + if !in_whitespace { + result.push_str(&label[start..index]); + in_whitespace = true; + } + } else if in_whitespace { + if start != 0 { + result.push(' '); + } + start = index; + in_whitespace = false; + } + } + if !in_whitespace { + result.push_str(&label[start..]); + } + + result.to_lowercase().to_uppercase().to_lowercase() +} + +/// What a closing bracket resolves to. +enum Kind { + Inline(Tail), + /// `derived` tells that the label is the text of the reference itself. + Reference { + label: String, + end: usize, + derived: bool, + }, + Footnote { + label: String, + }, +} + +/// Handles a `]` at the scanner position. Returns whether it closed a link or an image. +pub(super) fn close(scanner: &mut Scanner<'_>) -> bool { + let Some(&opener_index) = scanner.openers.last() else { + return false; + }; + let Item::Open(opener) = &scanner.items[opener_index] else { + return false; + }; + let (image, active, opener_start) = (opener.image, opener.active, opener.start); + scanner.openers.pop(); + if !active { + return false; + } + + let context = scanner.context; + let src = context.src(); + let pos = scanner.pos; + let label_start = opener_start + if image { 2 } else { 1 }; + let label = &src[label_start..pos]; + let after = pos + 1; + + let is_footnote = + !image && footnote_label(label).is_some_and(|name| context.references.footnotes.contains(&normalize(name))); + let kind = if is_footnote { + Some(Kind::Footnote { + label: label[1..].to_string(), + }) + } else if let Some(tail) = inline_tail(src, after) { + Some(Kind::Inline(tail)) + } else { + reference(context, label, after).map(|(label, end, derived)| Kind::Reference { label, end, derived }) + }; + let Some(kind) = kind else { + return false; + }; + + // Everything after the opener becomes the content. + let mut content = scanner.items.split_off(opener_index + 1); + scanner.items.pop(); + // Email addresses are not linked inside link text. + let inner = Context { + emails: false, + ..*context + }; + emphasis::process(&mut content, 0, &inner); + let values = to_nodes(content, &inner); + + let (node, end) = build(kind, image, values, after, |end| context.position(opener_start, end)); + + if !image { + // Links cannot contain links: earlier `[` openers can no longer become links. + for &index in &scanner.openers { + if let Item::Open(earlier) = &mut scanner.items[index] + && !earlier.image + { + earlier.active = false; + } + } + } + + scanner.items.push(Item::Node(node)); + scanner.pos = end; + scanner.run = end; + true +} + +/// Builds the node for a resolved bracket pair, returning it and the offset where scanning goes on. +/// `position` gives the position from the opener to the given end offset. +fn build( + kind: Kind, + image: bool, + values: Vec, + after: usize, + position: impl Fn(usize) -> Position, +) -> (Node, usize) { + match kind { + Kind::Footnote { label } => { + let node = Node::FootnoteRef(FootnoteRef { + ident: normalize(&label), + label: Some(unescape(&remove_line_indent(&label))), + position: Some(position(after)), + }); + (node, after) + } + Kind::Inline(tail) => { + let position = Some(position(tail.end)); + let node = if image { + Node::Image(Image { + alt: plain_text(&values), + url: tail.url, + title: tail.title, + position, + }) + } else { + make_link(tail.url, tail.title, values, position) + }; + (node, tail.end) + } + Kind::Reference { label, end, derived } => { + let position = Some(position(end)); + // A label taken from the text has its markup removed. + let shown = if derived { + plain_text(&values) + } else { + unescape(&remove_line_indent(&label)) + }; + let node = if image { + Node::ImageRef(ImageRef { + alt: plain_text(&values), + ident: normalize(&label), + label: Some(shown), + position, + }) + } else { + Node::LinkRef(LinkRef { + ident: normalize(&label), + label: Some(shown), + values, + position, + }) + }; + (node, end) + } + } +} + +/// The label of a footnote reference: `^` followed by text without whitespace. +fn footnote_label(label: &str) -> Option<&str> { + let name = label.strip_prefix('^')?; + (!name.is_empty() && !name.bytes().any(|b| matches!(b, b' ' | b'\t' | b'\n' | b'\r'))).then_some(name) +} + +/// Builds a link, unwrapping nested links to the same destination (from autolink literals). +pub(super) fn make_link(url: String, title: Option, values: Vec, position: Option) -> Node { + let values = values + .into_iter() + .flat_map(|child| match child { + Node::Link(Link { + url: Url(ref inner), + values, + .. + }) if *inner == url => values, + other => vec![other], + }) + .collect(); + + Node::Link(Link { + url: Url(url), + title: title.map(Title), + values, + position, + }) +} + +/// The plain text of inline nodes, as used for the alternative text of images. +fn plain_text(nodes: &[Node]) -> String { + let mut text = String::new(); + for node in nodes { + match node { + Node::Text(node) => text.push_str(&node.value), + Node::CodeInline(node) => text.push_str(&node.value), + Node::MathInline(node) => text.push_str(&node.value), + Node::Html(node) => text.push_str(&node.value), + Node::Emphasis(node) => text.push_str(&plain_text(&node.values)), + Node::Strong(node) => text.push_str(&plain_text(&node.values)), + Node::Delete(node) => text.push_str(&plain_text(&node.values)), + Node::Link(node) => text.push_str(&plain_text(&node.values)), + Node::LinkRef(node) => text.push_str(&plain_text(&node.values)), + _ => {} + } + } + text +} diff --git a/crates/mq-markdown/src/parser/inline/literal.rs b/crates/mq-markdown/src/parser/inline/literal.rs new file mode 100644 index 000000000..5818849f9 --- /dev/null +++ b/crates/mq-markdown/src/parser/inline/literal.rs @@ -0,0 +1,323 @@ +//! GFM autolink literals: `www.example.com`, `https://example.com` and `user@example.com`. +//! +//! Ported from the state machines of `markdown-rs` (MIT License, Titus Wormer) so that the same +//! text is linked, including how trailing punctuation and unbalanced parentheses are excluded. + +use super::punctuation::is_punctuation; +use super::{Item, Scanner, Value}; +use crate::node::{Link, Node, Text, Url}; + +#[derive(PartialEq)] +enum Kind { + Whitespace, + Punctuation, + Other, +} + +/// Classifies the character at byte `index`; the end of the text counts as whitespace. +fn kind_at(src: &str, index: usize) -> Kind { + let Some(char) = src.get(index..).and_then(|rest| rest.chars().next()) else { + return Kind::Whitespace; + }; + if char.is_whitespace() { + Kind::Whitespace + } else if is_punctuation(char) { + Kind::Punctuation + } else { + Kind::Other + } +} + +/// Whether the text from `index` is only trailing punctuation up to whitespace, the end or `<`. +fn trail(src: &str, mut index: usize) -> bool { + let bytes = src.as_bytes(); + loop { + match bytes.get(index) { + Some(b'!' | b'"' | b'\'' | b')' | b'*' | b',' | b'.' | b':' | b';' | b'?' | b'_' | b'~') => index += 1, + Some(b'&') => { + index += 1; + let letters = bytes[index..].iter().take_while(|b| b.is_ascii_alphabetic()).count(); + if letters == 0 || bytes.get(index + letters) != Some(&b';') { + return false; + } + index += letters + 1; + } + Some(b'<') => return true, + Some(b']') => { + index += 1; + if matches!(bytes.get(index), None | Some(b'\t' | b'\n' | b' ' | b'(' | b'[')) { + return true; + } + } + _ => return kind_at(src, index) == Kind::Whitespace, + } + } +} + +/// Matches the domain that starts at `index`, returning the offset after it. +fn domain(src: &str, mut index: usize) -> Option { + let bytes = src.as_bytes(); + let (mut seen, mut marker, mut marker_before) = (false, 0u8, 0u8); + + loop { + match bytes.get(index) { + Some(b'.' | b'_') => { + if trail(src, index) { + break; + } + if bytes[index] == b'_' { + marker = b'_'; + } else { + marker_before = marker; + marker = 0; + } + index += 1; + } + Some(b'-' | 0x80..=0xBF) => index += 1, + _ if kind_at(src, index) == Kind::Other => { + seen = true; + index += 1; + } + _ => break, + } + } + + // Underscores are not allowed in the last two segments. + (marker_before != b'_' && marker != b'_' && seen).then_some(index) +} + +/// Matches the path that starts at `index`, returning the offset after it. +fn path(src: &str, mut index: usize) -> usize { + let bytes = src.as_bytes(); + let (mut open, mut closed) = (0usize, 0usize); + + loop { + match bytes.get(index) { + None => return index, + Some(0x80..=0xBF) => index += 1, + Some(b'(') => { + open += 1; + index += 1; + } + Some( + punctuation @ (b'!' | b'"' | b'&' | b'\'' | b')' | b'*' | b',' | b'.' | b':' | b';' | b'<' | b'?' + | b']' | b'_' | b'~'), + ) => { + let unbalanced = *punctuation == b')' && closed < open; + if trail(src, index) && !unbalanced { + return index; + } + if *punctuation == b')' { + closed += 1; + } + index += 1; + } + _ if kind_at(src, index) == Kind::Whitespace => return index, + _ => index += 1, + } + } +} + +/// Matches `http://` or `https://` followed by a domain and a path. +fn protocol(src: &str, pos: usize) -> Option { + let bytes = src.as_bytes(); + if pos > 0 && bytes[pos - 1].is_ascii_alphabetic() { + return None; + } + let letters = bytes[pos..] + .iter() + .take(5) + .take_while(|b| b.is_ascii_alphabetic()) + .count(); + let name = src[pos..pos + letters].to_ascii_lowercase(); + if !matches!(name.as_str(), "http" | "https") || !src[pos + letters..].starts_with("://") { + return None; + } + let end = domain(src, pos + letters + 3)?; + Some(path(src, end)) +} + +/// Matches `www.` followed by a domain and a path. +fn www(src: &str, pos: usize) -> Option { + let bytes = src.as_bytes(); + if pos > 0 + && !matches!( + bytes[pos - 1], + b'\t' | b'\n' | b' ' | b'(' | b'*' | b'_' | b'[' | b']' | b'~' + ) + { + return None; + } + let prefix = src.get(pos..pos + 4)?; + if !prefix[..3].eq_ignore_ascii_case("www") || !prefix.ends_with('.') || pos + 4 >= bytes.len() { + return None; + } + let end = domain(src, pos)?; + Some(path(src, end)) +} + +/// Handles `h` or `w` at the scanner position: a protocol or `www.` autolink. +pub(super) fn url(scanner: &mut Scanner<'_>) { + let context = scanner.context; + let src = context.src(); + let pos = scanner.pos; + let is_www = matches!(src.as_bytes()[pos], b'w' | b'W'); + let end = if is_www { www(src, pos) } else { protocol(src, pos) }; + + let Some(end) = end.filter(|&end| end > pos) else { + scanner.pos += 1; + return; + }; + let text = &src[pos..end]; + let position = Some(context.position(pos, end)); + let node = Node::Link(Link { + url: Url(if is_www { + format!("http://{text}") + } else { + text.to_string() + }), + title: None, + values: vec![Node::Text(Text { + value: text.to_string(), + position: position.clone(), + })], + position, + }); + scanner.push_node(node, end); +} + +/// The start of the local part of an email address that ends at `at`, not before `min`. +fn local_start(bytes: &[u8], min: usize, at: usize) -> Option { + let mut index = at; + while index > min && matches!(bytes[index - 1], b'+' | b'-' | b'.' | b'0'..=b'9' | b'A'..=b'Z' | b'_' | b'a'..=b'z') + { + index -= 1; + } + (index != at && !(index > min && bytes[index - 1] == b'/')).then_some(index) +} + +/// The end of the domain of an email address that starts at `start`. +fn email_domain(bytes: &[u8], start: usize, xmpp: bool) -> Option { + let mut index = start; + let mut dot = false; + + while index < bytes.len() { + match bytes[index] { + b'-' | b'0'..=b'9' | b'A'..=b'Z' | b'_' | b'a'..=b'z' => {} + b'/' if xmpp => {} + b'.' if bytes.get(index + 1).is_some_and(u8::is_ascii_alphanumeric) => dot = true, + _ => break, + } + index += 1; + } + + (index > start && dot && matches!(bytes[index - 1], b'.' | b'A'..=b'Z' | b'a'..=b'z')).then_some(index) +} + +/// Appends the plain text `start..end` to `out`, turning email addresses in it into links. +fn split_emails(src: &str, start: usize, end: usize, out: &mut Vec) { + let bytes = src.as_bytes(); + let mut emitted = start; + let mut min = start; + let mut index = start; + + while index < end { + if bytes[index] == b'@' + && let Some(mut from) = local_start(bytes, min, index) + { + // `mailto:` and `xmpp:` are part of the link and are not prefixed again. + let (mut prefixed, mut xmpp) = (true, false); + if from > min && bytes[from - 1] == b':' { + let mut word = from - 1; + while word > min && bytes[word - 1].is_ascii_alphanumeric() { + word -= 1; + } + let name = src[word..from - 1].to_ascii_lowercase(); + if name == "xmpp" || name == "mailto" { + (from, prefixed, xmpp) = (word, false, name == "xmpp"); + } + } + if let Some(to) = email_domain(&bytes[..end], index + 1, xmpp) { + if emitted < from { + out.push(Item::Text { + start: emitted, + end: from, + value: Value::Slice(emitted, from), + }); + } + out.push(Item::Email { + start: from, + end: to, + prefixed, + }); + emitted = to; + min = to; + index = to; + continue; + } + } + index += 1; + } + + if emitted < end { + out.push(Item::Text { + start: emitted, + end, + value: Value::Slice(emitted, end), + }); + } +} + +/// The source range of an item that `markdown-rs` treats as plain data: text that is not decoded, +/// and delimiters and brackets that were not used. +fn data_range(item: &Item) -> Option<(usize, usize)> { + match item { + Item::Text { + start, + end, + value: Value::Slice(from, to), + } if from == start && to == end => Some((*start, *end)), + Item::Delim(delim) => Some((delim.start, delim.start + delim.count)), + Item::Open(opener) => Some((opener.start, opener.start + if opener.image { 2 } else { 1 })), + _ => None, + } +} + +/// Links the email addresses found in runs of plain data. A line ending ends a run. +pub(super) fn link_emails(src: &str, items: Vec) -> Vec { + let mut out = Vec::with_capacity(items.len()); + let mut group: Vec = Vec::new(); + let mut range = (0, 0); + + let finish = |group: &mut Vec, range: (usize, usize), out: &mut Vec| { + if group.is_empty() { + return; + } + if src[range.0..range.1].contains('@') { + group.clear(); + split_emails(src, range.0, range.1, out); + } else { + out.append(group); + } + }; + + for item in items { + let data = data_range(&item).filter(|&(start, _)| !matches!(src.as_bytes()[start], b'\n' | b'\r')); + match data { + Some((start, end)) => { + if group.is_empty() { + range.0 = start; + } + range.1 = end; + group.push(item); + } + None => { + finish(&mut group, range, &mut out); + out.push(item); + } + } + } + finish(&mut group, range, &mut out); + + out +} diff --git a/crates/mq-markdown/src/parser/inline/punctuation.rs b/crates/mq-markdown/src/parser/inline/punctuation.rs new file mode 100644 index 000000000..e313f1515 --- /dev/null +++ b/crates/mq-markdown/src/parser/inline/punctuation.rs @@ -0,0 +1,375 @@ +//! Unicode punctuation for emphasis flanking rules. +//! +//! Generated from the table of `markdown-rs` (MIT License, Titus Wormer): the general categories P +//! and S, as `CommonMark` defines punctuation. Sorted ranges. + +const RANGES: [(u32, u32); 349] = [ + (0x0021, 0x002F), + (0x003A, 0x0040), + (0x005B, 0x0060), + (0x007B, 0x007E), + (0x00A1, 0x00A9), + (0x00AB, 0x00AC), + (0x00AE, 0x00B1), + (0x00B4, 0x00B4), + (0x00B6, 0x00B8), + (0x00BB, 0x00BB), + (0x00BF, 0x00BF), + (0x00D7, 0x00D7), + (0x00F7, 0x00F7), + (0x02C2, 0x02C5), + (0x02D2, 0x02DF), + (0x02E5, 0x02EB), + (0x02ED, 0x02ED), + (0x02EF, 0x02FF), + (0x0375, 0x0375), + (0x037E, 0x037E), + (0x0384, 0x0385), + (0x0387, 0x0387), + (0x03F6, 0x03F6), + (0x0482, 0x0482), + (0x055A, 0x055F), + (0x0589, 0x058A), + (0x058D, 0x058F), + (0x05BE, 0x05BE), + (0x05C0, 0x05C0), + (0x05C3, 0x05C3), + (0x05C6, 0x05C6), + (0x05F3, 0x05F4), + (0x0606, 0x060F), + (0x061B, 0x061B), + (0x061D, 0x061F), + (0x066A, 0x066D), + (0x06D4, 0x06D4), + (0x06DE, 0x06DE), + (0x06E9, 0x06E9), + (0x06FD, 0x06FE), + (0x0700, 0x070D), + (0x07F6, 0x07F9), + (0x07FE, 0x07FF), + (0x0830, 0x083E), + (0x085E, 0x085E), + (0x0888, 0x0888), + (0x0964, 0x0965), + (0x0970, 0x0970), + (0x09F2, 0x09F3), + (0x09FA, 0x09FB), + (0x09FD, 0x09FD), + (0x0A76, 0x0A76), + (0x0AF0, 0x0AF1), + (0x0B70, 0x0B70), + (0x0BF3, 0x0BFA), + (0x0C77, 0x0C77), + (0x0C7F, 0x0C7F), + (0x0C84, 0x0C84), + (0x0D4F, 0x0D4F), + (0x0D79, 0x0D79), + (0x0DF4, 0x0DF4), + (0x0E3F, 0x0E3F), + (0x0E4F, 0x0E4F), + (0x0E5A, 0x0E5B), + (0x0F01, 0x0F17), + (0x0F1A, 0x0F1F), + (0x0F34, 0x0F34), + (0x0F36, 0x0F36), + (0x0F38, 0x0F38), + (0x0F3A, 0x0F3D), + (0x0F85, 0x0F85), + (0x0FBE, 0x0FC5), + (0x0FC7, 0x0FCC), + (0x0FCE, 0x0FDA), + (0x104A, 0x104F), + (0x109E, 0x109F), + (0x10FB, 0x10FB), + (0x1360, 0x1368), + (0x1390, 0x1399), + (0x1400, 0x1400), + (0x166D, 0x166E), + (0x169B, 0x169C), + (0x16EB, 0x16ED), + (0x1735, 0x1736), + (0x17D4, 0x17D6), + (0x17D8, 0x17DB), + (0x1800, 0x180A), + (0x1940, 0x1940), + (0x1944, 0x1945), + (0x19DE, 0x19FF), + (0x1A1E, 0x1A1F), + (0x1AA0, 0x1AA6), + (0x1AA8, 0x1AAD), + (0x1B4E, 0x1B4F), + (0x1B5A, 0x1B6A), + (0x1B74, 0x1B7F), + (0x1BFC, 0x1BFF), + (0x1C3B, 0x1C3F), + (0x1C7E, 0x1C7F), + (0x1CC0, 0x1CC7), + (0x1CD3, 0x1CD3), + (0x1FBD, 0x1FBD), + (0x1FBF, 0x1FC1), + (0x1FCD, 0x1FCF), + (0x1FDD, 0x1FDF), + (0x1FED, 0x1FEF), + (0x1FFD, 0x1FFE), + (0x2010, 0x2027), + (0x2030, 0x205E), + (0x207A, 0x207E), + (0x208A, 0x208E), + (0x20A0, 0x20C0), + (0x2100, 0x2101), + (0x2103, 0x2106), + (0x2108, 0x2109), + (0x2114, 0x2114), + (0x2116, 0x2118), + (0x211E, 0x2123), + (0x2125, 0x2125), + (0x2127, 0x2127), + (0x2129, 0x2129), + (0x212E, 0x212E), + (0x213A, 0x213B), + (0x2140, 0x2144), + (0x214A, 0x214D), + (0x214F, 0x214F), + (0x218A, 0x218B), + (0x2190, 0x2429), + (0x2440, 0x244A), + (0x249C, 0x24E9), + (0x2500, 0x2775), + (0x2794, 0x2B73), + (0x2B76, 0x2B95), + (0x2B97, 0x2BFF), + (0x2CE5, 0x2CEA), + (0x2CF9, 0x2CFC), + (0x2CFE, 0x2CFF), + (0x2D70, 0x2D70), + (0x2E00, 0x2E2E), + (0x2E30, 0x2E5D), + (0x2E80, 0x2E99), + (0x2E9B, 0x2EF3), + (0x2F00, 0x2FD5), + (0x2FF0, 0x2FFF), + (0x3001, 0x3004), + (0x3008, 0x3020), + (0x3030, 0x3030), + (0x3036, 0x3037), + (0x303D, 0x303F), + (0x309B, 0x309C), + (0x30A0, 0x30A0), + (0x30FB, 0x30FB), + (0x3190, 0x3191), + (0x3196, 0x319F), + (0x31C0, 0x31E5), + (0x31EF, 0x31EF), + (0x3200, 0x321E), + (0x322A, 0x3247), + (0x3250, 0x3250), + (0x3260, 0x327F), + (0x328A, 0x32B0), + (0x32C0, 0x33FF), + (0x4DC0, 0x4DFF), + (0xA490, 0xA4C6), + (0xA4FE, 0xA4FF), + (0xA60D, 0xA60F), + (0xA673, 0xA673), + (0xA67E, 0xA67E), + (0xA6F2, 0xA6F7), + (0xA700, 0xA716), + (0xA720, 0xA721), + (0xA789, 0xA78A), + (0xA828, 0xA82B), + (0xA836, 0xA839), + (0xA874, 0xA877), + (0xA8CE, 0xA8CF), + (0xA8F8, 0xA8FA), + (0xA8FC, 0xA8FC), + (0xA92E, 0xA92F), + (0xA95F, 0xA95F), + (0xA9C1, 0xA9CD), + (0xA9DE, 0xA9DF), + (0xAA5C, 0xAA5F), + (0xAA77, 0xAA79), + (0xAADE, 0xAADF), + (0xAAF0, 0xAAF1), + (0xAB5B, 0xAB5B), + (0xAB6A, 0xAB6B), + (0xABEB, 0xABEB), + (0xFB29, 0xFB29), + (0xFBB2, 0xFBC2), + (0xFD3E, 0xFD4F), + (0xFDCF, 0xFDCF), + (0xFDFC, 0xFDFF), + (0xFE10, 0xFE19), + (0xFE30, 0xFE52), + (0xFE54, 0xFE66), + (0xFE68, 0xFE6B), + (0xFF01, 0xFF0F), + (0xFF1A, 0xFF20), + (0xFF3B, 0xFF40), + (0xFF5B, 0xFF65), + (0xFFE0, 0xFFE6), + (0xFFE8, 0xFFEE), + (0xFFFC, 0xFFFD), + (0x10100, 0x10102), + (0x10137, 0x1013F), + (0x10179, 0x10189), + (0x1018C, 0x1018E), + (0x10190, 0x1019C), + (0x101A0, 0x101A0), + (0x101D0, 0x101FC), + (0x1039F, 0x1039F), + (0x103D0, 0x103D0), + (0x1056F, 0x1056F), + (0x10857, 0x10857), + (0x10877, 0x10878), + (0x1091F, 0x1091F), + (0x1093F, 0x1093F), + (0x10A50, 0x10A58), + (0x10A7F, 0x10A7F), + (0x10AC8, 0x10AC8), + (0x10AF0, 0x10AF6), + (0x10B39, 0x10B3F), + (0x10B99, 0x10B9C), + (0x10D6E, 0x10D6E), + (0x10D8E, 0x10D8F), + (0x10EAD, 0x10EAD), + (0x10F55, 0x10F59), + (0x10F86, 0x10F89), + (0x11047, 0x1104D), + (0x110BB, 0x110BC), + (0x110BE, 0x110C1), + (0x11140, 0x11143), + (0x11174, 0x11175), + (0x111C5, 0x111C8), + (0x111CD, 0x111CD), + (0x111DB, 0x111DB), + (0x111DD, 0x111DF), + (0x11238, 0x1123D), + (0x112A9, 0x112A9), + (0x113D4, 0x113D5), + (0x113D7, 0x113D8), + (0x1144B, 0x1144F), + (0x1145A, 0x1145B), + (0x1145D, 0x1145D), + (0x114C6, 0x114C6), + (0x115C1, 0x115D7), + (0x11641, 0x11643), + (0x11660, 0x1166C), + (0x116B9, 0x116B9), + (0x1173C, 0x1173F), + (0x1183B, 0x1183B), + (0x11944, 0x11946), + (0x119E2, 0x119E2), + (0x11A3F, 0x11A46), + (0x11A9A, 0x11A9C), + (0x11A9E, 0x11AA2), + (0x11B00, 0x11B09), + (0x11BE1, 0x11BE1), + (0x11C41, 0x11C45), + (0x11C70, 0x11C71), + (0x11EF7, 0x11EF8), + (0x11F43, 0x11F4F), + (0x11FD5, 0x11FF1), + (0x11FFF, 0x11FFF), + (0x12470, 0x12474), + (0x12FF1, 0x12FF2), + (0x16A6E, 0x16A6F), + (0x16AF5, 0x16AF5), + (0x16B37, 0x16B3F), + (0x16B44, 0x16B45), + (0x16D6D, 0x16D6F), + (0x16E97, 0x16E9A), + (0x16FE2, 0x16FE2), + (0x1BC9C, 0x1BC9C), + (0x1BC9F, 0x1BC9F), + (0x1CC00, 0x1CCEF), + (0x1CD00, 0x1CEB3), + (0x1CF50, 0x1CFC3), + (0x1D000, 0x1D0F5), + (0x1D100, 0x1D126), + (0x1D129, 0x1D164), + (0x1D16A, 0x1D16C), + (0x1D183, 0x1D184), + (0x1D18C, 0x1D1A9), + (0x1D1AE, 0x1D1EA), + (0x1D200, 0x1D241), + (0x1D245, 0x1D245), + (0x1D300, 0x1D356), + (0x1D6C1, 0x1D6C1), + (0x1D6DB, 0x1D6DB), + (0x1D6FB, 0x1D6FB), + (0x1D715, 0x1D715), + (0x1D735, 0x1D735), + (0x1D74F, 0x1D74F), + (0x1D76F, 0x1D76F), + (0x1D789, 0x1D789), + (0x1D7A9, 0x1D7A9), + (0x1D7C3, 0x1D7C3), + (0x1D800, 0x1D9FF), + (0x1DA37, 0x1DA3A), + (0x1DA6D, 0x1DA74), + (0x1DA76, 0x1DA83), + (0x1DA85, 0x1DA8B), + (0x1E14F, 0x1E14F), + (0x1E2FF, 0x1E2FF), + (0x1E5FF, 0x1E5FF), + (0x1E95E, 0x1E95F), + (0x1ECAC, 0x1ECAC), + (0x1ECB0, 0x1ECB0), + (0x1ED2E, 0x1ED2E), + (0x1EEF0, 0x1EEF1), + (0x1F000, 0x1F02B), + (0x1F030, 0x1F093), + (0x1F0A0, 0x1F0AE), + (0x1F0B1, 0x1F0BF), + (0x1F0C1, 0x1F0CF), + (0x1F0D1, 0x1F0F5), + (0x1F10D, 0x1F1AD), + (0x1F1E6, 0x1F202), + (0x1F210, 0x1F23B), + (0x1F240, 0x1F248), + (0x1F250, 0x1F251), + (0x1F260, 0x1F265), + (0x1F300, 0x1F6D7), + (0x1F6DC, 0x1F6EC), + (0x1F6F0, 0x1F6FC), + (0x1F700, 0x1F776), + (0x1F77B, 0x1F7D9), + (0x1F7E0, 0x1F7EB), + (0x1F7F0, 0x1F7F0), + (0x1F800, 0x1F80B), + (0x1F810, 0x1F847), + (0x1F850, 0x1F859), + (0x1F860, 0x1F887), + (0x1F890, 0x1F8AD), + (0x1F8B0, 0x1F8BB), + (0x1F8C0, 0x1F8C1), + (0x1F900, 0x1FA53), + (0x1FA60, 0x1FA6D), + (0x1FA70, 0x1FA7C), + (0x1FA80, 0x1FA89), + (0x1FA8F, 0x1FAC6), + (0x1FACE, 0x1FADC), + (0x1FADF, 0x1FAE9), + (0x1FAF0, 0x1FAF8), + (0x1FB00, 0x1FB92), + (0x1FB94, 0x1FBEF), +]; + +/// Whether `char` is a Unicode punctuation character. +pub(super) fn is_punctuation(char: char) -> bool { + if char.is_ascii() { + return char.is_ascii_punctuation(); + } + let code = char as u32; + RANGES + .binary_search_by(|&(start, end)| { + if code < start { + std::cmp::Ordering::Greater + } else if code > end { + std::cmp::Ordering::Less + } else { + std::cmp::Ordering::Equal + } + }) + .is_ok() +} diff --git a/crates/mq-markdown/src/parser/inline/tail.rs b/crates/mq-markdown/src/parser/inline/tail.rs new file mode 100644 index 000000000..241ecd6ee --- /dev/null +++ b/crates/mq-markdown/src/parser/inline/tail.rs @@ -0,0 +1,190 @@ +//! The parts that follow the text of a link: destinations, titles and reference labels. + +use super::Context; +use super::entity; +use super::link::normalize; + +/// The destination and title of an inline link, and the offset after its closing parenthesis. +pub(super) struct Tail { + pub(super) url: String, + pub(super) title: Option, + pub(super) end: usize, +} + +/// Resolves a reference whose text is `text` and whose closing bracket is followed by `after`. +/// Returns the label, the offset after the reference, and whether the label is the text itself. +pub(super) fn reference(context: &Context<'_>, text: &str, after: usize) -> Option<(String, usize, bool)> { + let src = context.src(); + let defined = |label: &str| context.references.definitions.contains(&normalize(label)); + + if src[after..].starts_with('[') { + match parse_label(src, after) { + // A full reference: `[text][label]`. + Some((end, label)) if !label.is_empty() => { + return defined(label).then(|| (label.to_string(), end, false)); + } + // A collapsed reference: `[text][]`. + Some((end, _)) => { + return (has_content(text) && defined(text)).then(|| (text.to_string(), end, true)); + } + None => {} + } + } + + // A shortcut reference: `[text]`. + (has_content(text) && defined(text)).then(|| (text.to_string(), after, true)) +} + +fn has_content(label: &str) -> bool { + label.bytes().any(|b| !matches!(b, b' ' | b'\t' | b'\n' | b'\r')) +} + +/// Parses a link label at `pos` (the `[`), returning the offset after the `]` and the raw label. +/// An empty label is returned for `[]`. +fn parse_label(src: &str, pos: usize) -> Option<(usize, &str)> { + let bytes = src.as_bytes(); + let mut index = pos + 1; + while index < bytes.len() && index - pos <= 1000 { + match bytes[index] { + b'\\' => index += 1, + b'[' => return None, + b']' => { + let label = &src[pos + 1..index]; + return (label.is_empty() || has_content(label)).then_some((index + 1, label)); + } + _ => {} + } + index += 1; + } + None +} + +/// Skips whitespace including at most one line ending. +fn skip_space(bytes: &[u8], mut index: usize) -> usize { + let mut line_endings = 0; + while let Some(&byte) = bytes.get(index) { + match byte { + b' ' | b'\t' => index += 1, + b'\n' | b'\r' => { + line_endings += 1; + if line_endings > 1 { + break; + } + index += super::eol_len(&bytes[index..]); + } + _ => break, + } + } + index +} + +/// Parses `(destination "title")` at `pos`. +pub(super) fn inline_tail(src: &str, pos: usize) -> Option { + let bytes = src.as_bytes(); + if bytes.get(pos) != Some(&b'(') { + return None; + } + + let index = skip_space(bytes, pos + 1); + let (url, after_url) = destination(src, index)?; + + let after_space = skip_space(bytes, after_url); + if bytes.get(after_space) == Some(&b')') { + return Some(Tail { + url, + title: None, + end: after_space + 1, + }); + } + // A title needs whitespace before it. + if after_space == after_url { + return None; + } + let (title, after_title) = title_at(src, after_space)?; + + let closing = skip_space(bytes, after_title); + (bytes.get(closing) == Some(&b')')).then_some(Tail { + url, + title: Some(title), + end: closing + 1, + }) +} + +/// Parses a link destination at `pos`, returning it decoded and the offset after it. +pub(in crate::parser) fn destination(src: &str, pos: usize) -> Option<(String, usize)> { + let bytes = src.as_bytes(); + + if bytes.get(pos) == Some(&b'<') { + let mut index = pos + 1; + while let Some(&byte) = bytes.get(index) { + match byte { + b'>' => return Some((entity::unescape(&src[pos + 1..index]), index + 1)), + b'<' | b'\n' | b'\r' => return None, + b'\\' => index += 1, + _ => {} + } + index += 1; + } + return None; + } + + let mut index = pos; + let mut depth = 0usize; + while let Some(&byte) = bytes.get(index) { + match byte { + b'\\' if bytes.get(index + 1).is_some_and(u8::is_ascii_punctuation) => index += 1, + b'(' => { + depth += 1; + if depth > 32 { + return None; + } + } + b')' => { + if depth == 0 { + break; + } + depth -= 1; + } + b' ' | b'\t' | b'\n' | b'\r' => break, + byte if byte.is_ascii_control() => break, + _ => {} + } + index += 1; + } + (depth == 0).then(|| (entity::unescape(&src[pos..index]), index)) +} + +/// Parses a link title at `pos`, returning it decoded and the offset after its closing delimiter. +pub(in crate::parser) fn title_at(src: &str, pos: usize) -> Option<(String, usize)> { + let bytes = src.as_bytes(); + let close = match bytes.get(pos)? { + b'"' => b'"', + b'\'' => b'\'', + b'(' => b')', + _ => return None, + }; + + let mut index = pos + 1; + while let Some(&byte) = bytes.get(index) { + match byte { + byte if byte == close => return Some((entity::unescape(&src[pos + 1..index]), index + 1)), + b'(' if close == b')' => return None, + b'\\' => index += 1, + b'\n' | b'\r' => { + // A title cannot contain a blank line. + let length = super::eol_len(&bytes[index..]); + let mut next = index + length; + while matches!(bytes.get(next), Some(b' ' | b'\t')) { + next += 1; + } + if matches!(bytes.get(next), Some(b'\n' | b'\r')) { + return None; + } + index += length - 1; + } + _ => {} + } + index += 1; + } + None +} diff --git a/crates/mq-markdown/src/parser/line.rs b/crates/mq-markdown/src/parser/line.rs new file mode 100644 index 000000000..e201fe3ca --- /dev/null +++ b/crates/mq-markdown/src/parser/line.rs @@ -0,0 +1,208 @@ +//! Source lines and their positions. + +use crate::node::Point; + +#[derive(Clone, Copy)] +pub(super) struct Line<'a> { + pub(super) number: usize, + /// Byte offset of `text` within the original line, non-zero inside containers. + pub(super) column: usize, + pub(super) text: &'a str, + /// Line terminator that follows `text`; empty on the last line without one. + pub(super) eol: &'a str, + /// Whether this is the last line of the document. + pub(super) eof: bool, + /// Whether this is the last line of a list item. + pub(super) item_end: bool, + /// Whether the line continues a paragraph without its container prefix. + pub(super) lazy: bool, + /// Whether indented code that starts on this line is a chunk of its own, as for the first line of + /// an item whose marker is followed by five or more spaces. + pub(super) own_chunk: bool, + /// The whole line as in the document, to compute columns when it contains tabs. + pub(super) origin: &'a str, + /// Columns left over from a tab that a container consumed only in part. They count as leading + /// whitespace of `text`. + pub(super) pad: usize, + /// Whether the document is MDX, which has no indented code, HTML, autolinks or GFM. + pub(super) mdx: bool, +} + +impl<'a> Line<'a> { + /// The indentation from which a line is code. Without indented code, indentation never matters. + pub(super) fn code_indent(&self) -> usize { + if self.mdx { usize::MAX } else { 4 } + } + + /// The zero-based visual column where `text` starts in the document. + fn start_column(&self) -> usize { + if self.origin.contains('\t') { + visual_column(self.origin[..self.column].chars(), 0) + } else { + self.column + } + } + + /// Column count and byte length of the leading whitespace. + pub(super) fn indent(&self) -> (usize, usize) { + let mut columns = self.pad; + let mut absolute = self.start_column(); + for (index, byte) in self.text.bytes().enumerate() { + match byte { + b' ' => { + columns += 1; + absolute += 1; + } + b'\t' => { + let width = (absolute / 4 + 1) * 4 - absolute; + columns += width; + absolute += width; + } + _ => return (columns, index), + } + } + (columns, self.text.len()) + } + + pub(super) fn is_blank(&self) -> bool { + self.text.trim_matches([' ', '\t']).is_empty() + } + + /// The position of the byte at `byte` in `text`. Columns are in bytes, except that a tab advances + /// to the next multiple of four, like `markdown-rs` counts them. + pub(super) fn point(&self, byte: usize) -> Point { + let offset = self.column + byte; + let mut column = if self.origin.contains('\t') { + visual_column(self.origin[..offset].chars(), 0) + 1 + } else { + offset + 1 + }; + // The start of a line inside a tab is before the end of that tab. + if byte == 0 { + column -= self.pad; + } + Point { + line: self.number, + column, + } + } + + /// The same line with the first `bytes` bytes removed. + pub(super) fn skip(self, bytes: usize) -> Line<'a> { + Line { + column: self.column + bytes, + text: &self.text[bytes..], + pad: 0, + ..self + } + } + + /// The same line with `columns` columns of leading whitespace removed. What is left of a tab that + /// is only consumed in part stays as padding. + pub(super) fn skip_columns(self, columns: usize) -> Line<'a> { + let mut remaining = columns; + let consumed = remaining.min(self.pad); + let mut pad = self.pad - consumed; + remaining -= consumed; + + let mut absolute = self.start_column(); + let mut bytes = 0; + for byte in self.text.bytes() { + if remaining == 0 { + break; + } + match byte { + b' ' => { + remaining -= 1; + absolute += 1; + } + b'\t' => { + let width = (absolute / 4 + 1) * 4 - absolute; + absolute += width; + if width > remaining { + pad = width - remaining; + remaining = 0; + } else { + remaining -= width; + } + } + _ => break, + } + bytes += 1; + } + + Line { + column: self.column + bytes, + text: &self.text[bytes..], + pad, + ..self + } + } + + pub(super) fn end(&self) -> Point { + self.point(self.text.len()) + } +} + +/// The zero-based column reached after `chars`, starting at column `start`, where tabs go to the next +/// multiple of four and other characters advance by their length in bytes. +pub(super) fn visual_column(chars: impl Iterator, start: usize) -> usize { + chars.fold(start, |column, char| { + if char == '\t' { + (column / 4 + 1) * 4 + } else { + column + char.len_utf8() + } + }) +} + +/// Splits `src` into lines on `\n`, `\r\n` and `\r`. A trailing terminator does not add a line. +pub(super) fn split_lines(src: &str, mdx: bool) -> Vec> { + let bytes = src.as_bytes(); + let mut lines = Vec::new(); + let (mut start, mut index) = (0, 0); + + while index < bytes.len() { + let end = index; + match bytes[index] { + b'\n' => index += 1, + b'\r' => index += if bytes.get(index + 1) == Some(&b'\n') { 2 } else { 1 }, + _ => { + index += 1; + continue; + } + } + lines.push(Line { + number: lines.len() + 1, + column: 0, + text: &src[start..end], + eol: &src[end..index], + eof: index == bytes.len(), + item_end: false, + lazy: false, + own_chunk: false, + origin: &src[start..end], + pad: 0, + mdx, + }); + start = index; + } + + if start < src.len() { + lines.push(Line { + number: lines.len() + 1, + column: 0, + text: &src[start..], + eol: "", + eof: true, + item_end: false, + lazy: false, + own_chunk: false, + origin: &src[start..], + pad: 0, + mdx, + }); + } + + lines +} diff --git a/crates/mq-markdown/src/parser/mdx.rs b/crates/mq-markdown/src/parser/mdx.rs new file mode 100644 index 000000000..36a3ebd57 --- /dev/null +++ b/crates/mq-markdown/src/parser/mdx.rs @@ -0,0 +1,371 @@ +//! MDX: JSX tags and expressions, ported from the rules of `markdown-rs` without a JavaScript parser, +//! so expressions are only checked for balanced braces. + +use crate::node::{MdxAttributeContent, MdxAttributeValue, MdxJsxAttribute}; +use smol_str::SmolStr; + +/// What the end of the input means for a construct it ends inside of. +pub(super) enum Fallback { + Nok, + Error(String), +} + +/// The outcome of parsing at a position. +pub(super) enum Parsed { + Ok(T), + /// Not this construct, so the text is something else. + Nok, + /// The input ended inside the construct. More input may complete it, and if there is none this + /// is what the end of the input means for it. + More(Fallback), + /// The construct is invalid. + Error(String), +} + +#[derive(Clone, Copy, PartialEq)] +pub(super) enum TagKind { + Open, + Close, + SelfClosing, +} + +/// A JSX tag: ``, `` or ``. A fragment has no name. +pub(super) struct Tag { + pub(super) name: Option, + pub(super) attributes: Vec, + pub(super) kind: TagKind, + /// Offset after the closing `>`. + pub(super) end: usize, +} + +fn id_start(char: char) -> bool { + char.is_alphabetic() || matches!(char, '$' | '_') +} + +fn id_continue(char: char) -> bool { + char.is_alphanumeric() + || matches!(char, '_' | '-' | '\u{200c}' | '\u{200d}') + || ('\u{0300}'..='\u{036f}').contains(&char) +} + +/// What a tag parser stops with: a failure of the construct, or of the input. +enum Stop { + Nok, + More(Fallback), + Error(String), +} + +type Step = Result; + +struct Cursor<'a> { + src: &'a str, + index: usize, +} + +impl Cursor<'_> { + fn peek(&self) -> Option { + self.src[self.index..].chars().next() + } + + fn bump(&mut self) { + if let Some(char) = self.peek() { + self.index += char.len_utf8(); + } + } + + /// The input ends inside a construct: more input may follow, otherwise it is not one. + fn end_of_input(&self) -> Step { + Err(Stop::More(Fallback::Nok)) + } + + fn crash(&self, message: String) -> Step { + Err(Stop::Error(message)) + } + + /// Skips whitespace, including line endings. Ends the tag at the end of the input. + fn skip_whitespace(&mut self) -> Step<()> { + while self.peek().is_some_and(char::is_whitespace) { + self.bump(); + } + if self.peek().is_none() { + return self.end_of_input(); + } + Ok(()) + } + + fn name_part(&mut self) -> Step<&str> { + let start = self.index; + self.bump(); + while self.peek().is_some_and(id_continue) { + self.bump(); + } + Ok(&self.src[start..self.index]) + } + + fn describe(&self) -> String { + match self.peek() { + Some(char) => format!("character `{char}` (U+{:04X})", char as u32), + None => "end of file".to_string(), + } + } + + /// The name of a tag, with its member (`.`) or namespace (`:`) parts, cleaned of whitespace. + fn tag_name(&mut self) -> Step { + let mut name = self.name_part()?.to_string(); + self.skip_whitespace()?; + match self.peek() { + Some('.') => { + while self.peek() == Some('.') { + self.bump(); + self.skip_whitespace()?; + if !self.peek().is_some_and(id_start) { + return self.crash(format!("Unexpected {} before member name", self.describe())); + } + name.push('.'); + name.push_str(self.name_part()?); + self.skip_whitespace()?; + } + } + Some(':') => { + self.bump(); + self.skip_whitespace()?; + if !self.peek().is_some_and(id_start) { + return self.crash(format!("Unexpected {} before local name", self.describe())); + } + name.push(':'); + name.push_str(self.name_part()?); + self.skip_whitespace()?; + } + _ => {} + } + Ok(name) + } + + /// A `{...}` expression with balanced braces, returning the offsets of its content. + fn expression(&mut self) -> Step<(usize, usize)> { + self.bump(); + let start = self.index; + let mut depth = 0usize; + loop { + match self.peek() { + None => { + return Err(Stop::More(Fallback::Error( + "Unexpected end of file in expression, expected a corresponding closing brace for `{`".into(), + ))); + } + Some('{') => depth += 1, + Some('}') if depth == 0 => { + let end = self.index; + self.bump(); + return Ok((start, end)); + } + Some('}') => depth -= 1, + Some(_) => {} + } + self.bump(); + } + } +} + +impl Cursor<'_> { + /// The value of an expression: its content, without up to two whitespace characters at the start + /// of each line after the first. + fn expression_value(&self, start: usize, end: usize) -> SmolStr { + let content = &self.src[start..end]; + if !content.contains('\n') { + return SmolStr::new(content); + } + let mut value = String::with_capacity(content.len()); + for (index, line) in content.split('\n').enumerate() { + if index > 0 { + value.push('\n'); + } + let skipped = if index == 0 { + 0 + } else { + line.chars().take(2).take_while(|c| matches!(c, ' ' | '\t')).count() + }; + value.push_str(&line[skipped..]); + } + SmolStr::new(value) + } + + /// Checks the character after a name: whitespace or one of `allowed`. + fn end_of_name(&self, allowed: &[char], what: &str) -> Step<()> { + match self.peek() { + None => self.end_of_input(), + Some(char) if char.is_whitespace() || allowed.contains(&char) => Ok(()), + Some(_) => self.crash(format!("Unexpected {} in {what}", self.describe())), + } + } + + fn attribute(&mut self) -> Step { + let mut name = self.name_part()?.to_string(); + self.end_of_name(&['/', ':', '=', '>', '{'], "attribute name")?; + self.skip_whitespace()?; + + if self.peek() == Some(':') { + self.bump(); + self.skip_whitespace()?; + if !self.peek().is_some_and(id_start) { + return self.crash(format!("Unexpected {} before local attribute name", self.describe())); + } + name.push(':'); + name.push_str(self.name_part()?); + self.end_of_name(&['/', '=', '>', '{'], "local attribute name")?; + self.skip_whitespace()?; + } + + let name = SmolStr::new(name); + if self.peek() != Some('=') { + if !matches!(self.peek(), Some('/' | '>' | '{')) && !self.peek().is_some_and(id_start) { + return self.crash(format!("Unexpected {} after attribute name", self.describe())); + } + return Ok(MdxAttributeContent::Property(MdxJsxAttribute { name, value: None })); + } + + self.bump(); + // Unlike before the `=`, no whitespace is allowed after it. + if self.peek().is_some_and(char::is_whitespace) { + return Err(Stop::Nok); + } + let value = match self.peek() { + Some(quote @ ('"' | '\'')) => { + self.bump(); + let start = self.index; + let Some(length) = self.src[start..].find(quote) else { + return Err(Stop::More(Fallback::Error(format!( + "Unexpected end of file in attribute value, expected a corresponding closing quote `{quote}`" + )))); + }; + self.index = start + length + 1; + MdxAttributeValue::Literal(SmolStr::new(super::inline::decode_references( + &self.src[start..start + length], + ))) + } + Some('{') => { + let (start, end) = self.expression()?; + MdxAttributeValue::Expression(self.expression_value(start, end)) + } + _ => { + return self.crash(format!("Unexpected {} before attribute value", self.describe())); + } + }; + Ok(MdxAttributeContent::Property(MdxJsxAttribute { + name, + value: Some(value), + })) + } + + fn tag(&mut self) -> Step { + // `<` is followed by a name, not by whitespace. + match self.peek() { + Some(' ' | '\t' | '\n') => return Err(Stop::Nok), + None => return self.end_of_input(), + Some(_) => {} + } + self.skip_whitespace()?; + + let mut kind = TagKind::Open; + if self.peek() == Some('/') { + self.bump(); + self.skip_whitespace()?; + kind = TagKind::Close; + } + + let name = match self.peek() { + Some('>') => None, + Some(char) if id_start(char) => Some(self.tag_name()?), + _ => return self.crash(format!("Unexpected {} before name", self.describe())), + }; + if name.is_some() { + self.end_of_tag_name()?; + } + + let mut attributes = Vec::new(); + // Errors about a closing tag are reported once the tag is known to end. + let mut misplaced = None; + loop { + match self.peek() { + Some('/') => { + self.bump(); + self.skip_whitespace()?; + if self.peek() != Some('>') { + return self.crash(format!("Unexpected {} after self-closing slash", self.describe())); + } + if kind == TagKind::Close { + return self.crash( + "Unexpected self-closing slash `/` in closing tag, expected the end of the tag".into(), + ); + } + kind = TagKind::SelfClosing; + } + Some('>') => { + self.bump(); + break; + } + Some(char) if char == '{' || id_start(char) => { + if kind == TagKind::Close { + misplaced = Some("Unexpected attribute in closing tag, expected the end of the tag"); + } + let attribute = if char == '{' { + let (start, end) = self.expression()?; + MdxAttributeContent::Expression(self.expression_value(start, end)) + } else { + self.attribute()? + }; + attributes.push(attribute); + self.skip_whitespace()?; + } + None => return self.end_of_input(), + Some(_) => return self.crash(format!("Unexpected {} before attribute name", self.describe())), + } + } + + if let Some(message) = misplaced { + return self.crash(message.into()); + } + + Ok(Tag { + name, + attributes, + kind, + end: self.index, + }) + } + + /// After the name of a tag comes a slash, the end, or an attribute. + fn end_of_tag_name(&self) -> Step<()> { + match self.peek() { + None => self.end_of_input(), + Some('/' | '>' | '{') => Ok(()), + Some(char) if id_start(char) => Ok(()), + Some(_) => self.crash(format!("Unexpected {} after name", self.describe())), + } + } +} + +fn parsed(step: Step) -> Parsed { + match step { + Ok(value) => Parsed::Ok(value), + Err(Stop::Nok) => Parsed::Nok, + Err(Stop::More(fallback)) => Parsed::More(fallback), + Err(Stop::Error(message)) => Parsed::Error(message), + } +} + +/// Parses the JSX tag that starts at `pos`, a `<`. +pub(super) fn tag(src: &str, pos: usize) -> Parsed { + let mut cursor = Cursor { src, index: pos + 1 }; + parsed(cursor.tag()) +} + +/// Parses the expression that starts at `pos`, a `{`, returning the offset after it and its value. +pub(super) fn expression(src: &str, pos: usize) -> Parsed<(usize, SmolStr)> { + let mut cursor = Cursor { src, index: pos }; + parsed( + cursor + .expression() + .map(|(start, end)| (end + 1, cursor.expression_value(start, end))), + ) +} diff --git a/crates/mq-markdown/src/parser/mdx_flow.rs b/crates/mq-markdown/src/parser/mdx_flow.rs new file mode 100644 index 000000000..6fc74b457 --- /dev/null +++ b/crates/mq-markdown/src/parser/mdx_flow.rs @@ -0,0 +1,208 @@ +//! MDX flow content: JSX tags and expressions that make up whole lines. + +use super::line::Line; +use super::mdx::{self, Fallback, Parsed}; +use super::tree::{Block, InlineSource, JsxTag}; +use crate::node::{MdxFlowExpression, Node, Position}; + +/// What the items of a line of MDX flow content add up to. +pub(super) enum Flow { + /// Not flow content. When some items were parsed, the offset where the last one ends. + Nok(Option), + More(Fallback), + Error(String), + /// The items, and the offset where the last one ends. + Done(Vec, usize), +} + +pub(super) enum FlowItem { + Tag(mdx::Tag, usize), + Expression(smol_str::SmolStr, usize, usize), +} + +/// Parses JSX tags and expressions that make up whole lines: they may only be separated by spaces. +fn flow_items(source: &InlineSource) -> Flow { + let text = source.text.as_str(); + let mut index = text.len() - text.trim_start_matches([' ', '\t']).len(); + let mut items = Vec::new(); + + loop { + let start = index; + let after_expression = match text[index..].chars().next() { + Some('<') => match mdx::tag(text, index) { + Parsed::Ok(tag) => { + index = tag.end; + items.push(FlowItem::Tag(tag, start)); + false + } + Parsed::Nok => return Flow::Nok(items.last().map(item_end)), + Parsed::More(fallback) => return Flow::More(fallback), + Parsed::Error(message) => return Flow::Error(message), + }, + Some('{') => match mdx::expression(text, index) { + Parsed::Ok((end, value)) => { + index = end; + items.push(FlowItem::Expression(value, start, end)); + true + } + Parsed::Nok => return Flow::Nok(items.last().map(item_end)), + Parsed::More(fallback) => return Flow::More(fallback), + Parsed::Error(message) => return Flow::Error(message), + }, + _ => return Flow::Nok(items.last().map(item_end)), + }; + + let end = index; + index += text[index..].len() - text[index..].trim_start_matches([' ', '\t']).len(); + match text[index..].chars().next() { + None | Some('\n' | '\r') => return Flow::Done(items, end), + Some('<') => {} + Some('{') if !after_expression => {} + Some(_) => return Flow::Nok(Some(end)), + } + } +} + +fn item_end(item: &FlowItem) -> usize { + match item { + FlowItem::Tag(tag, _) => tag.end, + FlowItem::Expression(_, _, end) => *end, + } +} + +/// What a line of MDX makes of flow content. +pub(super) enum FlowOutcome { + /// Flow content that uses the lines up to the given index. + Flow(usize), + /// Not flow content. Lines that its items span up to the given index still belong together. + Nok(Option), + /// The document ends inside a construct that could have been flow content. It is not, but a + /// paragraph is not continued lazily with such a line. + Pending, +} + +/// Parses MDX flow content that starts at `lines[start]` and pushes its blocks. +pub(super) fn mdx_flow(lines: &[Line<'_>], start: usize, blocks: &mut Vec) -> FlowOutcome { + let mut count = 1; + loop { + let end = (start + count).min(lines.len()); + let mut source = InlineSource::new( + lines[start..end] + .iter() + .map(|line| (line.text, line.eol, line.point(0))), + ); + // The last line has its line ending, unless the document ends without one. + source.text.push_str(lines[end - 1].eol); + // The container ends but the document goes on after it. + let container_end = end == lines.len() && lines.last().is_some_and(|line| !line.eof); + match flow_items(&source) { + // The construct goes on: read more lines, unless the container has none left. + Flow::More(_) if end < lines.len() => count *= 2, + Flow::More(_) if container_end => { + blocks.push(Block::Error( + "Unexpected lazy line in expression in container, expected line to be prefixed with `>` when in a block quote, whitespace when in a list, etc".into(), + )); + return FlowOutcome::Flow(lines.len()); + } + Flow::More(Fallback::Nok) => return FlowOutcome::Pending, + Flow::More(Fallback::Error(message)) => { + blocks.push(Block::Error(message)); + return FlowOutcome::Flow(lines.len()); + } + Flow::Nok(end) => { + let last_line = end.map(|end| { + start + + source + .lines + .partition_point(|(offset, _)| *offset < end) + .saturating_sub(1) + }); + return FlowOutcome::Nok(last_line); + } + Flow::Error(message) => { + blocks.push(Block::Error(message)); + return FlowOutcome::Flow(lines.len()); + } + Flow::Done(items, last) => { + let last_line = start + + source + .lines + .partition_point(|(offset, _)| *offset < last) + .saturating_sub(1); + if lines[start + 1..=last_line].iter().any(|line| line.lazy) { + blocks.push(Block::Error( + "Unexpected lazy line in expression in container, expected line to be prefixed with `>` when in a block quote, whitespace when in a list, etc".into(), + )); + return FlowOutcome::Flow(lines.len()); + } + for item in items { + blocks.push(match item { + FlowItem::Tag(tag, from) => Block::Jsx(JsxTag { + name: tag.name, + attributes: tag.attributes, + kind: tag.kind, + position: Position { + start: source.point(from), + end: source.point(tag.end), + }, + }), + FlowItem::Expression(value, from, to) => { + Block::Node(Node::MdxFlowExpression(MdxFlowExpression { + value, + position: Some(Position { + start: source.point(from), + end: source.point(to), + }), + })) + } + }); + } + return FlowOutcome::Flow(last_line + 1); + } + } + } +} + +/// Whether `lines[index]` cannot continue a paragraph lazily because MDX flow content starts there, +/// or could have started there if the document had gone on. +pub(super) fn blocks_lazy_continuation(lines: &[Line<'_>], index: usize) -> bool { + matches!( + probe_mdx_flow(lines, index), + Some(FlowOutcome::Flow(_) | FlowOutcome::Pending) + ) +} + +pub(super) fn probe_mdx_flow(lines: &[Line<'_>], index: usize) -> Option { + let line = &lines[index]; + (line.mdx + && !line.lazy + && matches!( + line.text.trim_start_matches([' ', '\t']).as_bytes().first(), + Some(b'<' | b'{') + )) + .then(|| mdx_flow(lines, index, &mut Vec::new())) +} + +/// Whether a single MDX line looks like flow content, for tracking what a paragraph can continue. +pub(super) fn looks_like_mdx_flow(line: &Line<'_>) -> bool { + if !line.mdx + || !matches!( + line.text.trim_start_matches([' ', '\t']).as_bytes().first(), + Some(b'<' | b'{') + ) + { + return false; + } + // The line ending counts, unless the document ends here. + let mut source = InlineSource::new(std::iter::once((line.text, "", line.point(0)))); + source.text.push_str(line.eol); + !matches!(flow_items(&source), Flow::Nok(_)) +} + +/// The last line of `first..=last` that a paragraph takes when flow content spans them: it stops +/// before a blank line, which ends the paragraph whatever a construct that spans it is. +pub(super) fn absorbed_until(lines: &[Line<'_>], first: usize, last: usize) -> usize { + (first + 1..=last) + .find(|&index| lines[index].is_blank()) + .map_or(last, |blank| blank - 1) +} diff --git a/crates/mq-markdown/src/parser/resolve.rs b/crates/mq-markdown/src/parser/resolve.rs new file mode 100644 index 000000000..0481d236f --- /dev/null +++ b/crates/mq-markdown/src/parser/resolve.rs @@ -0,0 +1,226 @@ +//! Turns the block tree into nodes: parses the inline content, pairs JSX tags and flattens lists and +//! tables. + +use super::inline; +use super::mdx::TagKind; +use super::tree::{Block, InlineBlock, InlineKind, JsxTag, ListBlock, TableItem}; +use crate::node::{Footnote, Heading, Level, List, MdxJsxFlowElement, Node, Position, TableAlign, TableCell}; +use rustc_hash::FxHashSet; + +/// The identifiers that references can resolve to. +#[derive(Default)] +pub(super) struct References { + /// Normalized labels of link reference definitions. + pub(super) definitions: FxHashSet, + /// Normalized labels of footnote definitions. + pub(super) footnotes: FxHashSet, + /// Whether the document is MDX. + pub(super) mdx: bool, +} + +pub(super) fn resolve(blocks: Vec, mdx: bool) -> Result, String> { + let mut references = References { + mdx, + ..References::default() + }; + collect(&blocks, &mut references); + + let mut nodes = Vec::new(); + flatten(blocks, &references, &mut nodes)?; + Ok(nodes) +} + +fn collect(blocks: &[Block], references: &mut References) { + for block in blocks { + match block { + Block::Node(Node::Definition(definition)) => { + references.definitions.insert(definition.ident.clone()); + } + Block::Quote(quote) => collect("e.children, references), + Block::Footnote(footnote) => { + references.footnotes.insert(footnote.ident.clone()); + collect(&footnote.children, references); + } + Block::List(list) => list.items.iter().for_each(|item| collect(&item.children, references)), + _ => {} + } + } +} + +/// The JSX elements that are open while blocks are turned into nodes, and the nodes inside the +/// innermost one. +struct Frames { + open: Vec<(JsxTag, Vec)>, + root: Vec, +} + +impl Frames { + fn push(&mut self, node: Node) { + match self.open.last_mut() { + Some((_, children)) => children.push(node), + None => self.root.push(node), + } + } + + fn tag(&mut self, tag: JsxTag) -> Result<(), String> { + match tag.kind { + TagKind::Open => self.open.push((tag, Vec::new())), + TagKind::SelfClosing => self.push(element(tag, Vec::new(), None)), + TagKind::Close => { + let Some((open, children)) = self.open.pop() else { + return Err("Unexpected closing slash `/` in tag, expected an open tag first".into()); + }; + if open.name != tag.name { + return Err(format!( + "Unexpected closing tag ``, expected corresponding closing tag for `<{}>` ({}:{})", + tag.name.as_deref().unwrap_or_default(), + open.name.as_deref().unwrap_or_default(), + open.position.start.line, + open.position.start.column + )); + } + let end = tag.position.end; + self.push(element(open, children, Some(end))); + } + } + Ok(()) + } +} + +fn element(tag: JsxTag, children: Vec, end: Option) -> Node { + Node::MdxJsxFlowElement(MdxJsxFlowElement { + children, + position: Some(Position { + start: tag.position.start, + end: end.unwrap_or(tag.position.end), + }), + name: tag.name, + attributes: tag.attributes, + }) +} + +fn flatten(blocks: Vec, references: &References, nodes: &mut Vec) -> Result<(), String> { + let mut frames = Frames { + open: Vec::new(), + root: Vec::new(), + }; + + for block in blocks { + match block { + Block::Node(node) => frames.push(node), + Block::Inline(block) => { + let mut out = Vec::new(); + inline_block(block, references, &mut out)?; + out.into_iter().for_each(|node| frames.push(node)); + } + Block::Quote(quote) => { + let mut values = Vec::new(); + flatten(quote.children, references, &mut values)?; + frames.push(Node::from_blockquote(values, Some(quote.position))); + } + Block::List(list) => { + let mut out = Vec::new(); + list_nodes(list, 0, references, &mut out)?; + out.into_iter().for_each(|node| frames.push(node)); + } + Block::Footnote(footnote) => { + let mut values = Vec::new(); + flatten(footnote.children, references, &mut values)?; + frames.push(Node::Footnote(Footnote { + ident: footnote.ident, + values, + position: Some(footnote.position), + })); + } + Block::Table(items) => { + for item in items { + let node = match item { + TableItem::Cell { + row, + column, + position, + source, + } => Node::TableCell(TableCell { + values: match source { + Some(source) => inline::parse(&source, references)?, + None => Vec::new(), + }, + column, + row, + position: Some(position), + }), + TableItem::Align { align, position } => Node::TableAlign(TableAlign { + align, + position: Some(position), + }), + }; + frames.push(node); + } + } + Block::Jsx(tag) => frames.tag(tag)?, + Block::Error(message) => return Err(message), + } + } + + if let Some((open, _)) = frames.open.last() { + return Err(format!( + "Expected a closing tag for `<{}>` ({}:{}) before the end of its container", + open.name.as_deref().unwrap_or_default(), + open.position.start.line, + open.position.start.column + )); + } + nodes.append(&mut frames.root); + + Ok(()) +} + +fn inline_block(block: InlineBlock, references: &References, nodes: &mut Vec) -> Result<(), String> { + let values = inline::parse(&block.source, references)?; + match block.kind { + InlineKind::Paragraph => nodes.extend(values), + InlineKind::Heading { depth, position } => nodes.push(Node::Heading(Heading { + depth, + values, + position: Some(position), + })), + } + Ok(()) +} + +/// Emits one flat `Node::List` per item, followed by the items of its nested lists one level deeper. +fn list_nodes(list: ListBlock, level: Level, references: &References, nodes: &mut Vec) -> Result<(), String> { + for (index, item) in list.items.into_iter().enumerate() { + let (nested, others): (Vec<_>, Vec<_>) = item.children.into_iter().partition(|b| matches!(b, Block::List(_))); + let mut values = Vec::new(); + flatten(others, references, &mut values)?; + let position = match ( + values.first().and_then(Node::position), + values.last().and_then(Node::position), + ) { + (Some(first), Some(last)) => Position { + start: first.start.clone(), + end: last.end.clone(), + }, + _ => item.position, + }; + + nodes.push(Node::List(List { + values, + index, + level, + ordered: list.ordered, + checked: item.checked, + spread: list.spread, + start: list.start, + position: Some(position), + })); + + for block in nested { + if let Block::List(sub_list) = block { + list_nodes(sub_list, level + 1, references, nodes)?; + } + } + } + Ok(()) +} diff --git a/crates/mq-markdown/src/parser/table.rs b/crates/mq-markdown/src/parser/table.rs new file mode 100644 index 000000000..47199a7b8 --- /dev/null +++ b/crates/mq-markdown/src/parser/table.rs @@ -0,0 +1,206 @@ +//! GFM tables. + +use super::line::Line; +use super::tree::{InlineSource, TableItem}; +use crate::node::{Point, Position, TableAlignKind}; + +/// One cell of a row, as byte offsets within the line. +struct Cell<'a> { + /// Start of the cell, at its leading pipe when it has one. + start: usize, + end: usize, + content: &'a str, + content_offset: usize, +} + +/// Splits a row into cells on unescaped pipes. Leading and trailing pipes do not make cells. +fn split_row<'a>(line: &Line<'a>) -> Vec> { + let text = line.text; + let indent = text.len() - text.trim_start_matches([' ', '\t']).len(); + let end = text.trim_end_matches([' ', '\t']).len().max(indent); + let bytes = text.as_bytes(); + + let mut pipes = Vec::new(); + let mut index = indent; + while index < end { + match bytes[index] { + b'\\' => index += 1, + b'|' => pipes.push(index), + _ => {} + } + index += 1; + } + + let leading = pipes.first() == Some(&indent); + let trailing = pipes.last().is_some_and(|&pipe| pipe + 1 == end); + + // Segments between pipes: (start of the leading pipe or content, content start, content end). + let mut segments = Vec::with_capacity(pipes.len() + 1); + let mut from = indent; + for &pipe in &pipes { + segments.push((from, pipe)); + from = pipe + 1; + } + segments.push((from, end)); + + let mut first = 0; + let mut last = segments.len(); + if leading { + first = 1; + } + if trailing && last - first > 1 { + last -= 1; + } + + (first..last) + .map(|index| { + let (from, to) = segments[index]; + let raw = &text[from..to]; + let content = raw.trim_matches([' ', '\t']); + let content_offset = from + (raw.len() - raw.trim_start_matches([' ', '\t']).len()); + // The first cell starts at the start of the line, later ones at their leading pipe. + let start = if index == first { 0 } else { pipes[index - 1] }; + let end = if index + 1 == last { end } else { pipes[index] }; + Cell { + start, + end, + content, + content_offset, + } + }) + .collect() +} + +/// Parses a delimiter cell such as `:-:` into its alignment. +fn align(content: &str) -> Option { + let left = content.starts_with(':'); + let right = content.ends_with(':') && content.len() > 1; + let dashes = content.trim_start_matches(':').trim_end_matches(':'); + if dashes.is_empty() || !dashes.bytes().all(|b| b == b'-') { + return None; + } + Some(match (left, right) { + (true, true) => TableAlignKind::Center, + (true, false) => TableAlignKind::Left, + (false, true) => TableAlignKind::Right, + (false, false) => TableAlignKind::None, + }) +} + +/// The alignments of a table that starts with `lines[start]` as its header, if it is one. +fn header<'a>( + lines: &[Line<'a>], + start: usize, + starts_block: fn(&Line<'_>) -> bool, +) -> Option<(Vec>, Vec)> { + let head = &lines[start]; + if head.mdx { + return None; + } + let delimiter = lines.get(start + 1)?; + // The delimiter row cannot start another block, such as a list item. + if head.indent().0 >= 4 || delimiter.indent().0 >= 4 || delimiter.lazy || starts_block(delimiter) { + return None; + } + let text = delimiter.text.trim_matches([' ', '\t']); + if !text.contains('|') + || !text.contains('-') + || !text.bytes().all(|b| matches!(b, b'|' | b':' | b'-' | b' ' | b'\t')) + { + return None; + } + + let cells = split_row(head); + let aligns = split_row(delimiter) + .iter() + .map(|cell| align(cell.content)) + .collect::>>()?; + (cells.len() == aligns.len()).then_some((cells, aligns)) +} + +/// Whether a table starts at `lines[start]`. +pub(super) fn starts_at(lines: &[Line<'_>], start: usize, starts_block: fn(&Line<'_>) -> bool) -> bool { + header(lines, start, starts_block).is_some() +} + +fn cell_item(line: &Line<'_>, row: usize, column: usize, cell: &Cell<'_>) -> TableItem { + let source = (!cell.content.is_empty()).then(|| { + let mut source = InlineSource::new(std::iter::once((cell.content, "", line.point(cell.content_offset)))); + source.table = true; + source + }); + TableItem::Cell { + row, + column, + position: Position { + start: line.point(cell.start), + end: line.point(cell.end), + }, + source, + } +} + +/// Parses a table starting at `lines[start]`, returning its nodes and the index after it. +/// `ends_row` tells which lines end the table besides blank lines. +pub(super) fn parse( + lines: &[Line<'_>], + start: usize, + ends_row: fn(&Line<'_>) -> bool, +) -> Option<(Vec, usize)> { + let (cells, aligns) = header(lines, start, ends_row)?; + + let mut items = Vec::new(); + for (column, cell) in cells.iter().enumerate() { + items.push(cell_item(&lines[start], 0, column, cell)); + } + let delimiter_line = lines[start + 1].number; + items.push(TableItem::Align { + align: aligns, + position: Position { + start: Point { + line: delimiter_line, + column: 1, + }, + end: Point { + line: delimiter_line, + column: 1, + }, + }, + }); + + let mut index = start + 2; + let mut row = 1; + while let Some(line) = lines.get(index) { + if line.is_blank() || line.indent().0 >= 4 || ends_row(line) { + break; + } + for (column, cell) in split_row(line).iter().enumerate() { + items.push(cell_item(line, row, column, cell)); + } + row += 1; + index += 1; + } + + Some((items, index)) +} + +/// The number of cells in the row `line`. +pub(super) fn row_cells(line: &Line<'_>) -> usize { + split_row(line).len() +} + +/// The number of cells of `line` when it is a delimiter row such as `|:-|-:|`. +pub(super) fn delimiter_cells(line: &Line<'_>) -> Option { + let text = line.text.trim_matches([' ', '\t']); + if !text.contains('|') + || !text.contains('-') + || !text.bytes().all(|b| matches!(b, b'|' | b':' | b'-' | b' ' | b'\t')) + { + return None; + } + let cells = split_row(line); + cells + .iter() + .all(|cell| align(cell.content).is_some()) + .then_some(cells.len()) +} diff --git a/crates/mq-markdown/src/parser/tree.rs b/crates/mq-markdown/src/parser/tree.rs new file mode 100644 index 000000000..292d4f9b5 --- /dev/null +++ b/crates/mq-markdown/src/parser/tree.rs @@ -0,0 +1,143 @@ +//! The block tree produced by block parsing, before inline content is parsed. +//! +//! Inline parsing needs to know every definition of the document, including those that come later, so +//! it runs in [`super::resolve`] once all blocks are known. + +use super::line::visual_column; +use super::mdx::TagKind; +use crate::node::{MdxAttributeContent, Node, Point, Position, TableAlignKind}; + +/// Raw inline content of a paragraph, heading or table cell, with the source position of each line. +pub(super) struct InlineSource { + pub(super) text: String, + /// For each line, the offset in `text` where it starts and its position in the document. + pub(super) lines: Vec<(usize, Point)>, + /// Whether this is the content of a table cell, where `\|` in code stands for `|`. + pub(super) table: bool, +} + +impl InlineSource { + /// Joins `(text, eol, start)` lines, keeping the terminators between them. + pub(super) fn new<'a>(lines: impl ExactSizeIterator) -> Self { + let count = lines.len(); + let mut text = String::new(); + let mut starts = Vec::with_capacity(count); + for (index, (line, eol, start)) in lines.enumerate() { + starts.push((text.len(), start)); + text.push_str(line); + if index + 1 < count { + text.push_str(eol); + } + } + Self { + text, + lines: starts, + table: false, + } + } + + /// The position where a node ends at `offset`. After a line ending that is the start of the next + /// line in the document, not the start of its content inside containers. + pub(super) fn end_point(&self, offset: usize) -> Point { + let index = self + .lines + .partition_point(|(start, _)| *start <= offset) + .saturating_sub(1); + let (start, point) = &self.lines[index]; + if *start == offset && offset > 0 { + return Point { + line: point.line, + column: 1, + }; + } + self.point(offset) + } + + /// The position of the byte at `offset` in `text`. + pub(super) fn point(&self, offset: usize) -> Point { + let index = self + .lines + .partition_point(|(start, _)| *start <= offset) + .saturating_sub(1); + let (start, point) = &self.lines[index]; + let passed = &self.text[*start..offset]; + let column = if passed.contains('\t') { + visual_column(passed.chars(), point.column - 1) + 1 + } else { + point.column + passed.len() + }; + Point { + line: point.line, + column, + } + } +} + +pub(super) enum Block { + /// A finished leaf such as a code block or a thematic break. + Node(Node), + Inline(InlineBlock), + Quote(QuoteBlock), + List(ListBlock), + Footnote(FootnoteBlock), + Table(Vec), + /// A JSX tag on a line of its own in MDX. Tags are paired when the blocks become nodes. + Jsx(JsxTag), + /// Invalid MDX, reported when the blocks become nodes. + Error(String), +} + +pub(super) struct InlineBlock { + pub(super) source: InlineSource, + pub(super) kind: InlineKind, +} + +pub(super) enum InlineKind { + Paragraph, + Heading { depth: u8, position: Position }, +} + +pub(super) struct QuoteBlock { + pub(super) children: Vec, + pub(super) position: Position, +} + +pub(super) struct FootnoteBlock { + /// The normalized label. + pub(super) ident: String, + pub(super) children: Vec, + pub(super) position: Position, +} + +pub(super) struct ListBlock { + pub(super) ordered: bool, + pub(super) start: Option, + pub(super) spread: bool, + pub(super) items: Vec, +} + +pub(super) struct Item { + pub(super) checked: Option, + pub(super) children: Vec, + pub(super) position: Position, +} + +pub(super) enum TableItem { + Cell { + row: usize, + column: usize, + position: Position, + source: Option, + }, + Align { + align: Vec, + position: Position, + }, +} + +pub(super) struct JsxTag { + pub(super) name: Option, + pub(super) attributes: Vec, + pub(super) kind: TagKind, + pub(super) position: Position, +} From 491290d4989951efa2ea21bcc75ab68f2389c228 Mon Sep 17 00:00:00 2001 From: harehare Date: Thu, 8 Oct 2026 13:05:06 +0900 Subject: [PATCH 02/88] =?UTF-8?q?=F0=9F=90=9B=20fix(typos):=20exclude=20th?= =?UTF-8?q?e=20HTML=20entity=20table=20from=20the=20check?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The entity names in the native parser are defined by the HTML spec and include short non-words such as `ges` and `curren`. --- .typos.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.typos.toml b/.typos.toml index 31075d51e..77c92976a 100644 --- a/.typos.toml +++ b/.typos.toml @@ -1,3 +1,7 @@ +[files] +# HTML entity names from the HTML spec, which include short non-words +extend-exclude = ["crates/mq-markdown/src/parser/inline/entities.rs"] + [default.extend-words] # HashiCorp is a proper noun Hashi = "Hashi" From 9432ec56f5917aa1492659494dde4a1d2873fd6f Mon Sep 17 00:00:00 2001 From: harehare Date: Thu, 8 Oct 2026 13:05:12 +0900 Subject: [PATCH 03/88] =?UTF-8?q?=E2=9C=A8=20feat(markdown):=20switch=20to?= =?UTF-8?q?=20the=20native=20parser=20and=20remove=20markdown-rs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Parse and render HTML with the parser of this crate only. The `native-parser` feature, the mdast conversion and the markdown dependency are gone, and `to_html` is rendered from the block tree by a new renderer. Differences from markdown-rs found by comparing both on the CommonMark and GFM spec, 83k real documents and generated input are fixed or documented: - a panic on `www` before a multibyte character - a byte order mark, multiline inline HTML, and tabs that a container consumes in part - tables: trailing whitespace, single column, and rows that end a table - control character references, escaped footnote labels, NUL in destinations - MDX: lone CR, `~` in emphasis, Unicode names of JSX tags `unicode-id` is now a direct dependency for the names of JSX tags. --- Cargo.lock | 11 +- Cargo.toml | 2 +- crates/mq-markdown/Cargo.toml | 4 +- crates/mq-markdown/src/lib.rs | 7 +- crates/mq-markdown/src/markdown.rs | 149 +- crates/mq-markdown/src/node.rs | 512 ------ crates/mq-markdown/src/parser.rs | 1404 +++++++++++------ crates/mq-markdown/src/parser/block.rs | 69 +- crates/mq-markdown/src/parser/code.rs | 11 +- crates/mq-markdown/src/parser/inline.rs | 16 +- .../mq-markdown/src/parser/inline/emphasis.rs | 7 +- .../mq-markdown/src/parser/inline/entity.rs | 8 +- .../mq-markdown/src/parser/inline/literal.rs | 4 +- crates/mq-markdown/src/parser/inline/tail.rs | 3 +- crates/mq-markdown/src/parser/line.rs | 13 +- crates/mq-markdown/src/parser/mdx.rs | 37 +- crates/mq-markdown/src/parser/render_html.rs | 711 +++++++++ crates/mq-markdown/src/parser/resolve.rs | 3 +- crates/mq-markdown/src/parser/snapshots.txt | 1311 +++++++++++++++ crates/mq-markdown/src/parser/table.rs | 19 +- crates/mq-markdown/src/parser/tree.rs | 10 + 21 files changed, 3092 insertions(+), 1219 deletions(-) create mode 100644 crates/mq-markdown/src/parser/render_html.rs create mode 100644 crates/mq-markdown/src/parser/snapshots.txt diff --git a/Cargo.lock b/Cargo.lock index f50853854..d057b3438 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2505,15 +2505,6 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" -[[package]] -name = "markdown" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5cab8f2cadc416a82d2e783a1946388b31654d391d1c7d92cc1f03e295b1deb" -dependencies = [ - "unicode-id", -] - [[package]] name = "markup5ever" version = "0.39.0" @@ -2911,7 +2902,6 @@ dependencies = [ "codspeed-divan-compat", "ego-tree", "itertools 0.15.0", - "markdown", "miette", "proptest", "rstest", @@ -2920,6 +2910,7 @@ dependencies = [ "serde", "serde_json", "smol_str", + "unicode-id", "unicode-width 0.2.2", "ureq", "url", diff --git a/Cargo.toml b/Cargo.toml index 95c5a1658..4492e6384 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -80,7 +80,6 @@ itertools = "0.15.0" js-sys = "0.3.77" libc = "0.2" libfuzzer-sys = "0.4" -markdown = "1.0.0" memchr = "2" nix = {version = "0.31", default-features = false, features = ["fs"]} md5 = "0.8.0" @@ -128,6 +127,7 @@ tower-lsp-server = "0.23.0" tracing = "0.1.44" tracing-opentelemetry = "0.34" tracing-subscriber = "0.3.23" +unicode-id = "0.3.6" unicode-normalization = "0.1.25" unicode-segmentation = "1.13.3" unicode-width = "0.2.2" diff --git a/crates/mq-markdown/Cargo.toml b/crates/mq-markdown/Cargo.toml index 26bd61f30..0dd25572f 100644 --- a/crates/mq-markdown/Cargo.toml +++ b/crates/mq-markdown/Cargo.toml @@ -14,13 +14,13 @@ version.workspace = true [dependencies] ego-tree = {workspace = true, optional = true} itertools = {workspace = true} -markdown = {workspace = true} miette = {workspace = true} rustc-hash = {workspace = true} scraper = {workspace = true, optional = true} serde = {workspace = true, features = ["derive"], optional = true} serde_json = {workspace = true, optional = true} smol_str = {workspace = true} +unicode-id = {workspace = true} unicode-width = {workspace = true} url = {workspace = true, optional = true} yaml-rust2 = {workspace = true, optional = true} @@ -38,10 +38,8 @@ name = "benchmark" [features] callout = [] color = [] -default = ["native-parser"] embed = [] html-to-markdown = ["dep:scraper", "dep:ego-tree", "dep:serde_json", "dep:yaml-rust2", "dep:url"] json = ["dep:serde", "dep:serde_json", "smol_str/serde"] -native-parser = [] obsidian = ["wikilink", "callout", "embed"] wikilink = [] diff --git a/crates/mq-markdown/src/lib.rs b/crates/mq-markdown/src/lib.rs index a40de0cef..5efa438ef 100644 --- a/crates/mq-markdown/src/lib.rs +++ b/crates/mq-markdown/src/lib.rs @@ -11,10 +11,8 @@ //! - **MDX Support**: Parse and manipulate MDX (Markdown + JSX) content //! - **JSON Export**: Serialize markdown AST to JSON (with `json` feature) //! - **Configurable Rendering**: Customize output formatting and styles -//! - **Built-in parser**: The `native-parser` feature, on by default, parses CommonMark, GFM (tables, -//! strikethrough, autolink literals, task lists, footnotes), frontmatter, math and MDX with a parser -//! of this crate. Without it, `markdown-rs` does the parsing, with the same results for all but a few -//! rare inputs. `Markdown::to_html` always uses `markdown-rs`. +//! - **Built-in parser**: CommonMark, GFM (tables, strikethrough, autolink literals, task lists, +//! footnotes), frontmatter, math and MDX are parsed, and rendered to HTML, by a parser of this crate. //! //! ## Quick Start //! @@ -86,7 +84,6 @@ mod html_to_markdown; mod markdown; mod node; mod node_kind; -#[cfg(any(feature = "native-parser", test))] mod parser; pub use markdown::{Markdown, to_html}; pub use node::{ diff --git a/crates/mq-markdown/src/markdown.rs b/crates/mq-markdown/src/markdown.rs index ccb77d6a6..e3eeafd6b 100644 --- a/crates/mq-markdown/src/markdown.rs +++ b/crates/mq-markdown/src/markdown.rs @@ -6,8 +6,7 @@ use crate::node::{ Code, ColorTheme, Node, Position, RenderOptions, TableAlign, TableCell, list_own_prefix_width, reindent_all_lines, render_values, }; -use markdown::{CompileOptions, Constructs, Options, ParseOptions}; -#[cfg(any(not(feature = "native-parser"), test, feature = "json", feature = "html-to-markdown"))] +#[cfg(any(feature = "json", feature = "html-to-markdown"))] use miette::miette; use std::{fmt, str::FromStr}; use table_layout::{TableLayout, write_padded_cell}; @@ -241,8 +240,7 @@ impl Markdown { } pub fn to_html(&self) -> String { - let md_str = self.to_string(); - markdown::to_html_with_options(&md_str, &html_options()).unwrap_or_else(|_| markdown::to_html(&md_str)) + to_html(&self.to_string()) } pub fn to_text(&self) -> String { @@ -314,149 +312,22 @@ impl Markdown { } } -/// Parses `content` with the parser selected by the `native-parser` feature. +/// Parses `content` into nodes. fn parse_nodes(content: &str) -> miette::Result> { - #[cfg(feature = "native-parser")] - { - crate::parser::parse(content) - } - #[cfg(not(feature = "native-parser"))] - { - parse_with_markdown_rs(content) - } + crate::parser::parse(content) } -/// Parses `content` as MDX with the parser selected by the `native-parser` feature. +/// Parses `content` as MDX into nodes. fn parse_mdx_nodes(content: &str) -> miette::Result> { - #[cfg(feature = "native-parser")] - { - crate::parser::parse_mdx(content) - } - #[cfg(not(feature = "native-parser"))] - { - parse_mdx_with_markdown_rs(content) - } -} - -/// Parses `content` as MDX with `markdown-rs` and converts the `mdast` tree into nodes. -#[cfg(any(not(feature = "native-parser"), test))] -pub(crate) fn parse_mdx_with_markdown_rs(content: &str) -> miette::Result> { - let root = markdown::to_mdast(content, &markdown::ParseOptions::mdx()).map_err(|e| miette!(e.reason))?; - Ok(Node::from_mdast_node(root)) + crate::parser::parse_mdx(content) } -/// Parses `content` with `markdown-rs` and converts the `mdast` tree into nodes. -#[cfg(any(not(feature = "native-parser"), test))] -pub(crate) fn parse_with_markdown_rs(content: &str) -> miette::Result> { - let root = markdown::to_mdast( - content, - &markdown::ParseOptions { - gfm_strikethrough_single_tilde: true, - math_text_single_dollar: true, - mdx_expression_parse: None, - mdx_esm_parse: None, - constructs: Constructs { - attention: true, - autolink: true, - block_quote: true, - character_escape: true, - character_reference: true, - code_indented: true, - code_fenced: true, - code_text: true, - definition: true, - frontmatter: true, - gfm_autolink_literal: true, - gfm_label_start_footnote: true, - gfm_footnote_definition: true, - gfm_strikethrough: true, - gfm_table: true, - gfm_task_list_item: true, - hard_break_escape: true, - hard_break_trailing: true, - heading_atx: true, - heading_setext: true, - html_flow: true, - html_text: true, - label_start_image: true, - label_start_link: true, - label_end: true, - list_item: true, - math_flow: true, - math_text: true, - mdx_esm: false, - mdx_expression_flow: false, - mdx_expression_text: false, - mdx_jsx_flow: false, - mdx_jsx_text: false, - thematic_break: true, - }, - }, - ) - .map_err(|e| miette!(e.reason))?; - Ok(Node::from_mdast_node(root)) -} - -/// Returns the shared `Options` used for both `Markdown::to_html` and the -/// standalone `to_html` helper. The options mirror the constructs that are -/// enabled during parsing (see `from_markdown_str`) so that every feature -/// that can be *parsed* is also correctly *rendered* to HTML, including: +/// Converts Markdown to HTML. /// -/// - GFM tables → `` -/// - GFM task-list items → `` -/// - GFM strikethrough → `` -/// - GFM footnotes -/// - GFM autolink literals -/// - Math (flow and inline) via `` -/// - YAML / TOML frontmatter (stripped from HTML output) -fn html_options() -> Options { - Options { - parse: ParseOptions { - gfm_strikethrough_single_tilde: true, - math_text_single_dollar: true, - constructs: Constructs { - attention: true, - autolink: true, - block_quote: true, - character_escape: true, - character_reference: true, - code_indented: true, - code_fenced: true, - code_text: true, - definition: true, - frontmatter: true, - gfm_autolink_literal: true, - gfm_label_start_footnote: true, - gfm_footnote_definition: true, - gfm_strikethrough: true, - gfm_table: true, - gfm_task_list_item: true, - hard_break_escape: true, - hard_break_trailing: true, - heading_atx: true, - heading_setext: true, - html_flow: true, - html_text: true, - label_start_image: true, - label_start_link: true, - label_end: true, - list_item: true, - math_flow: true, - math_text: true, - thematic_break: true, - ..Constructs::default() - }, - ..ParseOptions::default() - }, - compile: CompileOptions { - allow_dangerous_html: true, - ..CompileOptions::default() - }, - } -} - +/// Everything that is parsed is rendered: GFM tables, task list items, strikethrough, footnotes and +/// autolink literals, math, and raw HTML. Frontmatter is left out. pub fn to_html(s: &str) -> String { - markdown::to_html_with_options(s, &html_options()).unwrap_or_else(|_| markdown::to_html(s)) + crate::parser::to_html(s) } #[cfg(test)] diff --git a/crates/mq-markdown/src/node.rs b/crates/mq-markdown/src/node.rs index eb42d8be1..fd79f252a 100644 --- a/crates/mq-markdown/src/node.rs +++ b/crates/mq-markdown/src/node.rs @@ -3,8 +3,6 @@ use crate::node::attr_value::{ attr_keys::{self, CHILDREN}, }; use itertools::Itertools; -#[cfg(any(not(feature = "native-parser"), test))] -use markdown::mdast::{self}; use smol_str::SmolStr; use std::{ borrow::Cow, @@ -148,12 +146,6 @@ impl ColorTheme<'_> { pub(crate) type Level = u8; -#[cfg(any(not(feature = "native-parser"), test))] -pub const EMPTY_NODE: Node = Node::Text(Text { - value: String::new(), - position: None, -}); - #[derive(Debug, Clone, Default, PartialEq)] pub struct RenderOptions { pub list_style: ListStyle, @@ -257,18 +249,6 @@ pub enum TableAlignKind { None, } -#[cfg(any(not(feature = "native-parser"), test))] -impl From for TableAlignKind { - fn from(value: mdast::AlignKind) -> Self { - match value { - mdast::AlignKind::Left => Self::Left, - mdast::AlignKind::Right => Self::Right, - mdast::AlignKind::Center => Self::Center, - mdast::AlignKind::None => Self::None, - } - } -} - impl From<&str> for TableAlignKind { fn from(value: &str) -> Self { match value { @@ -883,22 +863,6 @@ pub struct Point { pub column: usize, } -#[cfg(any(not(feature = "native-parser"), test))] -impl From for Position { - fn from(value: markdown::unist::Position) -> Self { - Self { - start: Point { - line: value.start.line, - column: value.start.column, - }, - end: Point { - line: value.end.line, - column: value.end.column, - }, - } - } -} - impl From for Node { fn from(value: String) -> Self { Self::Text(Text { value, position: None }) @@ -3106,482 +3070,6 @@ impl Node { } } - #[cfg(any(not(feature = "native-parser"), test))] - pub(crate) fn from_mdast_node(node: mdast::Node) -> Vec { - match node { - mdast::Node::Root(root) => root - .children - .into_iter() - .flat_map(Self::from_mdast_node) - .collect::>(), - mdast::Node::ListItem(list_item) => list_item - .children - .into_iter() - .flat_map(Self::from_mdast_node) - .collect::>(), - mdast::Node::List(list) => Self::mdast_list_items(&list, 0), - mdast::Node::Table(table) => table - .children - .iter() - .enumerate() - .flat_map(|(row, n)| { - if let mdast::Node::TableRow(table_row) = n { - itertools::concat(vec![ - table_row - .children - .iter() - .enumerate() - .flat_map(|(column, node)| { - if let mdast::Node::TableCell(_) = node { - vec![Self::TableCell(TableCell { - row, - column, - values: Self::mdast_children_to_node(node.clone()), - position: node.position().map(|p| p.clone().into()), - })] - } else { - Vec::new() - } - }) - .collect(), - if row == 0 { - vec![Self::TableAlign(TableAlign { - align: table.align.iter().map(|a| (*a).into()).collect::>(), - position: n.position().map(|p| Position { - start: Point { - line: p.start.line + 1, - column: 1, - }, - end: Point { - line: p.start.line + 1, - column: 1, - }, - }), - })] - } else { - Vec::new() - }, - ]) - } else { - Vec::new() - } - }) - .collect(), - mdast::Node::Code(mdast::Code { - value, - position, - lang, - meta, - .. - }) => match lang { - Some(lang) => { - vec![Self::Code(Code { - value, - lang: Some(lang), - position: position.map(|p| p.clone().into()), - meta, - fence: true, - })] - } - None => { - let line_count = position - .as_ref() - .map(|p| p.end.line - p.start.line + 1) - .unwrap_or_default(); - let fence = value.lines().count() != line_count; - - vec![Self::Code(Code { - value, - lang, - position: position.map(|p| p.clone().into()), - meta, - fence, - })] - } - }, - mdast::Node::Blockquote(mdast::Blockquote { ref position, .. }) => { - let pos = position.clone().map(|p| p.into()); - vec![Self::from_blockquote(Self::mdast_children_to_node(node), pos)] - } - mdast::Node::Definition(mdast::Definition { - url, - title, - identifier, - label, - position, - .. - }) => { - vec![Self::Definition(Definition { - ident: identifier, - url: Url(url), - label, - title: title.map(Title), - position: position.map(|p| p.clone().into()), - })] - } - mdast::Node::Heading(mdast::Heading { - depth, ref position, .. - }) => { - let position = position.clone().map(|p| p.into()); - vec![Self::Heading(Heading { - values: Self::mdast_children_to_node(node), - depth, - position, - })] - } - mdast::Node::Break(mdast::Break { position }) => { - vec![Self::Break(Break { - position: position.map(|p| p.into()), - })] - } - mdast::Node::Delete(mdast::Delete { ref position, .. }) => { - let position = position.clone().map(|p| p.into()); - vec![Self::Delete(Delete { - values: Self::mdast_children_to_node(node), - position, - })] - } - mdast::Node::Emphasis(mdast::Emphasis { ref position, .. }) => { - let position = position.clone().map(|p| p.into()); - vec![Self::Emphasis(Emphasis { - values: Self::mdast_children_to_node(node), - position, - })] - } - mdast::Node::Strong(mdast::Strong { ref position, .. }) => { - let position = position.clone().map(|p| p.into()); - vec![Self::Strong(Strong { - values: Self::mdast_children_to_node(node), - position, - })] - } - mdast::Node::ThematicBreak(mdast::ThematicBreak { position, .. }) => { - vec![Self::HorizontalRule(HorizontalRule { - position: position.map(|p| p.clone().into()), - })] - } - mdast::Node::Html(mdast::Html { value, position }) => { - vec![Self::Html(Html { - value, - position: position.map(|p| p.clone().into()), - })] - } - mdast::Node::Yaml(mdast::Yaml { value, position }) => { - vec![Self::Yaml(Yaml { - value, - position: position.map(|p| p.clone().into()), - })] - } - mdast::Node::Toml(mdast::Toml { value, position }) => { - vec![Self::Toml(Toml { - value, - position: position.map(|p| p.clone().into()), - })] - } - mdast::Node::Image(mdast::Image { - alt, - url, - title, - position, - }) => { - vec![Self::Image(Image { - alt, - url, - title, - position: position.map(|p| p.clone().into()), - })] - } - mdast::Node::ImageReference(mdast::ImageReference { - alt, - identifier, - label, - position, - .. - }) => { - vec![Self::ImageRef(ImageRef { - alt, - ident: identifier, - label, - position: position.map(|p| p.clone().into()), - })] - } - mdast::Node::InlineCode(mdast::InlineCode { value, position, .. }) => { - vec![Self::CodeInline(CodeInline { - value: value.into(), - position: position.map(|p| p.clone().into()), - })] - } - mdast::Node::InlineMath(mdast::InlineMath { value, position }) => { - vec![Self::MathInline(MathInline { - value: value.into(), - position: position.map(|p| p.clone().into()), - })] - } - mdast::Node::Link(mdast::Link { - title, - url, - position, - children, - .. - }) => { - let converted: Vec = children.into_iter().flat_map(Self::from_mdast_node).collect(); - // Flatten nested links that arise from GFM autolink literal parsing. - // When the link text is a bare URL (e.g. `[https://x](https://x)`), - // markdown-rs parses the inner text as another Link node with the same - // URL, which causes double-nesting on re-serialisation. Unwrap any - // such inner link whose URL matches the outer one. - let values = converted - .into_iter() - .flat_map(|child| match child { - Self::Link(Link { - url: ref inner_url, - ref values, - .. - }) if inner_url.0 == url => values.clone(), - other => vec![other], - }) - .collect(); - vec![Self::Link(Link { - url: Url(url), - title: title.map(Title), - values, - position: position.map(|p| p.clone().into()), - })] - } - mdast::Node::LinkReference(mdast::LinkReference { - identifier, - label, - position, - children, - .. - }) => { - vec![Self::LinkRef(LinkRef { - ident: identifier, - values: children.into_iter().flat_map(Self::from_mdast_node).collect::>(), - label, - position: position.map(|p| p.clone().into()), - })] - } - mdast::Node::Math(mdast::Math { value, position, .. }) => { - vec![Self::Math(Math { - value, - position: position.map(|p| p.clone().into()), - })] - } - mdast::Node::FootnoteDefinition(mdast::FootnoteDefinition { - identifier, - position, - children, - .. - }) => { - vec![Self::Footnote(Footnote { - ident: identifier, - values: children.into_iter().flat_map(Self::from_mdast_node).collect::>(), - position: position.map(|p| p.clone().into()), - })] - } - mdast::Node::FootnoteReference(mdast::FootnoteReference { - identifier, - label, - position, - .. - }) => { - vec![Self::FootnoteRef(FootnoteRef { - ident: identifier, - label, - position: position.map(|p| p.clone().into()), - })] - } - mdast::Node::MdxFlowExpression(mdx) => { - vec![Self::MdxFlowExpression(MdxFlowExpression { - value: mdx.value.into(), - position: mdx.position.map(|position| position.into()), - })] - } - mdast::Node::MdxJsxFlowElement(mdx) => { - vec![Self::MdxJsxFlowElement(MdxJsxFlowElement { - children: mdx - .children - .into_iter() - .flat_map(Self::from_mdast_node) - .collect::>(), - position: mdx.position.map(|p| p.clone().into()), - name: mdx.name, - attributes: mdx - .attributes - .iter() - .map(|attr| match attr { - mdast::AttributeContent::Expression(mdast::MdxJsxExpressionAttribute { value, .. }) => { - MdxAttributeContent::Expression(value.into()) - } - mdast::AttributeContent::Property(mdast::MdxJsxAttribute { value, name, .. }) => { - MdxAttributeContent::Property(MdxJsxAttribute { - name: name.into(), - value: value.as_ref().map(|value| match value { - mdast::AttributeValue::Literal(value) => { - MdxAttributeValue::Literal(value.into()) - } - mdast::AttributeValue::Expression(mdast::AttributeValueExpression { - value, - .. - }) => MdxAttributeValue::Expression(value.into()), - }), - }) - } - }) - .collect(), - })] - } - mdast::Node::MdxJsxTextElement(mdx) => { - vec![Self::MdxJsxTextElement(MdxJsxTextElement { - children: mdx - .children - .into_iter() - .flat_map(Self::from_mdast_node) - .collect::>(), - position: mdx.position.map(|p| p.clone().into()), - name: mdx.name.map(|name| name.into()), - attributes: mdx - .attributes - .iter() - .map(|attr| match attr { - mdast::AttributeContent::Expression(mdast::MdxJsxExpressionAttribute { value, .. }) => { - MdxAttributeContent::Expression(value.into()) - } - mdast::AttributeContent::Property(mdast::MdxJsxAttribute { value, name, .. }) => { - MdxAttributeContent::Property(MdxJsxAttribute { - name: name.into(), - value: value.as_ref().map(|value| match value { - mdast::AttributeValue::Literal(value) => { - MdxAttributeValue::Literal(value.into()) - } - mdast::AttributeValue::Expression(mdast::AttributeValueExpression { - value, - .. - }) => MdxAttributeValue::Expression(value.into()), - }), - }) - } - }) - .collect(), - })] - } - mdast::Node::MdxTextExpression(mdx) => { - vec![Self::MdxTextExpression(MdxTextExpression { - value: mdx.value.into(), - position: mdx.position.map(|position| position.into()), - })] - } - mdast::Node::MdxjsEsm(mdx) => { - vec![Self::MdxJsEsm(MdxJsEsm { - value: mdx.value.into(), - position: mdx.position.map(|position| position.into()), - })] - } - mdast::Node::Text(mdast::Text { position, value, .. }) => { - vec![Self::Text(Text { - value, - position: position.map(|p| p.clone().into()), - })] - } - mdast::Node::Paragraph(mdast::Paragraph { children, .. }) => { - children.into_iter().flat_map(Self::from_mdast_node).collect::>() - } - _ => Vec::new(), - } - } - - /// Converts the children of `node`, moving them out to avoid deep-cloning the subtree at every level. - #[cfg(any(not(feature = "native-parser"), test))] - fn mdast_children_to_node(mut node: mdast::Node) -> Vec { - match node.children_mut() { - Some(children) => std::mem::take(children) - .into_iter() - .flat_map(Self::from_mdast_node) - .collect::>(), - None => vec![EMPTY_NODE], - } - } - - #[cfg(any(not(feature = "native-parser"), test))] - fn mdast_list_items(list: &mdast::List, level: Level) -> Vec { - let mut result = Vec::new(); - for node in &list.children { - match node { - mdast::Node::ListItem(list_item) => { - result.push(Self::mdast_list_item_node(list_item, list, level)); - for child in &list_item.children { - match child { - mdast::Node::List(sub_list) => { - result.extend(Self::mdast_list_items(sub_list, level + 1)); - } - mdast::Node::ListItem(child_item) => { - result.push(Self::mdast_list_item_node(child_item, list, level + 1)); - } - _ => {} - } - } - } - mdast::Node::List(sub_list) => result.extend(Self::mdast_list_items(sub_list, level + 1)), - _ => {} - } - } - - result - .into_iter() - .scan(0usize, |next_index, node| { - // Only renumber this call's own level; deeper items flattened in from a - // recursive call already have their own, independently-numbered index. - Some(match node { - Self::List(mut l) if l.level == level => { - l.index = *next_index; - *next_index += 1; - Self::List(l) - } - other => other, - }) - }) - .collect() - } - - /// Converts one list item without recursively converting its nested list children. - /// - /// The enclosing [`Self::mdast_list_items`] call emits those children at their own level, - /// so skipping them here prevents constructing and discarding the same subtree first. - #[cfg(any(not(feature = "native-parser"), test))] - fn mdast_list_item_node(list_item: &mdast::ListItem, list: &mdast::List, level: Level) -> Node { - let values = list_item - .children - .iter() - .filter(|node| !matches!(node, mdast::Node::List(_))) - .flat_map(|node| Self::from_mdast_node(node.clone())) - .collect::>(); - let position = if values.is_empty() { - list_item.position.as_ref().map(|position| position.clone().into()) - } else { - let first_pos = values.first().and_then(Node::position); - let last_pos = values.last().and_then(Node::position); - match (first_pos, last_pos) { - (Some(start), Some(end)) => Some(Position { - start: start.start.clone(), - end: end.end.clone(), - }), - _ => list_item.position.as_ref().map(|position| position.clone().into()), - } - }; - - Self::List(List { - level, - index: 0, - ordered: list.ordered, - checked: list_item.checked, - start: list.start, - spread: list.spread, - values, - position, - }) - } - fn mdx_attribute_content_to_string(attr: &MdxAttributeContent) -> SmolStr { match attr { MdxAttributeContent::Expression(value) => format!("{{{}}}", value).into(), diff --git a/crates/mq-markdown/src/parser.rs b/crates/mq-markdown/src/parser.rs index e174880e8..ef43bf1dd 100644 --- a/crates/mq-markdown/src/parser.rs +++ b/crates/mq-markdown/src/parser.rs @@ -4,19 +4,29 @@ //! [`resolve`] collects the definitions and turns the tree into nodes, and [`inline`] parses the raw //! text of paragraphs, headings and table cells along the way. //! -//! The output, positions included, is the same as that of `markdown-rs` for CommonMark, GFM, frontmatter, -//! math and MDX (without a JavaScript parser, so expressions only need balanced braces). This holds for -//! every example of the CommonMark and GFM specifications, and for randomly generated documents, except: +//! The output, positions included, is the one that `markdown-rs`, which this parser replaced, gave for +//! CommonMark, GFM, frontmatter, math and MDX (without a JavaScript parser, so expressions only need +//! balanced braces). HTML is rendered by [`render_html`] in the way `markdown-rs` did. Where the two +//! differ, this parser follows the specifications: +//! +//! - a byte order mark at the start is not content +//! - a table has no body rows that end it for starting with `*`, `#` or a backtick, a table of one +//! column needs no pipe, and its last cell takes the whitespace that follows it +//! - an info string loses its trailing whitespace, and a list is not loose for blank lines after it +//! - a tab that a container only partly consumes is not kept in text, the text starts where the tab ends +//! - `![^a](b)` is an image +//! - the end of text that has a tab before its line ending, which `markdown-rs` places beyond the text +//! +//! What `markdown-rs` did and this parser does not do: //! //! - lazy continuation lines after definitions, footnotes or thematic breaks in containers, in containers //! nested more than two deep, or with several container markers on a line, and unclosed fences in //! containers whose end depends on the line that follows -//! - documents that start with a `---` or `+++` line that never closes, and inputs on which -//! `markdown-rs` panics -//! - a few tab quirks of `markdown-rs`: a tab that a container only partly consumes keeps its rest in -//! text values +//! - documents that start with a `---` or `+++` line that never closes //! - character references in an image destination that contains an email address -//! - MDX text expressions and tags that span lines interrupted by container markers +//! - MDX text expressions and tags that span lines interrupted by container markers, and tabs in them +//! - an image in the alt text of an image, and the tabs that a container consumes in part inside of +//! code, in HTML mod block; mod code; mod definition; @@ -25,6 +35,7 @@ mod inline; mod line; mod mdx; mod mdx_flow; +mod render_html; mod resolve; mod table; mod tree; @@ -36,6 +47,11 @@ pub(crate) fn parse(content: &str) -> miette::Result> { resolve::resolve(block::parse(content, false), false).map_err(|message| miette::miette!(message)) } +/// Renders `content` as HTML. +pub(crate) fn to_html(content: &str) -> String { + render_html::render(content) +} + /// Parses `content` as MDX: no indented code, HTML, autolinks or GFM, but expressions and JSX. pub(crate) fn parse_mdx(content: &str) -> miette::Result> { resolve::resolve(block::parse(content, true), true).map_err(|message| miette::miette!(message)) @@ -44,432 +60,540 @@ pub(crate) fn parse_mdx(content: &str) -> miette::Result> { #[cfg(test)] mod tests { use super::*; - use crate::markdown::{parse_mdx_with_markdown_rs, parse_with_markdown_rs}; use proptest::prelude::*; use proptest::test_runner::{RngAlgorithm, TestRng, TestRunner}; - use rstest::rstest; - /// `Node`'s `PartialEq` compares rendered output, so compare `Debug` output to include positions. - fn assert_same_as_markdown_rs(input: &str) { - let expected = parse_with_markdown_rs(input).unwrap(); - let actual = parse(input).unwrap(); - assert_eq!(format!("{actual:#?}"), format!("{expected:#?}"), "input: {input:?}"); + const MDX_CASES: &[(&str, &str)] = &[ + ("jsx_self", ""), + ("jsx_attrs", ""), + ("jsx_member", ""), + ("jsx_namespace", ""), + ("jsx_fragment", "<>"), + ("jsx_text_pair", "x"), + ("jsx_flow_pair", "\n\nx\n\n"), + ("jsx_flow_indented_child", "\n x\n"), + ("jsx_inline", "a c d"), + ("jsx_inline_self", "a d"), + ("expr_flow", "{a}"), + ("expr_nested", "{a {b} c}"), + ("expr_multiline", "{a\nb}"), + ("expr_multiline_indented", "{a\n b\n c}"), + ("expr_text", "x {a} y"), + ("expr_then_tag", "{a} "), + ("tag_then_expr", " {a}"), + ("tag_then_text", " x"), + ("expr_then_text", "{a}x"), + ("jsx_in_expr_child", "{b}"), + ("jsx_unclosed_text", "b"), + ("jsx_mismatch", ""), + ("lt_space", "a < b"), + ("lt_digit", "a <3"), + ("attr_no_value", ""), + ("jsx_nested_flow", "\n\n\n"), + ("jsx_in_quote", "> \n> x\n> "), + ("jsx_in_list", "- \n x\n "), + ("attr_quote_inside", ""), + ("two_tags", ""), + ("two_text_elements", "xy"), + ("interrupt_paragraph_tag", "a\n"), + ("interrupt_paragraph_expr", "a\n{b}"), + ("two_expressions", "{a}\n{b}"), + ("expr_in_flow_element", "\n\n{b}\n\n"), + ("indented_tag", " "), + ("very_indented_tag", " "), + ("heading_jsx", "# h b"), + ("emphasis_crossing", "*x*"), + ("emphasis_crossing_bad", "*x*"), + ("link_jsx", "[x](y)"), + ("code_not_jsx", "`` {b}"), + ("escaped", "\\ \\{b}"), + ("attr_entities", ""), + ("attr_multiline_literal", ""), + ("closing_spaces", "x"), + ("lt_space_name", "< a>"), + ("self_closing_space", ""), + ("attr_on_next_line", ""), + ("dashed_name", ""), + ("dashed_attr", ""), + ("bad_name_char", ""), + ("expr_unclosed", "{a"), + ("expr_empty", "{}"), + ("expr_space", "{ }"), + ("empty_flow_element", "\n"), + ("text_element_unclosed_line", "x\n"), + ("stray_close", ""), + ("stray_close_text", "x "), + ("element_with_emphasis", "*b*"), + ("flow_element_with_emphasis", "\n*b*\n"), + ("flow_element_with_list", "\n- b\n"), + ("list_items_unbalanced", "- \n- "), + ("spaced_equals", ""), + ("heading", "# h\n\ntext *a* [b](c) `d`\n"), + ("indented_is_text", " not code\n"), + ("indented_heading", " # h\n"), + ("no_autolink", ""), + ("no_gfm", "| a |\n|-|\n\n~a~ www.a.b [^a]\n\n- [ ] a\n"), + ("no_math", "$a$\n\n$$\na\n$$\n"), + ("no_frontmatter", "---\na\n---\n"), + ("esm_is_text", "import a from 'b'\n\nexport const c = 1\n"), + ("definition", "[a]: /u\n\n[a]\n"), + ("list_deep_indent", "- a\n"), + ("fence", "```rust\ncode\n```\n"), + ("lone_cr_after_lt", "<\r/-x"), + ("crlf_after_lt", "<\r\n/-x"), + ("expression_lone_cr", "{\r }"), + ("expression_cr_indent", "{a\r b\r\n c\n d}"), + ("tilde_not_delimiter", "*~*a"), + ("tilde_strong_not_delimiter", "**~**あ"), + ("dollar_after_name", ""), + ("dollar_after_space", ""), + ("cjk_tag_name", "<あ b=\"c\" />"), + ("fullwidth_underscore_name", ""), + ("jsx_text_cjk", "あ い う"), + ("expression_cjk", "{あ} い"), + ]; + + const CASES: &[(&str, &str)] = &[ + ("empty", ""), + ("blank_lines", "\n\n \n"), + ("paragraph", "hello\n"), + ("paragraph_no_eol", "hello"), + ("paragraph_multiline", "a\nb\nc\n"), + ("paragraph_indented_continuation", "a\n b\n"), + ("paragraph_trailing_space", "あい\n うえ \n"), + ("two_paragraphs", "a\n\nb\n"), + ("crlf", "a\r\nb\r\n\r\nc\r\n"), + ("crlf_fence", "```\r\na\r\nb\r\n```\r\n"), + ("cr_only", "a\rb\r"), + ("atx_h1", "# title\n"), + ("atx_h6", "###### title\n"), + ("atx_seven", "####### title\n"), + ("atx_no_space", "#title\n"), + ("atx_empty", "#\n"), + ("atx_closing", "## title ##\n"), + ("atx_closing_no_space", "## title##\n"), + ("atx_indent", " ## title ## \n"), + ("atx_multibyte", "# あ h #\n"), + ("atx_interrupts_paragraph", "a\n# b\n"), + ("setext_h1", "Title\n===\n"), + ("setext_h2", "a\n ---\n"), + ("setext_multiline", "a\nb\n---\n"), + ("thematic_star", "***\n"), + ("thematic_dash_spaced", "- - -\n"), + ("thematic_underscore", " ___ \n"), + ("thematic_two", "**\n"), + ("fence_lang_meta", "```rust title\nlet a;\n```\n"), + ("fence_indented", " ```\n a\n b\n ```\n"), + ("fence_tilde", "~~~\na\n~~~\n"), + ("fence_unclosed", "```\na\n"), + ("fence_empty", "para\n\n\n```\n```\n"), + ("fence_longer_close", "```\na\n`````\n"), + ("fence_shorter_close", "````\na\n```\nb\n````\n"), + ("fence_blank_inside", "```\na\n\nb\n```\n"), + ("fence_backtick_info", "``` a`b\nc\n"), + ("fence_interrupts_paragraph", "a\n```\nb\n```\n"), + ("indented_code", " code\n\n more\n\nx\n"), + ("quote", "> a\n> b\n"), + ("quote_indented_lazy", " > a\nb\n"), + ("quote_heading", "> # h\n>\n> c\n"), + ("quote_no_space", ">a\n"), + ("quote_empty", ">\n"), + ("quote_nested", "> > a\n> b\n"), + ("quote_lazy_nested", "> > a\nb\n"), + ("quote_blank_ends", "> a\n\n> b\n"), + ("quote_fence", "> ```\n> a\n> ```\n"), + ("quote_no_lazy_after_fence", "> ```\n> a\nb\n"), + ("quote_interrupts_paragraph", "a\n> b\n"), + ("quote_thematic_not_lazy", "> a\n---\n"), + ("quote_multibyte", "> あ\n> い\n"), + ("bullet", "- a\n- b\n"), + ("bullet_loose", "- a\n\n- b\n"), + ("bullet_indent", " - a\n"), + ("bullet_mixed_markers", "* a\n+ b\n"), + ("ordered", "1. a\n2. b\n"), + ("ordered_paren", "5) a\n"), + ("ordered_zero", "0. a\n"), + ("ordered_too_long", "1234567890. a\n"), + ("ordered_mixed_delimiters", "1. a\n2) b\n"), + ("task", "- [ ] x\n- [x] y\n- [X] z\n"), + ("task_empty", "- [ ]\n"), + ("task_no_space", "- [ ]x\n"), + ("nested", "- a\n - b\n c\n"), + ("nested_ordered", "1. a\n 1. b\n 2. c\n2. d\n"), + ("nested_dedent", "- a\n - b\n- c\n"), + ("two_paragraphs", "- a\n\n b\n"), + ("two_paragraphs_then_item", "- a\n\n b\n- c\n"), + ("empty_item", "-\n"), + ("empty_item_content", "-\n foo\n"), + ("empty_item_blank_then_text", "-\n\n foo\n"), + ("empty_items", "-\n-\n"), + ("wide_marker_gap", "- a\n b\n"), + ("code_in_item", "- a\n"), + ("lazy", "- a\nb\n"), + ("lazy_after_blank", "- a\n\nb\n"), + ("list_interrupts_paragraph", "a\n- b\n"), + ("ordered_two_no_interrupt", "a\n2. b\n"), + ("ordered_one_interrupts", "a\n1. b\n"), + ("empty_item_no_interrupt", "a\n-\n"), + ("thematic_over_list", "- - -\n"), + ("list_in_quote", "> - a\n> - b\n"), + ("quote_in_list", "- > a\n > b\n"), + ("quote_in_list_level", "- a\n > - b\n"), + ("heading_in_item", "- # h\n text\n"), + ("fence_in_item", "- ```\n a\n ```\n"), + ("multibyte_item", "- あ\n- い\n"), + ("list_after_paragraph_blank", "a\n\n- b\n"), + ("two_lists_split", "- a\n\n\n- b\n"), + ("deep_quote", ">>>>>> a\n"), + ("fence_unclosed_no_eol", "```\na"), + ("fence_unclosed_in_item", "- ```\n a\n"), + ("fence_unclosed_in_item_no_eol", "- ```\n a"), + ("fence_unclosed_in_quote_no_eol", "> ```\n> a"), + ("fence_unclosed_in_quote_eof", "> ```\n> a\n"), + ("fence_unclosed_trailing_blank", "```\na\n\n"), + ("fence_unclosed_item_then_para", "- ```\n a\nb\n"), + ("item_code_then_unindented_text", "- ===\n\n a\n==="), + ("quote_setext_dashes", "> a\n> ---\n> ---\n> a"), + ("table_basic", "| a | b |\n|---|:-:|\n| 1 | 2 |\n"), + ("table_no_edge_pipes", "a|b\n-|-\n1|2|3\n"), + ("table_short_row", "|a|b|\n|-|-|\n|1|\n"), + ("table_escaped_pipe", "| a \\| b | c |\n|--|--|\n"), + ("table_indented", " | a |\n |---|\n | b |\n"), + ("table_text_after", "| a |\n|---|\ntext after\n"), + ("table_in_quote", "> | a |\n> |---|\n> | b |\n"), + ("table_after_paragraph", "x\n| a |\n|---|\n"), + ("table_column_mismatch", "| a | b |\n|---|\n"), + ("table_bad_delimiter", "|a|\n|:|\n"), + ("table_then_blank_para", "|a|\n|-|\n\nnext\n"), + ("table_then_quote", "|a|\n|-|\n> q\n"), + ("table_then_heading", "|a|\n|-|\n# h\n"), + ("table_then_fence", "|a|\n|-|\n```\nx\n```\n"), + ("table_then_list", "|a|\n|-|\n- x\n"), + ("table_two", "|a|\n|-|\n\n|b|\n|-|\n"), + ("table_align_spaces", "|a|b|\n|:-|-:|\n| x | y|\n"), + ("table_indented_code_row", "|a|\n|-|\n|b|\n |c|\n"), + ("table_no_body", "| a |\n| - |\n"), + ("table_empty_cells", "||\n|-|\n"), + ("table_lone_pipe_row", "|a|\n|-|\n|\n"), + ("table_no_trailing_pipe", "| a | b\n|-|-\n| c | d\n"), + ("table_multibyte", "| あ | い |\n|---|---|\n| う | え |\n"), + ("not_table_setext", "Title\n---\n"), + ("not_table_no_pipe_delim", "a|b\n---\n"), + ("table_header_no_pipe", "a\n|-|\n"), + ("table_one_col_dash", "|a|\n-\n"), + ("inline_escape", "a\\*b"), + ("inline_escape_nonpunct", "a\\qb"), + ("inline_entities", "a&b © # A &unknown; &"), + ("inline_emphasis_all", "a *b* **c** _d_ __e__ ***f***"), + ("inline_code", "`code` `` a`b `` ` a `"), + ("inline_code_unclosed", "`a ``b`"), + ("inline_hard_break_spaces", "a \nb"), + ("inline_hard_break_backslash", "a\\\nb"), + ("inline_soft_break", "a\nb"), + ("inline_soft_break_trailing_space", "a \nb"), + ("inline_emphasis_multiline", "*a\nb*"), + ("inline_strike", "a ~b~ ~~c~~ ~~~d~~~"), + ("inline_math", "$x$ $$y$$ $ a"), + ("inline_autolinks", "
"), + ("inline_links", "[a](http://x \"t\") [b][c] [d][] [e] ![f](g) ![h][i]"), + ( + "inline_link_forms", + "[a]( 'x') [d]( e ) [f](g (h)) [i](j \"k\\\"l\") [m](n&o)", + ), + ("inline_link_multiline", "[a](b\n\"t\") [c](d"), + ("inline_image_alt", "![a *b* `c`](d) ![e ![f](g)](h)"), + ("inline_link_in_link", "[a [b](c) d](e)"), + ("inline_emphasis_link", "*[a](b)* **[c][a]**"), + ("inline_html", "
t "), + ( + "inline_html_misc", + " ", + ), + ("inline_nested_emphasis", "*a **b** c* **a *b* c**"), + ("inline_intraword", "a_b_c a*b*c _a_b"), + ("inline_unmatched", "*a **b _c ~d [e ![f"), + ("inline_rule_of_three", "*foo**bar**baz* ***a** b*"), + ("inline_heading", "# a *b* `c`\n"), + ("inline_setext_heading", "a *b*\nc\n===\n"), + ("inline_list_item", "- a *b*\n c **d**\n"), + ("inline_table_cell", "| a *b* | `c\\|d` |\n|-|-|\n"), + ("inline_multibyte", "あ*い*う `え` [お](か)"), + ("inline_crlf_break", "a \r\nb\r\n"), + ("math_multiline", "x$\nx$"), + ("decl_email", "x"), + ("code_multiline", "x`\nx`"), + ("def_basic", "[a]: b"), + ("def_title", "[a]: 'T'"), + ("def_next_lines", "[a]:\nb\n\"t\""), + ("def_title_junk_next_line", "[a]: b\n\"t\" x"), + ("def_title_junk_same_line", "[a]: b \"t\" x"), + ("def_then_paragraph", "[a]: b\nc"), + ("def_two_then_text", "[a]: b\n[c]: d\ntext"), + ("def_not_at_start", "text\n[a]: b"), + ("def_then_setext_equals", "[a]: b\n=== "), + ("def_then_setext_dashes", "[a]: b\n---"), + ("def_indent1", " [a]: b"), + ("def_indent3", " [a]: b"), + ("def_indent4", " [a]: b"), + ("def_in_quote", "> [a]: b"), + ("def_in_list", "- [a]: b"), + ("def_no_dest", "[a]:"), + ("def_empty_label", "[]: b"), + ("def_blank_label", "[ ]: b"), + ("def_escaped_label", "[a\\]b]: c"), + ("def_multiline_title", "[a]: b \"t\nt\""), + ("def_title_indented", "[a]: b\n \"t\""), + ("def_no_space", "[a]:b"), + ("def_dest_junk", "[a]: b c"), + ("def_duplicate_labels", "[A b]: c\n[a b]: d"), + ("def_empty_angle", "[a]: <>"), + ("def_quote_in_dest", "[a]: 'x"), + ("def_use_shortcut", "[a]: /u\n\n[a] [a][] [b][a] [a][b] [A b]"), + ("def_use_normalized", "[ab cd]: /u\n\n[AB CD] [ab\ncd]"), + ("def_use_before", "[a] and [b][a]\n\n[a]: /u \"T\""), + ("def_use_in_link_text", "[a]: b\n\n[a](x) [a]"), + ("def_use_image", "[x]: /u\n\n![x] ![y][x] ![x][]"), + ("def_link_in_emphasis", "[a]: b\n\n*[a]* **[c][a]**"), + ("fn_basic", "[^a]: b"), + ("fn_lazy", "[^a]: b\nc"), + ("fn_indented_continuation", "[^a]: b\n c"), + ("fn_two_paragraphs", "[^a]: b\n\n c"), + ("fn_two_paragraphs_then_text", "[^a]: b\n\n c\n\nd"), + ("fn_two_space_continuation", "[^a]: b\n c"), + ("fn_empty", "[^a]:"), + ("fn_empty_space", "[^a]: "), + ("fn_content_next_line", "[^a]:\n b"), + ("fn_two_adjacent", "[^a]: b\n[^c]: d"), + ("fn_two_separated", "[^a]: b\n\n[^c]: d"), + ("fn_label_space", "[^ a]: b"), + ("fn_label_inner_space", "[^a b]: b"), + ("fn_label_empty", "[^]: b"), + ("fn_indent1", " [^a]: b"), + ("fn_indent4", " [^a]: b"), + ("fn_in_quote", "> [^a]: b"), + ("fn_in_list", "- [^a]: b"), + ("fn_then_quote", "[^a]: b\n> q"), + ("fn_then_list", "[^a]: b\n- x"), + ("fn_heading", "[^a]: # h"), + ("fn_list", "[^a]: - x\n - y"), + ("fn_fence", "[^a]: b\n ```\n x\n ```"), + ("fn_ref_case", "[^A]: b\n\nx[^a]"), + ("fn_no_space", "[^a]:b"), + ("fn_interrupts_paragraph", "text\n[^a]: b"), + ("fn_then_dashes", "[^a]: b\n---"), + ("fn_then_equals", "[^a]: b\n==="), + ("fn_ref_basic", "[^a]\n\n[^a]: note"), + ("fn_ref_no_def", "[^b] no def"), + ("fn_ref_inline", "x[^a] and [^a]!\n\n[^a]: note"), + ("fn_ref_in_link_text", "[a[^a]](b)\n\n[^a]: note"), + ("html_div", "
\nx\n
\n\ny"), + ("html_indented", "
\n\n\n
"), + ("html_img_self_close", "\nz"), + ("html_img_then_text", " y"), + ("html_multiline_tag_inline", "\nz"), + ("html_indent4", "
"), + ("html_comment_then_text", "y\nz"), + ("html_then_code_line", "
\n code"), + ("html_comment_short", "\nx"), + ("html_instruction_short", "\nx"), + ("html_lazy_quote", ">
\nx"), + ("html_attr_forms", "\nx"), + ("html_bad_attr", "\nx"), + ("fm_yaml", "---\na: b\n---\ntext"), + ("fm_yaml_only", "---\na: b\n---"), + ("fm_empty", "---\n---"), + ("fm_unclosed", "---\na\n"), + ("fm_blank_content", "---\n\n---\nx"), + ("fm_toml", "+++\na = 1\n+++\nx"), + ("fm_indented", " ---\na\n---"), + ("fm_close_trailing_space", "---\na\n--- \nx"), + ("fm_close_longer", "---\na\n----\nx"), + ("fm_open_trailing_space", "--- \na\n---"), + ("fm_open_junk", "---x\na\n---"), + ("fm_not_first", "text\n---\na\n---"), + ("fm_blank_before", "\n---\na\n---"), + ("fm_multiline_content", "---\na: 1\n\nb: 2\n---\n# h"), + ("fm_hr_after", "---\na\n---\n\n---"), + ("math_basic", "$$\na\n$$"), + ("math_then_text", "$$\na\n$$\nx"), + ("math_meta", "$$ meta\na\n$$"), + ("math_unclosed", "$$\na"), + ("math_longer_open", "$$$\na\n$$"), + ("math_longer_close", "$$\na\n$$$$"), + ("math_indented", " $$\n a\n b\n $$"), + ("math_empty", "$$\n$$"), + ("math_inline_not_block", "$$a$$"), + ("math_info_no_close", "$$a\n$$"), + ("math_in_quote", "> $$\n> a\n> $$"), + ("math_in_list", "- $$\n a\n $$"), + ("math_close_junk", "$$\na\n$$ x"), + ("math_interrupts_paragraph", "text\n$$\na\n$$"), + ("math_blank_inside", "$$\n\na\n$$"), + ("math_single_dollar_line", "$ $\nx"), + ("crlf_frontmatter", "---\r\na\r\n---\r\nx"), + ("break_end_indented", "a \n b"), + ("text_end_indented_continuation", "*a*\n "), + ("soft_end_before_node", "a\n `b`"), + ("break_in_quote", "> a \n> b"), + ("text_eol_in_footnote", "[^a]: x \n [a]"), + ("math_whitespace_content", " $$\n \n $$"), + ("math_whitespace_content2", " $$\n \n $$"), + ("table_indented_no_pipe", " a|b\n|-|:-:|"), + ("tab_nested_list", "- a\n\t- b\n\t\t- c\n"), + ("tab_nested_ordered", "1. a\n\t1. b\n\t\t1. c\n"), + ("tab_list_code", "- a\n\n\t\tcode\n"), + ("tab_in_text", "a\tb\t*c*\n"), + ("tab_code_block", "\tcode\n\t\tmore\n"), + ("tab_after_marker", "-\ta\n"), + ("atx_leading_sequences", "# # a"), + ("atx_leading_sequences2", "## # a"), + ("atx_only_sequences", "# # #"), + ("atx_no_space_sequence", "# #a"), + ("atx_double_sequence", "# ## a"), + ("atx_tab_sequence", "#\t# a"), + ("atx_closing_and_leading", "# # a #"), + ("atx_inner_hash", "# a # b"), + ("quote_footnote_trailing_space", "- a\n > [^a]: x\n "), + ("indented_code_trailing_blank", " code\n\n\n"), + ("bom_heading", "\u{feff}# a\n"), + ("bom_frontmatter", "\u{feff}---\na: b\n---\n"), + ("bom_text", "\u{feff}a\n"), + ("www_multibyte_after_ww", "wwß x"), + ("www_combining", "ww\u{301}.a.com"), + ("www_cjk", "www.あ.com/パス x"), + ("http_cjk", "http://あ.jp/パス ok"), + ("email_cjk_domain", "a@あ.com"), + ("email_cjk_local", "あ@b.com"), + ("html_inline_continuation", ""), + ("html_inline_continuation_tab", ""), + ("table_trailing_space", "| a | b | \n|-|-| \n| c | d | \n"), + ("table_no_pipes_trailing_space", "a | b\n--|--\nd | e \n"), + ("table_trailing_tab", "| a | b |\t\n|-|-|\n| c | d | \n"), + ("table_cjk", "| あ | 🎉 |\n|:-|-:|\n| é | ß |\n"), + ("table_single_column_colon", "a\n:-\nb\n"), + ("table_single_column_right", "a\n---:\nb\nc\n"), + ("table_double_colon_not_delimiter", "a\n::-\n"), + ("table_body_empty_marker", "a|b\n|-|-|\n*\n"), + ("table_body_ordered_marker", "a|b\n|-|-|\n2. x\n"), + ("table_body_empty_ordered", "a|b\n|-|-|\n1.\n"), + ("paragraph_after_partial_tab", "- a\n\n\t[P](h) x\n"), + ("fence_after_partial_tab", "- a\n\n\t```ts\n\t\tx\n\t```\n"), + ("math_after_partial_tab", "- a\n\n\t$$\n\tm\n\t$$\n"), + ("quote_after_partial_tab", "- a\n\n\t> q\n"), + ("heading_after_partial_tab", "- a\n\n\t# h\n"), + ( + "control_character_references", + "… Ÿ ���", + ), + ("footnote_escaped_bracket", "[^a\\]: x"), + ("footnote_escaped_backslash", "[^a\\\\]: x"), + ("footnote_escaped_open", "[^a\\[]: x"), + ("footnote_unescaped_open", "[^a[]: x"), + ("footnote_empty_then_blank", "[^a]: \n\n"), + ("footnote_empty_then_spaces", "[^a]: \n \n"), + ("footnote_empty_then_text", "[^a]: \n\nb"), + ("definition_nul_destination", "[a]: \0\n\n[a]\n"), + ("link_nul_destination", "[a](x\0y)"), + ("definition_then_empty_marker", "[a]: b\n-"), + ("definition_then_item", "[a]: b\n- x"), + ("definition_then_ordered", "[a]: b\n1."), + ("emphasis_cjk", "*あ*。*い*"), + ("strong_cjk", "**日本語**です"), + ("underscore_accent", "_é_ _ß_"), + ("fullwidth_markers", "*a* _b_"), + ("emphasis_after_cjk_punctuation", "。*a*、**b**"), + ("combining_mark", "e\u{301}*a*"), + ("emoji_zwj", "👨\u{200d}👩\u{200d}👧 *a*"), + ("fence_cjk_info", "```あ\n日本語\n```\n"), + ("fence_in_quote_emoji", "> ```🎉\n> x\n> ```\n"), + ("fence_in_list_cjk", "- ```あ\n 日本語\n ```\n"), + ("math_cjk", "$$\nあ\n$$\n\n$い$\n"), + ("heading_cjk", "# 見出し\n\n見出し2\n===\n"), + ("link_cjk", "[あ](http://い.jp/う \"え\")"), + ("reference_cjk", "[あ]\n\n[あ]: /u\n"), + ("footnote_cjk", "あ[^い]\n\n[^い]: う\n"), + ("ideographic_space", "a\u{3000}b\n\u{3000}c"), + ("nbsp_around_emphasis", "a\u{a0}*b*\u{a0}c"), + ("entity_cjk_context", "あ&いあ🎉"), + ("autolink_cjk", ""), + ("code_span_cjk", "`あ`と``い`う``"), + ("hard_break_cjk", "あ \nい\\\nう"), + ("nul_in_text", "a\0b"), + ]; + + /// One line with the nodes, positions written as `@line:column-line:column`. + fn dump(nodes: &[Node]) -> String { + format!("{nodes:?}") + .replace("position: Some(Position { start: Point { line: ", "@") + .replace(", column: ", ":") + .replace(" }, end: Point { line: ", "-") + .replace(" } })", "") } - /// Compares the MDX parsers. Invalid MDX has to fail in both. - fn assert_same_mdx(input: &str) { - let expected = parse_mdx_with_markdown_rs(input); - let actual = parse_mdx(input); - match (expected, actual) { - (Ok(expected), Ok(actual)) => { - assert_eq!(format!("{actual:#?}"), format!("{expected:#?}"), "input: {input:?}"); - } - (Err(_), Err(_)) => {} - (expected, actual) => panic!("input: {input:?}\n markdown-rs: {expected:?}\n native: {actual:?}"), + /// What the parser and the HTML renderer make of each case, to compare with `snapshots.txt`. + /// Run the tests with `UPDATE_SNAPSHOTS=1` to write it after a change that is meant to alter it. + fn snapshot() -> String { + let mut out = String::new(); + for (name, input) in CASES { + let nodes = dump(&parse(input).unwrap()); + out.push_str(&format!( + "## {name} {input:?}\nnodes: {nodes}\nhtml: {:?}\n", + to_html(input) + )); } + for (name, input) in MDX_CASES { + let nodes = match parse_mdx(input) { + Ok(nodes) => dump(&nodes), + Err(error) => format!("error: {error}"), + }; + out.push_str(&format!("## mdx {name} {input:?}\nnodes: {nodes}\n")); + } + out } - #[rstest] - #[case::jsx_self("")] - #[case::jsx_attrs("")] - #[case::jsx_member("")] - #[case::jsx_namespace("")] - #[case::jsx_fragment("<>")] - #[case::jsx_text_pair("x")] - #[case::jsx_flow_pair("\n\nx\n\n")] - #[case::jsx_flow_indented_child("\n x\n")] - #[case::jsx_inline("a c d")] - #[case::jsx_inline_self("a d")] - #[case::expr_flow("{a}")] - #[case::expr_nested("{a {b} c}")] - #[case::expr_multiline("{a\nb}")] - #[case::expr_multiline_indented("{a\n b\n c}")] - #[case::expr_text("x {a} y")] - #[case::expr_then_tag("{a} ")] - #[case::tag_then_expr(" {a}")] - #[case::tag_then_text(" x")] - #[case::expr_then_text("{a}x")] - #[case::jsx_in_expr_child("{b}")] - #[case::jsx_unclosed_text("b")] - #[case::jsx_mismatch("")] - #[case::lt_space("a < b")] - #[case::lt_digit("a <3")] - #[case::attr_no_value("")] - #[case::jsx_nested_flow("\n\n\n")] - #[case::jsx_in_quote("> \n> x\n> ")] - #[case::jsx_in_list("- \n x\n ")] - #[case::attr_quote_inside("")] - #[case::two_tags("")] - #[case::two_text_elements("xy")] - #[case::interrupt_paragraph_tag("a\n")] - #[case::interrupt_paragraph_expr("a\n{b}")] - #[case::two_expressions("{a}\n{b}")] - #[case::expr_in_flow_element("\n\n{b}\n\n")] - #[case::indented_tag(" ")] - #[case::very_indented_tag(" ")] - #[case::heading_jsx("# h b")] - #[case::emphasis_crossing("*x*")] - #[case::emphasis_crossing_bad("*x*")] - #[case::link_jsx("[x](y)")] - #[case::code_not_jsx("`` {b}")] - #[case::escaped("\\ \\{b}")] - #[case::attr_entities("")] - #[case::attr_multiline_literal("")] - #[case::closing_spaces("x")] - #[case::lt_space_name("< a>")] - #[case::self_closing_space("")] - #[case::attr_on_next_line("")] - #[case::dashed_name("")] - #[case::dashed_attr("")] - #[case::bad_name_char("")] - #[case::expr_unclosed("{a")] - #[case::expr_empty("{}")] - #[case::expr_space("{ }")] - #[case::empty_flow_element("\n")] - #[case::text_element_unclosed_line("x\n")] - #[case::stray_close("")] - #[case::stray_close_text("x ")] - #[case::element_with_emphasis("*b*")] - #[case::flow_element_with_emphasis("\n*b*\n")] - #[case::flow_element_with_list("\n- b\n")] - #[case::list_items_unbalanced("- \n- ")] - #[case::spaced_equals("")] - #[case::heading("# h\n\ntext *a* [b](c) `d`\n")] - #[case::indented_is_text(" not code\n")] - #[case::indented_heading(" # h\n")] - #[case::no_autolink("")] - #[case::no_gfm("| a |\n|-|\n\n~a~ www.a.b [^a]\n\n- [ ] a\n")] - #[case::no_math("$a$\n\n$$\na\n$$\n")] - #[case::no_frontmatter("---\na\n---\n")] - #[case::esm_is_text("import a from 'b'\n\nexport const c = 1\n")] - #[case::definition("[a]: /u\n\n[a]\n")] - #[case::list_deep_indent("- a\n")] - #[case::fence("```rust\ncode\n```\n")] - fn mdx_matches_markdown_rs(#[case] input: &str) { - assert_same_mdx(input); + #[test] + fn snapshots_match() { + let actual = snapshot(); + if std::env::var("UPDATE_SNAPSHOTS").is_ok() { + std::fs::write( + concat!(env!("CARGO_MANIFEST_DIR"), "/src/parser/snapshots.txt"), + &actual, + ) + .unwrap(); + return; + } + let expected = include_str!("parser/snapshots.txt"); + for (actual, expected) in actual.split("## ").zip(expected.split("## ")) { + assert_eq!(actual, expected, "snapshot differs, UPDATE_SNAPSHOTS=1 rewrites it"); + } + assert_eq!(actual.len(), expected.len(), "number of snapshots differs"); } - #[rstest] - #[case::empty("")] - #[case::blank_lines("\n\n \n")] - #[case::paragraph("hello\n")] - #[case::paragraph_no_eol("hello")] - #[case::paragraph_multiline("a\nb\nc\n")] - #[case::paragraph_indented_continuation("a\n b\n")] - #[case::paragraph_trailing_space("あい\n うえ \n")] - #[case::two_paragraphs("a\n\nb\n")] - #[case::crlf("a\r\nb\r\n\r\nc\r\n")] - #[case::crlf_fence("```\r\na\r\nb\r\n```\r\n")] - #[case::cr_only("a\rb\r")] - #[case::atx_h1("# title\n")] - #[case::atx_h6("###### title\n")] - #[case::atx_seven("####### title\n")] - #[case::atx_no_space("#title\n")] - #[case::atx_empty("#\n")] - #[case::atx_closing("## title ##\n")] - #[case::atx_closing_no_space("## title##\n")] - #[case::atx_indent(" ## title ## \n")] - #[case::atx_multibyte("# あ h #\n")] - #[case::atx_interrupts_paragraph("a\n# b\n")] - #[case::setext_h1("Title\n===\n")] - #[case::setext_h2("a\n ---\n")] - #[case::setext_multiline("a\nb\n---\n")] - #[case::thematic_star("***\n")] - #[case::thematic_dash_spaced("- - -\n")] - #[case::thematic_underscore(" ___ \n")] - #[case::thematic_two("**\n")] - #[case::fence_lang_meta("```rust title\nlet a;\n```\n")] - #[case::fence_indented(" ```\n a\n b\n ```\n")] - #[case::fence_tilde("~~~\na\n~~~\n")] - #[case::fence_unclosed("```\na\n")] - #[case::fence_empty("para\n\n\n```\n```\n")] - #[case::fence_longer_close("```\na\n`````\n")] - #[case::fence_shorter_close("````\na\n```\nb\n````\n")] - #[case::fence_blank_inside("```\na\n\nb\n```\n")] - #[case::fence_backtick_info("``` a`b\nc\n")] - #[case::fence_interrupts_paragraph("a\n```\nb\n```\n")] - #[case::indented_code(" code\n\n more\n\nx\n")] - #[case::quote("> a\n> b\n")] - #[case::quote_indented_lazy(" > a\nb\n")] - #[case::quote_heading("> # h\n>\n> c\n")] - #[case::quote_no_space(">a\n")] - #[case::quote_empty(">\n")] - #[case::quote_nested("> > a\n> b\n")] - #[case::quote_lazy_nested("> > a\nb\n")] - #[case::quote_blank_ends("> a\n\n> b\n")] - #[case::quote_fence("> ```\n> a\n> ```\n")] - #[case::quote_no_lazy_after_fence("> ```\n> a\nb\n")] - #[case::quote_interrupts_paragraph("a\n> b\n")] - #[case::quote_thematic_not_lazy("> a\n---\n")] - #[case::quote_multibyte("> あ\n> い\n")] - #[case::bullet("- a\n- b\n")] - #[case::bullet_loose("- a\n\n- b\n")] - #[case::bullet_indent(" - a\n")] - #[case::bullet_mixed_markers("* a\n+ b\n")] - #[case::ordered("1. a\n2. b\n")] - #[case::ordered_paren("5) a\n")] - #[case::ordered_zero("0. a\n")] - #[case::ordered_too_long("1234567890. a\n")] - #[case::ordered_mixed_delimiters("1. a\n2) b\n")] - #[case::task("- [ ] x\n- [x] y\n- [X] z\n")] - #[case::task_empty("- [ ]\n")] - #[case::task_no_space("- [ ]x\n")] - #[case::nested("- a\n - b\n c\n")] - #[case::nested_ordered("1. a\n 1. b\n 2. c\n2. d\n")] - #[case::nested_dedent("- a\n - b\n- c\n")] - #[case::two_paragraphs("- a\n\n b\n")] - #[case::two_paragraphs_then_item("- a\n\n b\n- c\n")] - #[case::empty_item("-\n")] - #[case::empty_item_content("-\n foo\n")] - #[case::empty_item_blank_then_text("-\n\n foo\n")] - #[case::empty_items("-\n-\n")] - #[case::wide_marker_gap("- a\n b\n")] - #[case::code_in_item("- a\n")] - #[case::lazy("- a\nb\n")] - #[case::lazy_after_blank("- a\n\nb\n")] - #[case::list_interrupts_paragraph("a\n- b\n")] - #[case::ordered_two_no_interrupt("a\n2. b\n")] - #[case::ordered_one_interrupts("a\n1. b\n")] - #[case::empty_item_no_interrupt("a\n-\n")] - #[case::thematic_over_list("- - -\n")] - #[case::list_in_quote("> - a\n> - b\n")] - #[case::quote_in_list("- > a\n > b\n")] - #[case::quote_in_list_level("- a\n > - b\n")] - #[case::heading_in_item("- # h\n text\n")] - #[case::fence_in_item("- ```\n a\n ```\n")] - #[case::multibyte_item("- あ\n- い\n")] - #[case::list_after_paragraph_blank("a\n\n- b\n")] - #[case::two_lists_split("- a\n\n\n- b\n")] - #[case::deep_quote(">>>>>> a\n")] - #[case::fence_unclosed_no_eol("```\na")] - #[case::fence_unclosed_in_item("- ```\n a\n")] - #[case::fence_unclosed_in_item_no_eol("- ```\n a")] - #[case::fence_unclosed_in_quote_no_eol("> ```\n> a")] - #[case::fence_unclosed_in_quote_eof("> ```\n> a\n")] - #[case::fence_unclosed_trailing_blank("```\na\n\n")] - #[case::fence_unclosed_item_then_para("- ```\n a\nb\n")] - #[case::item_code_then_unindented_text("- ===\n\n a\n===")] - #[case::quote_setext_dashes("> a\n> ---\n> ---\n> a")] - #[case::table_basic("| a | b |\n|---|:-:|\n| 1 | 2 |\n")] - #[case::table_no_edge_pipes("a|b\n-|-\n1|2|3\n")] - #[case::table_short_row("|a|b|\n|-|-|\n|1|\n")] - #[case::table_escaped_pipe("| a \\| b | c |\n|--|--|\n")] - #[case::table_indented(" | a |\n |---|\n | b |\n")] - #[case::table_text_after("| a |\n|---|\ntext after\n")] - #[case::table_in_quote("> | a |\n> |---|\n> | b |\n")] - #[case::table_after_paragraph("x\n| a |\n|---|\n")] - #[case::table_column_mismatch("| a | b |\n|---|\n")] - #[case::table_bad_delimiter("|a|\n|:|\n")] - #[case::table_then_blank_para("|a|\n|-|\n\nnext\n")] - #[case::table_then_quote("|a|\n|-|\n> q\n")] - #[case::table_then_heading("|a|\n|-|\n# h\n")] - #[case::table_then_fence("|a|\n|-|\n```\nx\n```\n")] - #[case::table_then_list("|a|\n|-|\n- x\n")] - #[case::table_two("|a|\n|-|\n\n|b|\n|-|\n")] - #[case::table_align_spaces("|a|b|\n|:-|-:|\n| x | y|\n")] - #[case::table_indented_code_row("|a|\n|-|\n|b|\n |c|\n")] - #[case::table_no_body("| a |\n| - |\n")] - #[case::table_empty_cells("||\n|-|\n")] - #[case::table_lone_pipe_row("|a|\n|-|\n|\n")] - #[case::table_no_trailing_pipe("| a | b\n|-|-\n| c | d\n")] - #[case::table_multibyte("| あ | い |\n|---|---|\n| う | え |\n")] - #[case::not_table_setext("Title\n---\n")] - #[case::not_table_no_pipe_delim("a|b\n---\n")] - #[case::table_header_no_pipe("a\n|-|\n")] - #[case::table_one_col_dash("|a|\n-\n")] - #[case::inline_escape("a\\*b")] - #[case::inline_escape_nonpunct("a\\qb")] - #[case::inline_entities("a&b © # A &unknown; &")] - #[case::inline_emphasis_all("a *b* **c** _d_ __e__ ***f***")] - #[case::inline_code("`code` `` a`b `` ` a `")] - #[case::inline_code_unclosed("`a ``b`")] - #[case::inline_hard_break_spaces("a \nb")] - #[case::inline_hard_break_backslash("a\\\nb")] - #[case::inline_soft_break("a\nb")] - #[case::inline_soft_break_trailing_space("a \nb")] - #[case::inline_emphasis_multiline("*a\nb*")] - #[case::inline_strike("a ~b~ ~~c~~ ~~~d~~~")] - #[case::inline_math("$x$ $$y$$ $ a")] - #[case::inline_autolinks("
")] - #[case::inline_links("[a](http://x \"t\") [b][c] [d][] [e] ![f](g) ![h][i]")] - #[case::inline_link_forms("[a]( 'x') [d]( e ) [f](g (h)) [i](j \"k\\\"l\") [m](n&o)")] - #[case::inline_link_multiline("[a](b\n\"t\") [c](d")] - #[case::inline_image_alt("![a *b* `c`](d) ![e ![f](g)](h)")] - #[case::inline_link_in_link("[a [b](c) d](e)")] - #[case::inline_emphasis_link("*[a](b)* **[c][a]**")] - #[case::inline_html("
t ")] - #[case::inline_html_misc(" ")] - #[case::inline_nested_emphasis("*a **b** c* **a *b* c**")] - #[case::inline_intraword("a_b_c a*b*c _a_b")] - #[case::inline_unmatched("*a **b _c ~d [e ![f")] - #[case::inline_rule_of_three("*foo**bar**baz* ***a** b*")] - #[case::inline_heading("# a *b* `c`\n")] - #[case::inline_setext_heading("a *b*\nc\n===\n")] - #[case::inline_list_item("- a *b*\n c **d**\n")] - #[case::inline_table_cell("| a *b* | `c\\|d` |\n|-|-|\n")] - #[case::inline_multibyte("あ*い*う `え` [お](か)")] - #[case::inline_crlf_break("a \r\nb\r\n")] - #[case::math_multiline("x$\nx$")] - #[case::decl_email("x")] - #[case::code_multiline("x`\nx`")] - #[case::def_basic("[a]: b")] - #[case::def_title("[a]: 'T'")] - #[case::def_next_lines("[a]:\nb\n\"t\"")] - #[case::def_title_junk_next_line("[a]: b\n\"t\" x")] - #[case::def_title_junk_same_line("[a]: b \"t\" x")] - #[case::def_then_paragraph("[a]: b\nc")] - #[case::def_two_then_text("[a]: b\n[c]: d\ntext")] - #[case::def_not_at_start("text\n[a]: b")] - #[case::def_then_setext_equals("[a]: b\n=== ")] - #[case::def_then_setext_dashes("[a]: b\n---")] - #[case::def_indent1(" [a]: b")] - #[case::def_indent3(" [a]: b")] - #[case::def_indent4(" [a]: b")] - #[case::def_in_quote("> [a]: b")] - #[case::def_in_list("- [a]: b")] - #[case::def_no_dest("[a]:")] - #[case::def_empty_label("[]: b")] - #[case::def_blank_label("[ ]: b")] - #[case::def_escaped_label("[a\\]b]: c")] - #[case::def_multiline_title("[a]: b \"t\nt\"")] - #[case::def_title_indented("[a]: b\n \"t\"")] - #[case::def_no_space("[a]:b")] - #[case::def_dest_junk("[a]: b c")] - #[case::def_duplicate_labels("[A b]: c\n[a b]: d")] - #[case::def_empty_angle("[a]: <>")] - #[case::def_quote_in_dest("[a]: 'x")] - #[case::def_use_shortcut("[a]: /u\n\n[a] [a][] [b][a] [a][b] [A b]")] - #[case::def_use_normalized("[ab cd]: /u\n\n[AB CD] [ab\ncd]")] - #[case::def_use_before("[a] and [b][a]\n\n[a]: /u \"T\"")] - #[case::def_use_in_link_text("[a]: b\n\n[a](x) [a]")] - #[case::def_use_image("[x]: /u\n\n![x] ![y][x] ![x][]")] - #[case::def_link_in_emphasis("[a]: b\n\n*[a]* **[c][a]**")] - #[case::fn_basic("[^a]: b")] - #[case::fn_lazy("[^a]: b\nc")] - #[case::fn_indented_continuation("[^a]: b\n c")] - #[case::fn_two_paragraphs("[^a]: b\n\n c")] - #[case::fn_two_paragraphs_then_text("[^a]: b\n\n c\n\nd")] - #[case::fn_two_space_continuation("[^a]: b\n c")] - #[case::fn_empty("[^a]:")] - #[case::fn_empty_space("[^a]: ")] - #[case::fn_content_next_line("[^a]:\n b")] - #[case::fn_two_adjacent("[^a]: b\n[^c]: d")] - #[case::fn_two_separated("[^a]: b\n\n[^c]: d")] - #[case::fn_label_space("[^ a]: b")] - #[case::fn_label_inner_space("[^a b]: b")] - #[case::fn_label_empty("[^]: b")] - #[case::fn_indent1(" [^a]: b")] - #[case::fn_indent4(" [^a]: b")] - #[case::fn_in_quote("> [^a]: b")] - #[case::fn_in_list("- [^a]: b")] - #[case::fn_then_quote("[^a]: b\n> q")] - #[case::fn_then_list("[^a]: b\n- x")] - #[case::fn_heading("[^a]: # h")] - #[case::fn_list("[^a]: - x\n - y")] - #[case::fn_fence("[^a]: b\n ```\n x\n ```")] - #[case::fn_ref_case("[^A]: b\n\nx[^a]")] - #[case::fn_no_space("[^a]:b")] - #[case::fn_interrupts_paragraph("text\n[^a]: b")] - #[case::fn_then_dashes("[^a]: b\n---")] - #[case::fn_then_equals("[^a]: b\n===")] - #[case::fn_ref_basic("[^a]\n\n[^a]: note")] - #[case::fn_ref_no_def("[^b] no def")] - #[case::fn_ref_inline("x[^a] and [^a]!\n\n[^a]: note")] - #[case::fn_ref_in_link_text("[a[^a]](b)\n\n[^a]: note")] - #[case::html_div("
\nx\n
\n\ny")] - #[case::html_indented("
\n x\n")] - #[case::html_oneline("
x
")] - #[case::html_script("\nz")] - #[case::html_pre("
\na\n\nb
\nc")] - #[case::html_style_inline_end(" y\nz")] - #[case::html_comment("\nx")] - #[case::html_instruction("\ny")] - #[case::html_declaration("\nx")] - #[case::html_cdata("\ny")] - #[case::html_complete_tag("\ny")] - #[case::html_tag_then_text("y")] - #[case::html_interrupt_basic("text\n\nz")] - #[case::html_multiline_open("\nz")] - #[case::html_in_quote(">
\n> x")] - #[case::html_in_list("-
\n x")] - #[case::html_uppercase("
\nx")] - #[case::html_self_closing("
\nx")] - #[case::html_ins("\nx")] - #[case::html_br("
\nx")] - #[case::html_textarea("")] - #[case::html_table_two_blocks("\n\n\n
")] - #[case::html_img_self_close("\nz")] - #[case::html_img_then_text(" y")] - #[case::html_multiline_tag_inline("\nz")] - #[case::html_indent4("
")] - #[case::html_comment_then_text("y\nz")] - #[case::html_then_code_line("
\n code")] - #[case::html_comment_short("\nx")] - #[case::html_instruction_short("\nx")] - #[case::html_lazy_quote(">
\nx")] - #[case::html_attr_forms("\nx")] - #[case::html_bad_attr("\nx")] - #[case::fm_yaml("---\na: b\n---\ntext")] - #[case::fm_yaml_only("---\na: b\n---")] - #[case::fm_empty("---\n---")] - #[case::fm_unclosed("---\na\n")] - #[case::fm_blank_content("---\n\n---\nx")] - #[case::fm_toml("+++\na = 1\n+++\nx")] - #[case::fm_indented(" ---\na\n---")] - #[case::fm_close_trailing_space("---\na\n--- \nx")] - #[case::fm_close_longer("---\na\n----\nx")] - #[case::fm_open_trailing_space("--- \na\n---")] - #[case::fm_open_junk("---x\na\n---")] - #[case::fm_not_first("text\n---\na\n---")] - #[case::fm_blank_before("\n---\na\n---")] - #[case::fm_multiline_content("---\na: 1\n\nb: 2\n---\n# h")] - #[case::fm_hr_after("---\na\n---\n\n---")] - #[case::math_basic("$$\na\n$$")] - #[case::math_then_text("$$\na\n$$\nx")] - #[case::math_meta("$$ meta\na\n$$")] - #[case::math_unclosed("$$\na")] - #[case::math_longer_open("$$$\na\n$$")] - #[case::math_longer_close("$$\na\n$$$$")] - #[case::math_indented(" $$\n a\n b\n $$")] - #[case::math_empty("$$\n$$")] - #[case::math_inline_not_block("$$a$$")] - #[case::math_info_no_close("$$a\n$$")] - #[case::math_in_quote("> $$\n> a\n> $$")] - #[case::math_in_list("- $$\n a\n $$")] - #[case::math_close_junk("$$\na\n$$ x")] - #[case::math_interrupts_paragraph("text\n$$\na\n$$")] - #[case::math_blank_inside("$$\n\na\n$$")] - #[case::math_single_dollar_line("$ $\nx")] - #[case::crlf_frontmatter("---\r\na\r\n---\r\nx")] - #[case::break_end_indented("a \n b")] - #[case::text_end_indented_continuation("*a*\n ")] - #[case::soft_end_before_node("a\n `b`")] - #[case::break_in_quote("> a \n> b")] - #[case::text_eol_in_footnote("[^a]: x \n [a]")] - #[case::math_whitespace_content(" $$\n \n $$")] - #[case::math_whitespace_content2(" $$\n \n $$")] - #[case::table_indented_no_pipe(" a|b\n|-|:-:|")] - #[case::tab_nested_list("- a\n\t- b\n\t\t- c\n")] - #[case::tab_nested_ordered("1. a\n\t1. b\n\t\t1. c\n")] - #[case::tab_list_code("- a\n\n\t\tcode\n")] - #[case::tab_in_text("a\tb\t*c*\n")] - #[case::tab_code_block("\tcode\n\t\tmore\n")] - #[case::tab_after_marker("-\ta\n")] - #[case::atx_leading_sequences("# # a")] - #[case::atx_leading_sequences2("## # a")] - #[case::atx_only_sequences("# # #")] - #[case::atx_no_space_sequence("# #a")] - #[case::atx_double_sequence("# ## a")] - #[case::atx_tab_sequence("#\t# a")] - #[case::atx_closing_and_leading("# # a #")] - #[case::atx_inner_hash("# a # b")] - #[case::quote_footnote_trailing_space("- a\n > [^a]: x\n ")] - #[case::indented_code_trailing_blank(" code\n\n\n")] - fn matches_markdown_rs(#[case] input: &str) { - assert_same_as_markdown_rs(input); + fn env_number(name: &str) -> Option { + std::env::var(name).ok()?.parse().ok() } /// Lines built from container prefixes and block-level bodies only, so inline syntax (not @@ -529,51 +653,6 @@ mod tests { }) } - fn env_number(name: &str) -> Option { - std::env::var(name).ok()?.parse().ok() - } - - /// Runs `strategy` with a fixed seed so the outcome never varies between runs. `PROPTEST_CASES` - /// and `PARSER_TEST_SEED` widen the exploration, which finds more differences in the cases listed - /// on [`block_input`] that are not covered yet. - fn check(strategy: impl Strategy) { - check_with(strategy, parse_with_markdown_rs, assert_same_as_markdown_rs); - } - - fn check_with( - strategy: impl Strategy, - reference: fn(&str) -> miette::Result>, - assert_same: fn(&str) -> R, - ) { - let config = ProptestConfig { - cases: env_number("PROPTEST_CASES").unwrap_or(3000), - failure_persistence: None, - ..ProptestConfig::default() - }; - let seed = env_number("PARSER_TEST_SEED").unwrap_or(7); - let mut runner = TestRunner::new_with_rng(config, TestRng::from_seed(RngAlgorithm::ChaCha, &[seed; 32])); - let result = runner.run(&strategy, |input| { - // markdown-rs misparses what follows an opening `---` or `+++` that never closes, and even - // panics on some inputs. Those cannot be compared. - let mut lines = input.lines().map(str::trim_end); - if let Some(first @ ("---" | "+++")) = lines.next() { - prop_assume!(lines.any(|line| line == first)); - } - let outcome = std::panic::catch_unwind(|| reference(&input)); - prop_assume!(outcome.is_ok()); - assert_same(&input); - Ok(()) - }); - if let Err(error) = result { - panic!("{error}"); - } - } - - #[test] - fn block_structure_matches_markdown_rs() { - check(block_input()); - } - /// Lines of inline syntax. Each line starts with `x` so that no line is a block on its own. fn inline_input() -> impl Strategy { let token = prop::sample::select(vec![ @@ -691,21 +770,345 @@ mod tests { prop::collection::vec(token, 1..16).prop_map(|tokens| tokens.concat()) } + /// Markdown syntax mixed with multibyte text: CJK, emoji, combining marks, `ß`, full-width and + /// ideographic punctuation and spaces, and characters that change case or width. + fn unicode_input() -> impl Strategy { + let word = prop::sample::select(vec![ + "あ", + "日本語", + "テスト", + "🎉", + "👨‍👩‍👧", + "é", + "e\u{301}", + "ß", + "Dž", + "İ", + "fi", + "www", + "www.あ.com", + "wwß", + "ww\u{301}", + "http://あ.jp/パス", + "https://例え.jp", + "a@あ.com", + "あ@b.com", + "。", + "、", + "「", + "」", + "(", + ")", + "!", + "?", + "*", + "_", + "`", + "#", + "\u{3000}", + "\u{a0}", + "\u{200b}", + "\u{2028}", + "\u{feff}", + "\u{fe0f}", + "ا", + "א", + "𠮷", + "\u{10ffff}", + "\u{0}", + "x", + "foo", + ]); + let syntax = prop::sample::select(vec![ + " ", + " ", + "\t", + "*", + "**", + "_", + "__", + "~", + "~~", + "`", + "``", + "[", + "]", + "(", + ")", + "![", + "<", + ">", + "&", + "&", + "あ", + "🎉", + "&あ;", + "\\", + "|", + ":", + "-", + "#", + "^", + "\"", + "'", + "{", + "}", + "/", + "=", + "://", + ".", + "\n", + "\n\n", + "\n> ", + "\n- ", + "\n1. ", + "\n# ", + "\n| ", + "\n|-|-|\n", + "\n ", + "\n\t", + "\r\n", + "\n[a]: ", + "\n[^a]: ", + ]); + let token = prop_oneof![3 => word, 2 => syntax]; + prop::collection::vec(token, 1..24).prop_map(|tokens| tokens.concat()) + } + + /// Any characters at all, with extra weight on the ones that matter to Markdown. + fn arbitrary_input() -> impl Strategy { + let ch = prop_oneof![ + 4 => any::(), + 3 => prop::sample::select(vec!['*', '_', '`', '[', ']', '(', ')', '<', '>', '&', '|', '\\', '#', '-', '~', ' ', '\t', '\n', '!', 'w', 'h', '@', ':', '/', '.', '{', '}']), + 3 => prop::sample::select(vec!['あ', 'é', 'ß', '🎉', '\u{301}', '\u{3000}', '\u{a0}', '。', '*']), + ]; + prop::collection::vec(ch, 0..40).prop_map(|chars| chars.into_iter().collect()) + } + + /// Closed fences whose info string and content are multibyte, alone or inside a container. + fn unicode_fence_input() -> impl Strategy { + let text = prop::sample::select(vec![ + "あ", + "日本語 x", + "🎉", + "é", + "ß", + "www", + "a b", + "", + "> あ", + "- 🎉", + "\u{3000}", + ]); + ( + prop::sample::select(vec!["", "> ", "- ", " ", "1. "]), + prop::sample::select(vec![("```", "```"), ("~~~", "~~~"), ("````", "````"), ("$$", "$$")]), + text.clone(), + text.clone(), + text, + ) + .prop_map(|(container, (open, close), info, first, second)| { + let pad = if container == "> " { + "> ".to_string() + } else { + " ".repeat(container.len()) + }; + let lead = |line: &str, first: bool| { + if first { + format!("{container}{line}") + } else { + format!("{pad}{line}") + } + }; + [ + lead(&format!("{open}{info}"), true), + lead(first, false), + lead(second, false), + lead(close, false), + ] + .join("\n") + }) + } + + /// Checks what has to hold for any input: nothing panics, and every position lies inside the + /// document and does not end before it starts. + fn assert_valid(input: &str) { + let lines = 1 + input.matches(['\n', '\r']).count() - input.matches("\r\n").count(); + let mut stack = parse(input).expect("Markdown always parses"); + stack.extend(parse_mdx(input).unwrap_or_default()); + while let Some(node) = stack.pop() { + if let Some(position) = node.position() { + let (start, end) = (&position.start, &position.end); + let ordered = (start.line, start.column) <= (end.line, end.column); + assert!( + ordered && start.line >= 1 && start.column >= 1 && end.line <= lines, + "{node:?} has a position outside of {input:?}" + ); + } + stack.extend(node.children()); + } + let _ = to_html(input); + } + + /// Runs `strategy` with a fixed seed so the outcome never varies between runs. `PROPTEST_CASES` + /// and `PARSER_TEST_SEED` widen the exploration. + fn check(strategy: impl Strategy) { + let config = ProptestConfig { + cases: env_number("PROPTEST_CASES").unwrap_or(3000), + failure_persistence: None, + ..ProptestConfig::default() + }; + let seed = env_number("PARSER_TEST_SEED").unwrap_or(7); + let mut runner = TestRunner::new_with_rng(config, TestRng::from_seed(RngAlgorithm::ChaCha, &[seed; 32])); + let result = runner.run(&strategy, |input| { + assert_valid(&input); + Ok(()) + }); + if let Err(error) = result { + panic!("{error}"); + } + } + #[test] - fn mdx_syntax_matches_markdown_rs() { - check_with(mdx_input(), parse_mdx_with_markdown_rs, assert_same_mdx); + fn block_structure_is_valid() { + check(block_input()); } #[test] - fn inline_syntax_matches_markdown_rs() { + fn inline_syntax_is_valid() { check(inline_input()); } + #[test] + fn mdx_syntax_is_valid() { + check(mdx_input()); + } + + #[test] + fn multibyte_syntax_is_valid() { + check(unicode_input()); + } + + #[test] + fn multibyte_fences_are_valid() { + check(unicode_fence_input()); + } + + #[test] + fn arbitrary_text_is_valid() { + check(arbitrary_input()); + } + + /// Every control character and unusual space or separator at each syntax position. + #[test] + fn special_characters_are_valid() { + let chars = (0u32..=0x9f) + .chain([ + 0xa0, 0xad, 0x1680, 0x180e, 0x2000, 0x200a, 0x200b, 0x200d, 0x2028, 0x2029, 0x202f, 0x205f, 0x2060, + 0x3000, 0xfeff, 0xfffd, 0xfe0f, + ]) + .filter_map(char::from_u32); + let templates = [ + "[a]: {}", + "[a]: x{}y", + "[a]: <{}>", + "[a]: /u '{}'", + "[a]: /u {}", + "[a]\n\n[a]: {}x\n\n[a]", + "[a]({})", + "[a](x{}y)", + "[a](<{}>)", + "[a](/u '{}')", + "[a](/u \"{}\")", + "![{}](x)", + "[{}](x)", + "[x{}]: y", + "", + "", + "<{}a>", + "", + "", + "", + "", + "http://a{}b.c", + "www.a{}b.c", + "www.a.b/{}x", + "x@y{}z.com", + "{}x@y.com", + "a{}b", + "{}a", + "a{}", + "a {}", + "{} a", + "# {}", + "# a{}", + "#{}a", + "{}# a", + "> {}", + "- {}", + "-{}a", + "1.{}a", + "1. {}", + "```{}", + "```a{}\nx\n```", + "```\n{}\n```", + "~~~{}", + "$${}", + "$$\n{}\n$$", + "`{}`", + "`a{}b`", + "*{}*", + "*a{}*", + "{}*a*", + "*a*{}", + "**{}**", + "_a{}_", + "_{}a_", + "~~a{}~~", + "~~{}~~", + "a\\{}", + "&{};", + "&a{};", + "&#{};", + "&#x{};", + "&{}", + "{}\n---", + "a{}\n---", + "a\n{}---", + "---{}", + "***{}", + "- - {}", + "| a{} | b |\n|-|-|", + "| {} |\n|-|\n| {} |", + "|{}|\n|{}|", + "[^a]: {}", + "[^a{}]: x", + "[^a]\n\n[^a]: {}x", + "x\n\n {}", + "x\n\n a{}", + "", + "
{}", + "
", + "", + "", + "---\n{}\n---\na", + "+++\n{}\n+++\na", + "{} \nb", + "a{}\nb", + "a\n{}b", + ]; + for char in chars { + for template in templates { + assert_valid(&template.replace("{}", &char.to_string())); + } + } + } + const SPEC_URL: &str = "https://raw.githubusercontent.com/github/cmark-gfm/828322d1ee4facdab56f0d3edccb13e9af90dcd2/test/spec.txt"; - /// The markdown of each example of the `CommonMark` and GFM `spec.txt`, with its number. - fn spec_examples(text: &str) -> Vec<(usize, String)> { + /// The markdown and the HTML of each example of the `CommonMark` and GFM `spec.txt`, with its number. + fn spec_examples(text: &str) -> Vec<(usize, String, String)> { let fence = "`".repeat(32); let open = format!("{fence} example"); let mut examples = Vec::new(); @@ -716,24 +1119,35 @@ mod tests { continue; } let markdown = lines.by_ref().take_while(|line| *line != ".").collect::>(); - for line in lines.by_ref() { - if line == fence { - break; - } - } + let html = lines.by_ref().take_while(|line| *line != fence).collect::>(); // `→` stands for a tab in the spec. - examples.push((examples.len() + 1, markdown.join("\n").replace('\u{2192}', "\t") + "\n")); + let text = |lines: Vec<&str>| lines.join("\n").replace('\u{2192}', "\t"); + let html = text(html); + let html = if html.is_empty() { html } else { html + "\n" }; + examples.push((examples.len() + 1, text(markdown) + "\n", html)); } examples } - /// Examples of the spec on which the native parser differs from `markdown-rs`, by number. - const KNOWN_DIFFERENCES: &[usize] = &[]; + /// Examples of the spec that are not rendered as in the spec, by number. The parser has kept the + /// behavior of `markdown-rs` here: + /// + /// - code with a tab that a container consumes in part, which keeps the rest of the tab as spaces in + /// the spec, and an image in the alt text of an image (5, 6, 7, 570) + /// - a document that starts with `---` has frontmatter (66, 68) + /// - emphasis next to other runs of `*` and `_` follows the algorithm of `markdown-rs` (388, 416, + /// 424, 425, 426, 463, 464, 465, 467) + /// - links with a protocol other than http, https, irc, ircs, mailto and xmpp have no `href` (496, + /// 594, 595, 597) + /// - GFM autolink literals, which the spec does not have (598, 604, 607, 608) + const KNOWN_DIFFERENCES: &[usize] = &[ + 5, 6, 7, 66, 68, 388, 416, 424, 425, 426, 463, 464, 465, 467, 496, 570, 594, 595, 597, 598, 604, 607, 608, + ]; - /// Compares every example of the spec, from `SPEC_FILE` or fetched over the network. + /// Renders every example of the spec to HTML, from `SPEC_FILE` or fetched over the network. #[test] #[ignore = "fetches spec.txt over the network; set SPEC_FILE to use a local copy"] - fn spec_examples_match_markdown_rs() { + fn spec_examples_render_as_in_the_spec() { let text = match std::env::var("SPEC_FILE") { Ok(path) => std::fs::read_to_string(path).unwrap(), Err(_) => ureq::get(SPEC_URL) @@ -746,24 +1160,14 @@ mod tests { let examples = spec_examples(&text); assert!(examples.len() > 600, "only {} examples were found", examples.len()); - if let Ok(number) = std::env::var("SPEC_SHOW") { - let (_, markdown) = &examples[number.parse::().unwrap() - 1]; - println!("{markdown:?}"); - println!("native: {:?}", parse(markdown).unwrap()); - println!("markdown-rs: {:?}", parse_with_markdown_rs(markdown).unwrap()); - } - let differing = examples .iter() - .filter(|(_, markdown)| { - let Ok(expected) = std::panic::catch_unwind(|| parse_with_markdown_rs(markdown)) else { - return false; - }; - format!("{:#?}", parse(markdown).unwrap()) != format!("{:#?}", expected.unwrap()) - }) - .map(|(number, _)| *number) + .filter(|(_, markdown, html)| to_html(markdown) != *html) + .map(|(number, ..)| *number) .collect::>(); - - assert_eq!(differing, KNOWN_DIFFERENCES, "examples that differ from markdown-rs"); + assert_eq!( + differing, KNOWN_DIFFERENCES, + "examples that are not rendered as in the spec" + ); } } diff --git a/crates/mq-markdown/src/parser/block.rs b/crates/mq-markdown/src/parser/block.rs index bd5b9030f..a17d52f84 100644 --- a/crates/mq-markdown/src/parser/block.rs +++ b/crates/mq-markdown/src/parser/block.rs @@ -53,7 +53,11 @@ struct Interrupt { } pub(super) fn parse(src: &str, mdx: bool) -> Vec { - let lines = split_lines(src, mdx); + let mut lines = split_lines(src, mdx); + // A byte order mark at the start of the document is not content. + if let Some(first) = lines.first_mut().filter(|line| line.text.starts_with('\u{feff}')) { + *first = first.skip('\u{feff}'.len_utf8()); + } match frontmatter(&lines).filter(|_| !mdx) { Some((node, next)) => { let mut blocks = vec![Block::Node(node)]; @@ -162,7 +166,7 @@ fn parse_blocks(lines: &[Line<'_>], depth: usize, interrupting: Interrupt) -> Ve } else if let Some(marker) = footnote_marker(line).filter(|_| containers) { index = footnote(lines, index, &marker, depth, interrupting, &mut blocks); after_container = true; - } else if let Some((items, next)) = table::parse(lines, index, interrupts_paragraph) { + } else if let Some((items, next)) = table::parse(lines, index, interrupts_paragraph, ends_table) { blocks.push(Block::Table(items)); index = next; } else if line.mdx @@ -568,6 +572,7 @@ fn list( fn last_end(block: &Block) -> Option { match block { Block::Node(node) => node.position().map(|position| position.end), + Block::Fenced(fenced) => fenced.node.position().map(|position| position.end), Block::Quote(quote) => Some(quote.position.end.clone()), Block::List(list) => list.items.last().map(|item| item.position.end.clone()), Block::Footnote(footnote) => Some(footnote.position.end.clone()), @@ -617,7 +622,7 @@ fn ends_in_text(block: &Block) -> bool { Block::Footnote(footnote) => footnote.children.last().is_some_and(ends_in_text), // A definition is a paragraph whose content was consumed. Block::Node(node) => matches!(node, Node::Definition(_)), - Block::Table(_) | Block::Jsx(_) | Block::Error(_) => false, + Block::Fenced(_) | Block::Table(_) | Block::Jsx(_) | Block::Error(_) => false, } } @@ -911,6 +916,13 @@ fn interrupts_paragraph(line: &Line<'_>) -> bool { || ListMarker::parse(line).is_some_and(|marker| marker.interrupts_paragraph())) } +/// Whether `line` ends a table. Unlike a paragraph, a table is also ended by a list item that is empty +/// or that is numbered from other than one. +fn ends_table(line: &Line<'_>) -> bool { + interrupts_paragraph(line) + || (!line.lazy && line.indent().0 < line.code_indent() && ListMarker::parse(line).is_some()) +} + /// Returns the index after the block and what it restricts on the first line of a following container. /// /// A paragraph of `dashes` right after a setext heading is cut short as described by [`Dashes`]. A paragraph with a `release` has its second line ended by list markers that could not interrupt it. @@ -984,7 +996,15 @@ fn paragraph( if first == index { // Nothing is left of the paragraph, so a setext underline starts a paragraph of text instead. return match setext { - Some((_, underline)) => (underline, Interrupt::default(), true), + Some((_, underline)) => ( + underline, + // The underline is still a line of a paragraph, which an empty list item cannot interrupt. + Interrupt { + code: false, + list: true, + }, + true, + ), None => (index, Interrupt::default(), false), }; } @@ -1038,7 +1058,13 @@ fn paragraph_source(lines: &[Line<'_>]) -> InlineSource { } else { text }; - (text, line.eol, line.point(offset_in(line.text, text))) + let offset = offset_in(line.text, text); + let point = if index == 0 { + line.content_point(offset) + } else { + line.point(offset) + }; + (text, line.eol, point) })) } @@ -1058,9 +1084,24 @@ fn footnote_marker<'a>(line: &Line<'a>) -> Option> { if columns >= CODE_INDENT { return None; } - let close = rest.find(']')?; + // A backslash escapes the bracket that follows it, as well as another backslash. + let bytes = rest.as_bytes(); + let mut close = 0; + while close < bytes.len() && bytes[close] != b']' { + if bytes[close] == b'[' { + return None; + } + close += if bytes[close] == b'\\' && matches!(bytes.get(close + 1), Some(b'[' | b'\\' | b']')) { + 2 + } else { + 1 + }; + } + if close >= bytes.len() { + return None; + } let label = &rest[..close]; - if label.is_empty() || label.bytes().any(|b| matches!(b, b' ' | b'\t' | b'[' | b'\r' | b'\n')) { + if label.is_empty() || label.bytes().any(|b| matches!(b, b' ' | b'\t' | b'\r' | b'\n')) { return None; } let after = rest[close + 1..].strip_prefix(':')?; @@ -1086,10 +1127,13 @@ fn footnote( state.feed(&first); let mut blanks = Vec::new(); let mut index = start + 1; + // The end of a definition without content: it reaches to the end of the blank lines that follow. + let mut empty_end = None; while let Some(line) = lines.get(index) { if line.is_blank() { if first.is_blank() && inner.len() == 1 { + empty_end = lines[index..].iter().take_while(|l| l.is_blank()).last().map(Line::end); break; } blanks.push(line.skip(line.text.len())); @@ -1113,12 +1157,15 @@ fn footnote( // Trailing blank lines belong to whatever follows, but still extend the position. index -= blanks.len(); - let position = Position { - start: lines[start].point(0), - end: blanks + let end = empty_end.unwrap_or_else(|| { + blanks .last() .or(inner.last()) - .map_or_else(|| lines[start].end(), Line::end), + .map_or_else(|| lines[start].end(), Line::end) + }); + let position = Position { + start: lines[start].point(0), + end, }; blocks.push(Block::Footnote(FootnoteBlock { ident: inline::normalize(marker.label), diff --git a/crates/mq-markdown/src/parser/code.rs b/crates/mq-markdown/src/parser/code.rs index 41158614f..edf07fcf7 100644 --- a/crates/mq-markdown/src/parser/code.rs +++ b/crates/mq-markdown/src/parser/code.rs @@ -3,7 +3,7 @@ use super::block::{CODE_INDENT, join_lines}; use super::inline; use super::line::Line; -use super::tree::Block; +use super::tree::{Block, FencedBlock}; use crate::node::{Code, Math, Node, Point, Position}; /// An opening code fence, or math fence. @@ -70,6 +70,7 @@ pub(super) fn fenced_code( // An unclosed fence runs to the end of its container. When that is the end of the document the // final line terminator is part of it, so the end is the start of the following line. // Right after a container, a fence without content ends with its own line instead. + let closed = end.is_some(); let end = end.unwrap_or_else(|| match lines.last() { Some(last) if !last.eol.is_empty() && reaches_line_end(last, body.is_empty(), own_end) => Point { line: last.number + 1, @@ -80,7 +81,7 @@ pub(super) fn fenced_code( }); let position = Some(Position { - start: lines[start].point(indent), + start: lines[start].content_point(indent), end, }); let value = join_lines(&body); @@ -95,7 +96,11 @@ pub(super) fn fenced_code( position, }) }; - blocks.push(Block::Node(node)); + blocks.push(Block::Fenced(FencedBlock { + node, + closed, + lines: body.len(), + })); index } diff --git a/crates/mq-markdown/src/parser/inline.rs b/crates/mq-markdown/src/parser/inline.rs index 976018d4a..77692e166 100644 --- a/crates/mq-markdown/src/parser/inline.rs +++ b/crates/mq-markdown/src/parser/inline.rs @@ -473,7 +473,7 @@ impl Scanner<'_> { let before = src[..start].chars().next_back(); let after = src[end..].chars().next(); - let (can_open, can_close) = emphasis::flanking(ch, before, after); + let (can_open, can_close) = emphasis::flanking(ch, before, after, !self.context.references.mdx); if !can_open && !can_close { self.pos = end; return; @@ -528,8 +528,20 @@ impl Scanner<'_> { }); self.push_node(node, autolink.end); } else if let Some(end) = html::inline_html(src, pos) { + // Lines of a paragraph lose their leading whitespace, also inside a tag. + let value = src[pos..end] + .split_inclusive('\n') + .enumerate() + .map(|(index, line)| { + if index == 0 { + line + } else { + line.trim_start_matches([' ', '\t']) + } + }) + .collect::(); let node = Node::Html(crate::node::Html { - value: src[pos..end].to_string(), + value, position: Some(self.context.position(pos, end)), }); self.push_node(node, end); diff --git a/crates/mq-markdown/src/parser/inline/emphasis.rs b/crates/mq-markdown/src/parser/inline/emphasis.rs index f1c2ab1ec..eee94e5d9 100644 --- a/crates/mq-markdown/src/parser/inline/emphasis.rs +++ b/crates/mq-markdown/src/parser/inline/emphasis.rs @@ -24,10 +24,11 @@ fn kind(char: Option) -> Kind { } } -/// Whether a run of `ch` between `before` and `after` can open and can close. -pub(super) fn flanking(ch: u8, before: Option, after: Option) -> (bool, bool) { +/// Whether a run of `ch` between `before` and `after` can open and can close. `~` is a delimiter too +/// when `gfm` is set. +pub(super) fn flanking(ch: u8, before: Option, after: Option, gfm: bool) -> (bool, bool) { let (before_kind, after_kind) = (kind(before), kind(after)); - let marker = |char: Option| matches!(char, Some('*' | '_' | '~')); + let marker = |char: Option| matches!(char, Some('*' | '_')) || (gfm && char == Some('~')); let open = after_kind == Kind::Other || (after_kind == Kind::Punctuation && before_kind != Kind::Other) diff --git a/crates/mq-markdown/src/parser/inline/entity.rs b/crates/mq-markdown/src/parser/inline/entity.rs index 3300f6c80..54bf92354 100644 --- a/crates/mq-markdown/src/parser/inline/entity.rs +++ b/crates/mq-markdown/src/parser/inline/entity.rs @@ -21,9 +21,13 @@ pub(super) fn decode(src: &str, pos: usize) -> Option<(usize, String)> { return None; } let number = u32::from_str_radix(&src[pos + 1 + digits_from..pos + 1 + digits_from + digits], radix).ok()?; + // Control characters other than whitespace are not allowed, nor are surrogates and + // out of range numbers. let char = match char::from_u32(number) { - Some(char) if number != 0 => char, - _ => char::REPLACEMENT_CHARACTER, + Some('\0'..='\u{8}' | '\u{b}' | '\u{e}'..='\u{1f}' | '\u{7f}'..='\u{9f}') | None => { + char::REPLACEMENT_CHARACTER + } + Some(char) => char, }; return Some((pos + 1 + digits_from + digits + 1, char.to_string())); } diff --git a/crates/mq-markdown/src/parser/inline/literal.rs b/crates/mq-markdown/src/parser/inline/literal.rs index 5818849f9..6be17b217 100644 --- a/crates/mq-markdown/src/parser/inline/literal.rs +++ b/crates/mq-markdown/src/parser/inline/literal.rs @@ -148,8 +148,8 @@ fn www(src: &str, pos: usize) -> Option { { return None; } - let prefix = src.get(pos..pos + 4)?; - if !prefix[..3].eq_ignore_ascii_case("www") || !prefix.ends_with('.') || pos + 4 >= bytes.len() { + let prefix = bytes.get(pos..pos + 4)?; + if !prefix[..3].eq_ignore_ascii_case(b"www") || prefix[3] != b'.' || pos + 4 >= bytes.len() { return None; } let end = domain(src, pos)?; diff --git a/crates/mq-markdown/src/parser/inline/tail.rs b/crates/mq-markdown/src/parser/inline/tail.rs index 241ecd6ee..87d76bb7d 100644 --- a/crates/mq-markdown/src/parser/inline/tail.rs +++ b/crates/mq-markdown/src/parser/inline/tail.rs @@ -146,7 +146,8 @@ pub(in crate::parser) fn destination(src: &str, pos: usize) -> Option<(String, u depth -= 1; } b' ' | b'\t' | b'\n' | b'\r' => break, - byte if byte.is_ascii_control() => break, + // markdown-rs lets a NUL through, though it is a control character. + byte if byte.is_ascii_control() && byte != 0 => break, _ => {} } index += 1; diff --git a/crates/mq-markdown/src/parser/line.rs b/crates/mq-markdown/src/parser/line.rs index e201fe3ca..e6c9d8eec 100644 --- a/crates/mq-markdown/src/parser/line.rs +++ b/crates/mq-markdown/src/parser/line.rs @@ -77,7 +77,8 @@ impl<'a> Line<'a> { } else { offset + 1 }; - // The start of a line inside a tab is before the end of that tab. + // Whitespace at the start of a line inside a tab starts before the end of that tab, content + // starts where that tab ends. if byte == 0 { column -= self.pad; } @@ -87,6 +88,16 @@ impl<'a> Line<'a> { } } + /// Like [`Line::point`] for content that `markdown-rs` places where the rest of a tab ends, instead + /// of before it: paragraphs and fences. + pub(super) fn content_point(&self, byte: usize) -> Point { + let mut point = self.point(byte); + if byte == 0 { + point.column += self.pad; + } + point + } + /// The same line with the first `bytes` bytes removed. pub(super) fn skip(self, bytes: usize) -> Line<'a> { Line { diff --git a/crates/mq-markdown/src/parser/mdx.rs b/crates/mq-markdown/src/parser/mdx.rs index 36a3ebd57..5d358fcbe 100644 --- a/crates/mq-markdown/src/parser/mdx.rs +++ b/crates/mq-markdown/src/parser/mdx.rs @@ -38,14 +38,14 @@ pub(super) struct Tag { pub(super) end: usize, } +/// Whether `char` can start a JavaScript identifier. fn id_start(char: char) -> bool { - char.is_alphabetic() || matches!(char, '$' | '_') + unicode_id::UnicodeID::is_id_start(char) || matches!(char, '$' | '_') } +/// Whether `char` can continue a JSX identifier, which may contain dashes. fn id_continue(char: char) -> bool { - char.is_alphanumeric() - || matches!(char, '_' | '-' | '\u{200c}' | '\u{200d}') - || ('\u{0300}'..='\u{036f}').contains(&char) + unicode_id::UnicodeID::is_id_continue(char) || matches!(char, '\u{200c}' | '\u{200d}' | '-') } /// What a tag parser stops with: a failure of the construct, or of the input. @@ -172,20 +172,24 @@ impl Cursor<'_> { /// of each line after the first. fn expression_value(&self, start: usize, end: usize) -> SmolStr { let content = &self.src[start..end]; - if !content.contains('\n') { + if !content.contains(['\n', '\r']) { return SmolStr::new(content); } let mut value = String::with_capacity(content.len()); - for (index, line) in content.split('\n').enumerate() { - if index > 0 { - value.push('\n'); + // How many more whitespace characters may be dropped at the start of the current line. + let mut droppable = 0; + for char in content.chars() { + match char { + '\n' | '\r' => { + droppable = 2; + value.push(char); + } + ' ' | '\t' if droppable > 0 => droppable -= 1, + _ => { + droppable = 0; + value.push(char); + } } - let skipped = if index == 0 { - 0 - } else { - line.chars().take(2).take_while(|c| matches!(c, ' ' | '\t')).count() - }; - value.push_str(&line[skipped..]); } SmolStr::new(value) } @@ -260,7 +264,7 @@ impl Cursor<'_> { fn tag(&mut self) -> Step { // `<` is followed by a name, not by whitespace. match self.peek() { - Some(' ' | '\t' | '\n') => return Err(Stop::Nok), + Some(' ' | '\t' | '\n' | '\r') => return Err(Stop::Nok), None => return self.end_of_input(), Some(_) => {} } @@ -339,7 +343,8 @@ impl Cursor<'_> { match self.peek() { None => self.end_of_input(), Some('/' | '>' | '{') => Ok(()), - Some(char) if id_start(char) => Ok(()), + // An attribute needs whitespace before it. + Some(char) if id_start(char) && self.src[..self.index].ends_with(char::is_whitespace) => Ok(()), Some(_) => self.crash(format!("Unexpected {} after name", self.describe())), } } diff --git a/crates/mq-markdown/src/parser/render_html.rs b/crates/mq-markdown/src/parser/render_html.rs new file mode 100644 index 000000000..5fe3362da --- /dev/null +++ b/crates/mq-markdown/src/parser/render_html.rs @@ -0,0 +1,711 @@ +//! Renders the block tree as HTML, with CommonMark, GFM, math and frontmatter, and with raw HTML +//! allowed. +//! +//! The output is the same as that of `markdown-rs`, line endings included: blocks start on a line of +//! their own, and the line ending that the source has after the last block is kept. + +use super::inline; +use super::resolve::{self, References}; +use super::tree::{Block, FencedBlock, InlineBlock, InlineKind, InlineSource, ListBlock, TableItem}; +use crate::node::{Code, Node, TableAlignKind}; +use rustc_hash::FxHashMap; + +/// Protocols that a link may have, others are dropped. +const SAFE_PROTOCOL_HREF: [&str; 6] = ["http", "https", "irc", "ircs", "mailto", "xmpp"]; +/// Protocols that an image source may have. +const SAFE_PROTOCOL_SRC: [&str; 2] = ["http", "https"]; + +/// Renders `content` as HTML. +pub(super) fn render(content: &str) -> String { + let blocks = super::block::parse(content, false); + let mut references = References::default(); + resolve::collect(&blocks, &mut references); + + let mut html = Html { + out: String::new(), + eol: first_line_ending(content), + references, + targets: FxHashMap::default(), + tight: Vec::new(), + calls: Vec::new(), + notes: Vec::new(), + check: None, + in_link: false, + }; + html.collect_targets(&blocks); + html.blocks(&blocks); + + if !html.calls.is_empty() { + html.footnote_section(); + } else if has_line_ending_after(content, last_rendered(&blocks)) && !list_then_hidden(content, &blocks) { + html.line_ending_if_needed(); + } + html.out +} + +/// The line ending of the first line that has one, as the document uses it throughout. +fn first_line_ending(content: &str) -> &'static str { + match content.find(['\n', '\r']) { + Some(index) if content[index..].starts_with("\r\n") => "\r\n", + Some(index) if content[index..].starts_with('\r') => "\r", + _ => "\n", + } +} + +/// Whether the source has a line ending after the line on which `end` ends. A document that has no +/// block to render has none. +fn has_line_ending_after(content: &str, end: Option) -> bool { + let Some(end) = end else { return false }; + let mut lines = 1; + let mut bytes = content.bytes().peekable(); + while let Some(byte) = bytes.next() { + match byte { + b'\n' => lines += 1, + b'\r' => { + if bytes.peek() == Some(&b'\n') { + bytes.next(); + } + lines += 1; + } + _ => {} + } + } + end < lines +} + +/// Whether a list is followed by nothing but blocks without output, such as definitions, and no blank +/// line at the end: its last line ending is then part of the list, where it is dropped. +fn list_then_hidden(content: &str, blocks: &[Block]) -> bool { + let hidden = |block: &Block| { + matches!( + block, + Block::Node(Node::Definition(_) | Node::Yaml(_) | Node::Toml(_)) | Block::Footnote(_) + ) + }; + let trailing = blocks.iter().rev().take_while(|block| hidden(block)).count(); + let blank_end = { + let trimmed = content.trim_end_matches([' ', '\t']); + ["\n\n", "\r\n\r\n", "\n\r\n", "\r\n\n", "\r\r"] + .iter() + .any(|end| trimmed.ends_with(end)) + }; + if trailing == 0 || blank_end || blocks.len() == trailing { + return false; + } + let list = &blocks[blocks.len() - trailing - 1]; + // Without a blank line in between, what follows would belong to the list. + matches!(list, Block::List(_)) + && matches!((end_line(list), start_line(&blocks[blocks.len() - trailing])), (Some(end), Some(start)) if start > end + 1) +} + +/// The line on which the last block that produces output ends. +fn last_rendered(blocks: &[Block]) -> Option { + blocks.iter().rev().find_map(|block| match block { + Block::Node(Node::Definition(_) | Node::Yaml(_) | Node::Toml(_)) | Block::Footnote(_) => None, + _ => end_line(block), + }) +} + +fn end_line(block: &Block) -> Option { + match block { + Block::Node(node) => node.position().map(|position| position.end.line), + Block::Fenced(fenced) => fenced.node.position().map(|position| position.end.line), + Block::Inline(InlineBlock { + kind: InlineKind::Heading { position, .. }, + .. + }) => Some(position.end.line), + Block::Inline(block) => block.source.lines.last().map(|(_, point)| point.line), + Block::Quote(quote) => Some(quote.position.end.line), + // The position of an item includes the blank lines that follow it. + Block::List(list) => list.items.last().and_then(|item| { + item.children + .iter() + .rev() + .find_map(end_line) + .or(Some(item.position.start.line)) + }), + Block::Footnote(footnote) => Some(footnote.position.end.line), + Block::Table(items) => items.last().map(|item| match item { + TableItem::Cell { position, .. } | TableItem::Align { position, .. } => position.end.line, + }), + Block::Jsx(_) | Block::Error(_) => None, + } +} + +fn start_line(block: &Block) -> Option { + match block { + Block::Node(node) => node.position().map(|position| position.start.line), + Block::Fenced(fenced) => fenced.node.position().map(|position| position.start.line), + Block::Inline(InlineBlock { + kind: InlineKind::Heading { position, .. }, + .. + }) => Some(position.start.line), + Block::Inline(block) => block.source.lines.first().map(|(_, point)| point.line), + Block::Quote(quote) => Some(quote.position.start.line), + Block::List(list) => list.items.first().map(|item| item.position.start.line), + Block::Footnote(footnote) => Some(footnote.position.start.line), + Block::Table(items) => items.first().map(|item| match item { + TableItem::Cell { position, .. } | TableItem::Align { position, .. } => position.start.line, + }), + Block::Jsx(_) | Block::Error(_) => None, + } +} + +/// Whether the list is loose: blank lines between its items, or between the blocks of an item. +fn is_loose(list: &ListBlock) -> bool { + list.spread + || list.items.iter().any(|item| { + item.children.windows(2).any(|pair| { + matches!((end_line(&pair[0]), start_line(&pair[1])), (Some(end), Some(start)) if start > end + 1) + }) + }) +} + +struct Html { + out: String, + eol: &'static str, + references: References, + /// The destination and title of each definition, by normalized label. The first one wins. + targets: FxHashMap)>, + /// Whether the innermost container is a tight list, which has paragraphs without `

`. + tight: Vec, + /// Footnotes in the order they are first referenced, with the number of references. + calls: Vec<(String, usize)>, + /// The rendered bodies of footnote definitions. + notes: Vec<(String, String)>, + /// The task list check that starts the next paragraph. + check: Option, + /// Whether the node is inside a link, which cannot contain a link. + in_link: bool, +} + +impl Html { + fn line_ending(&mut self) { + self.out.push_str(self.eol); + } + + fn line_ending_if_needed(&mut self) { + if !matches!(self.out.as_bytes().last(), None | Some(b'\n' | b'\r')) { + self.line_ending(); + } + } + + fn collect_targets(&mut self, blocks: &[Block]) { + for block in blocks { + match block { + Block::Node(Node::Definition(definition)) => { + self.targets.entry(definition.ident.clone()).or_insert_with(|| { + ( + definition.url.0.clone(), + definition.title.as_ref().map(|title| title.0.clone()), + ) + }); + } + Block::Quote(quote) => self.collect_targets("e.children), + Block::Footnote(footnote) => self.collect_targets(&footnote.children), + Block::List(list) => list.items.iter().for_each(|item| self.collect_targets(&item.children)), + _ => {} + } + } + } + + fn blocks(&mut self, blocks: &[Block]) { + for block in blocks { + self.block(block); + } + } + + fn block(&mut self, block: &Block) { + match block { + Block::Node(node) => self.leaf(node), + Block::Fenced(fenced) => self.fenced(fenced), + Block::Inline(block) => self.inline_block(block), + Block::Quote(quote) => { + self.tight.push(false); + self.line_ending_if_needed(); + self.out.push_str("

"); + self.blocks("e.children); + self.tight.pop(); + self.line_ending_if_needed(); + self.out.push_str("
"); + } + Block::List(list) => self.list(list), + Block::Footnote(footnote) => { + self.tight.push(false); + let outer = std::mem::take(&mut self.out); + self.blocks(&footnote.children); + let body = std::mem::replace(&mut self.out, outer); + self.tight.pop(); + self.notes.push((footnote.ident.clone(), body)); + } + Block::Table(items) => self.table(items), + Block::Jsx(_) | Block::Error(_) => {} + } + } + + fn leaf(&mut self, node: &Node) { + match node { + Node::HorizontalRule(_) => { + self.line_ending_if_needed(); + self.out.push_str("
"); + } + Node::Html(html) => { + self.line_ending_if_needed(); + self.out.push_str(&html.value); + } + Node::Code(code) => self.code(code, usize::from(!code.value.is_empty()), true), + _ => {} + } + } + + fn fenced(&mut self, fenced: &FencedBlock) { + match &fenced.node { + Node::Code(code) => self.code(code, fenced.lines, fenced.closed), + Node::Math(math) => { + self.line_ending_if_needed(); + self.out.push_str("
");
+                self.raw_flow(&math.value, fenced.lines, fenced.closed);
+            }
+            _ => {}
+        }
+    }
+
+    fn code(&mut self, code: &Code, lines: usize, closed: bool) {
+        self.line_ending_if_needed();
+        self.out.push_str("
');
+        self.raw_flow(&code.value, lines, closed);
+    }
+
+    /// The content of a code or math block: every line ends with a line ending. A fence that is not
+    /// closed ends with one as well.
+    fn raw_flow(&mut self, value: &str, lines: usize, closed: bool) {
+        encode(&mut self.out, value);
+        if lines > 0 {
+            self.line_ending();
+        }
+        self.out.push_str("
"); + if !closed { + self.line_ending_if_needed(); + } + } + + fn inline_block(&mut self, block: &InlineBlock) { + let nodes = inline::parse(&block.source, &self.references).unwrap_or_default(); + match &block.kind { + InlineKind::Paragraph => { + let tight = self.tight.last().copied().unwrap_or(false); + if !tight { + self.line_ending_if_needed(); + self.out.push_str("

"); + } else if !self.out.ends_with("

  • ") { + // What comes before a tight paragraph is followed by the line ending of the source. + self.line_ending_if_needed(); + } + self.task_check(); + self.inlines(&nodes); + if !tight { + self.out.push_str("

    "); + } + } + InlineKind::Heading { depth, .. } => { + self.line_ending_if_needed(); + self.out.push_str(&format!("")); + self.inlines(&nodes); + self.out.push_str(&format!("")); + } + } + } + + fn task_check(&mut self) { + if let Some(checked) = self.check.take() { + self.out.push_str(" "); + } + } + + fn list(&mut self, list: &ListBlock) { + let tight = !is_loose(list); + self.tight.push(tight); + self.line_ending_if_needed(); + self.out.push_str(if list.ordered { "'); + + for item in &list.items { + self.line_ending_if_needed(); + self.out.push_str("
  • "); + self.check = item.checked; + let empty = !item.children.iter().any(is_content); + self.blocks(&item.children); + self.check = None; + + let tight_paragraph = + tight + && item.children.iter().rfind(|block| is_content(block)).is_some_and( + |block| matches!(block, Block::Inline(b) if matches!(b.kind, InlineKind::Paragraph)), + ); + if !tight_paragraph && !empty { + self.line_ending_if_needed(); + } + self.out.push_str("
  • "); + } + + self.tight.pop(); + self.line_ending(); + self.out.push_str(if list.ordered { "" } else { "" }); + } + + fn table(&mut self, items: &[TableItem]) { + let aligns = items + .iter() + .find_map(|item| match item { + TableItem::Align { align, .. } => Some(align.as_slice()), + TableItem::Cell { .. } => None, + }) + .unwrap_or_default(); + + let mut rows: Vec>> = Vec::new(); + for item in items { + if let TableItem::Cell { row, source, .. } = item { + if rows.len() <= *row { + rows.resize_with(row + 1, Vec::new); + } + rows[*row].push(source); + } + } + + self.line_ending_if_needed(); + self.out.push_str(""); + for (index, row) in rows.iter().enumerate() { + if index == 0 { + self.line_ending_if_needed(); + self.out.push_str(""); + } else if index == 1 { + self.line_ending_if_needed(); + self.out.push_str(""); + } + self.line_ending_if_needed(); + self.out.push_str(""); + let tag = if index == 0 { "th" } else { "td" }; + for (column, align) in aligns.iter().enumerate() { + self.line_ending_if_needed(); + self.out.push_str(&format!("<{tag}")); + match align { + TableAlignKind::Left => self.out.push_str(" align=\"left\""), + TableAlignKind::Right => self.out.push_str(" align=\"right\""), + TableAlignKind::Center => self.out.push_str(" align=\"center\""), + TableAlignKind::None => {} + } + self.out.push('>'); + if let Some(Some(source)) = row.get(column) { + let nodes = inline::parse(source, &self.references).unwrap_or_default(); + self.inlines(&nodes); + } + self.out.push_str(&format!("")); + } + self.line_ending_if_needed(); + self.out.push_str(""); + if index == 0 { + self.line_ending_if_needed(); + self.out.push_str(""); + } + } + if rows.len() > 1 { + self.line_ending_if_needed(); + self.out.push_str(""); + } + self.line_ending_if_needed(); + self.out.push_str("
    "); + } + + fn inlines(&mut self, nodes: &[Node]) { + for node in nodes { + self.inline_node(node); + } + } + + fn inline_node(&mut self, node: &Node) { + match node { + Node::Text(text) => encode(&mut self.out, &text.value), + Node::Emphasis(node) => self.wrapped("em", &node.values), + Node::Strong(node) => self.wrapped("strong", &node.values), + Node::Delete(node) => self.wrapped("del", &node.values), + Node::CodeInline(code) => { + self.out.push_str(""); + self.code_span(&code.value); + } + Node::MathInline(math) => { + self.out.push_str(""); + self.code_span(&math.value); + } + Node::Break(_) => { + self.out.push_str("
    "); + self.line_ending(); + } + Node::Html(html) => self.out.push_str(&html.value), + Node::Link(link) => { + let title = link.title.as_ref().map(|title| title.0.as_str()); + self.link(&link.url.0, title, &link.values); + } + Node::LinkRef(link) => { + let (url, title) = self.targets.get(&link.ident).cloned().unwrap_or_default(); + self.link(&url, title.as_deref(), &link.values); + } + Node::Image(image) => { + self.image(&image.url, &image.alt, image.title.as_deref()); + } + Node::ImageRef(image) => { + let (url, title) = self.targets.get(&image.ident).cloned().unwrap_or_default(); + self.image(&url, &image.alt, title.as_deref()); + } + Node::FootnoteRef(reference) => self.footnote_call(&reference.ident), + _ => {} + } + } + + /// The content of a code span: line endings are spaces, and one space is dropped on both sides + /// unless there is nothing but spaces. + fn code_span(&mut self, value: &str) { + let mut text = String::with_capacity(value.len()); + let mut chars = value.chars().peekable(); + while let Some(char) = chars.next() { + match char { + '\r' if chars.peek() == Some(&'\n') => {} + '\n' | '\r' => text.push(' '), + _ => text.push(char), + } + } + let bytes = text.as_bytes(); + // Spaces at the edges were already dropped in the value, but line endings were not. + let edge = value.starts_with(['\n', '\r']) || value.ends_with(['\n', '\r']); + let trim = edge + && bytes.len() > 2 + && bytes[0] == b' ' + && bytes[bytes.len() - 1] == b' ' + && bytes.iter().any(|&b| b != b' '); + encode(&mut self.out, if trim { &text[1..text.len() - 1] } else { &text }); + self.out.push_str("
    "); + } + + fn wrapped(&mut self, tag: &str, nodes: &[Node]) { + self.out.push_str(&format!("<{tag}>")); + self.inlines(nodes); + self.out.push_str(&format!("")); + } + + fn link(&mut self, url: &str, title: Option<&str>, values: &[Node]) { + if self.in_link { + self.inlines(values); + return; + } + self.out.push_str("
    '); + self.in_link = true; + self.inlines(values); + self.in_link = false; + self.out.push_str(""); + } + + fn image(&mut self, url: &str, alt: &str, title: Option<&str>) { + self.out.push_str("\"");"); + } + + fn title(&mut self, title: Option<&str>) { + if let Some(title) = title { + self.out.push_str(" title=\""); + encode(&mut self.out, title); + self.out.push('"'); + } + } + + fn footnote_call(&mut self, ident: &str) { + let index = match self.calls.iter().position(|(id, _)| id == ident) { + Some(index) => index, + None => { + self.calls.push((ident.to_string(), 0)); + self.calls.len() - 1 + } + }; + self.calls[index].1 += 1; + let count = self.calls[index].1; + + let id = sanitize(ident); + self.out.push_str(&format!( + " 1 { + self.out.push_str(&format!("-{count}")); + } + self.out.push_str(&format!( + "\" data-footnote-ref=\"\" aria-describedby=\"footnote-label\">{}", + index + 1 + )); + } + + /// The list of the footnotes that were referenced, after everything else. + fn footnote_section(&mut self) { + self.line_ending_if_needed(); + self.out + .push_str("

    Footnotes

    "); + self.line_ending(); + self.out.push_str("
      "); + + for (ident, count) in std::mem::take(&mut self.calls) { + let id = sanitize(&ident); + self.line_ending(); + self.out.push_str(&format!("
    1. ")); + self.line_ending(); + + let mut backreferences = String::new(); + for number in 1..=count { + if number > 1 { + backreferences.push(' '); + } + backreferences.push_str(&format!(" 1 { + backreferences.push_str(&format!("-{number}")); + } + backreferences.push_str( + "\" data-footnote-backref=\"\" aria-label=\"Back to content\" class=\"data-footnote-backref\">\u{21a9}", + ); + if number > 1 { + backreferences.push_str(&format!("{number}")); + } + backreferences.push_str(""); + } + + let body = self + .notes + .iter() + .find(|(note, _)| *note == ident) + .map(|(_, body)| body.as_str()) + .unwrap_or_default(); + let trimmed = body.trim_end_matches(['\n', '\r']); + if let Some(before) = trimmed.strip_suffix("

      ") { + self.out.push_str(before); + self.out.push(' '); + self.out.push_str(&backreferences); + self.out.push_str("

      "); + self.out.push_str(&body[trimmed.len()..]); + } else { + self.out.push_str(body); + self.line_ending_if_needed(); + self.out.push_str(&backreferences); + } + self.line_ending_if_needed(); + self.out.push_str("
    2. "); + } + + self.line_ending(); + self.out.push_str("
    "); + self.line_ending(); + self.out.push_str("
    "); + self.line_ending(); + } +} + +/// Whether a block shows up in the output of a list item. +fn is_content(block: &Block) -> bool { + !matches!(block, Block::Node(Node::Definition(_)) | Block::Footnote(_)) +} + +/// Writes `value` with the characters that are special in HTML escaped, and NUL replaced. +fn encode(out: &mut String, value: &str) { + let mut start = 0; + for (index, byte) in value.bytes().enumerate() { + let replacement = match byte { + b'\0' => "\u{fffd}", + b'&' => "&", + b'"' => """, + b'<' => "<", + b'>' => ">", + _ => continue, + }; + out.push_str(&value[start..index]); + out.push_str(replacement); + start = index + 1; + } + out.push_str(&value[start..]); +} + +/// Percent-encodes what is not allowed in a URL, keeps what already is encoded, and escapes it for HTML. +fn sanitize(value: &str) -> String { + // NUL is replaced before the URL is encoded. + let chars = value + .chars() + .map(|char| if char == '\0' { '\u{fffd}' } else { char }) + .collect::>(); + let mut normalized = String::with_capacity(value.len()); + let mut index = 0; + let mut start = 0; + let mut buffer = [0; 4]; + + while index < chars.len() { + let char = chars[index]; + + // A correct percent encoded value. + if char == '%' + && index + 2 < chars.len() + && chars[index + 1].is_ascii_alphanumeric() + && chars[index + 2].is_ascii_alphanumeric() + { + index += 3; + continue; + } + + if char >= '\u{80}' || !matches!(char, '!' | '#' | '$' | '&'..=';' | '=' | '?'..='Z' | '_' | 'a'..='z' | '~') { + normalized.extend(&chars[start..index]); + char.encode_utf8(&mut buffer); + for byte in &buffer[..char.len_utf8()] { + normalized.push_str(&format!("%{byte:02X}")); + } + start = index + 1; + } + index += 1; + } + normalized.extend(&chars[start..]); + + let mut result = String::with_capacity(normalized.len()); + encode(&mut result, &normalized); + result +} + +/// Like [`sanitize`], and empty when the URL has a protocol that is not in `protocols`. +fn sanitize_with_protocols(value: &str, protocols: &[&str]) -> String { + let value = sanitize(value); + let end = value.find(['?', '#', '/']); + let mut colon = value.find(':'); + + // A colon after `?`, `#` or `/` is not the end of a protocol. + if let (Some(end), Some(index)) = (end, colon) + && index > end + { + colon = None; + } + if let Some(colon) = colon + && !protocols.contains(&value[..colon].to_lowercase().as_str()) + { + return String::new(); + } + value +} diff --git a/crates/mq-markdown/src/parser/resolve.rs b/crates/mq-markdown/src/parser/resolve.rs index 0481d236f..d76eea7dd 100644 --- a/crates/mq-markdown/src/parser/resolve.rs +++ b/crates/mq-markdown/src/parser/resolve.rs @@ -30,7 +30,7 @@ pub(super) fn resolve(blocks: Vec, mdx: bool) -> Result, String Ok(nodes) } -fn collect(blocks: &[Block], references: &mut References) { +pub(super) fn collect(blocks: &[Block], references: &mut References) { for block in blocks { match block { Block::Node(Node::Definition(definition)) => { @@ -108,6 +108,7 @@ fn flatten(blocks: Vec, references: &References, nodes: &mut Vec) - for block in blocks { match block { Block::Node(node) => frames.push(node), + Block::Fenced(fenced) => frames.push(fenced.node), Block::Inline(block) => { let mut out = Vec::new(); inline_block(block, references, &mut out)?; diff --git a/crates/mq-markdown/src/parser/snapshots.txt b/crates/mq-markdown/src/parser/snapshots.txt new file mode 100644 index 000000000..584c7b917 --- /dev/null +++ b/crates/mq-markdown/src/parser/snapshots.txt @@ -0,0 +1,1311 @@ +## empty "" +nodes: [] +html: "" +## blank_lines "\n\n \n" +nodes: [] +html: "" +## paragraph "hello\n" +nodes: [Text(Text { value: "hello", @1:1-1:6 })] +html: "

    hello

    \n" +## paragraph_no_eol "hello" +nodes: [Text(Text { value: "hello", @1:1-1:6 })] +html: "

    hello

    " +## paragraph_multiline "a\nb\nc\n" +nodes: [Text(Text { value: "a\nb\nc", @1:1-3:2 })] +html: "

    a\nb\nc

    \n" +## paragraph_indented_continuation "a\n b\n" +nodes: [Text(Text { value: "a\nb", @1:1-2:6 })] +html: "

    a\nb

    \n" +## paragraph_trailing_space "あい\n うえ \n" +nodes: [Text(Text { value: "あい\nうえ", @1:1-2:9 })] +html: "

    あい\nうえ

    \n" +## two_paragraphs "a\n\nb\n" +nodes: [Text(Text { value: "a", @1:1-1:2 }), Text(Text { value: "b", @3:1-3:2 })] +html: "

    a

    \n

    b

    \n" +## crlf "a\r\nb\r\n\r\nc\r\n" +nodes: [Text(Text { value: "a\r\nb", @1:1-2:2 }), Text(Text { value: "c", @4:1-4:2 })] +html: "

    a\r\nb

    \r\n

    c

    \r\n" +## crlf_fence "```\r\na\r\nb\r\n```\r\n" +nodes: [Code(Code { value: "a\r\nb", lang: None, @1:1-4:4, meta: None, fence: true })] +html: "
    a\r\nb\r\n
    \r\n" +## cr_only "a\rb\r" +nodes: [Text(Text { value: "a\rb", @1:1-2:2 })] +html: "

    a\rb

    \r" +## atx_h1 "# title\n" +nodes: [Heading(Heading { depth: 1, values: [Text(Text { value: "title", @1:3-1:8 })], @1:1-1:8 })] +html: "

    title

    \n" +## atx_h6 "###### title\n" +nodes: [Heading(Heading { depth: 6, values: [Text(Text { value: "title", @1:8-1:13 })], @1:1-1:13 })] +html: "
    title
    \n" +## atx_seven "####### title\n" +nodes: [Text(Text { value: "####### title", @1:1-1:14 })] +html: "

    ####### title

    \n" +## atx_no_space "#title\n" +nodes: [Text(Text { value: "#title", @1:1-1:7 })] +html: "

    #title

    \n" +## atx_empty "#\n" +nodes: [Heading(Heading { depth: 1, values: [], @1:1-1:2 })] +html: "

    \n" +## atx_closing "## title ##\n" +nodes: [Heading(Heading { depth: 2, values: [Text(Text { value: "title", @1:4-1:9 })], @1:1-1:12 })] +html: "

    title

    \n" +## atx_closing_no_space "## title##\n" +nodes: [Heading(Heading { depth: 2, values: [Text(Text { value: "title##", @1:4-1:11 })], @1:1-1:11 })] +html: "

    title##

    \n" +## atx_indent " ## title ## \n" +nodes: [Heading(Heading { depth: 2, values: [Text(Text { value: "title", @1:6-1:11 })], @1:1-1:16 })] +html: "

    title

    \n" +## atx_multibyte "# あ h #\n" +nodes: [Heading(Heading { depth: 1, values: [Text(Text { value: "あ h", @1:3-1:8 })], @1:1-1:10 })] +html: "

    あ h

    \n" +## atx_interrupts_paragraph "a\n# b\n" +nodes: [Text(Text { value: "a", @1:1-1:2 }), Heading(Heading { depth: 1, values: [Text(Text { value: "b", @2:3-2:4 })], @2:1-2:4 })] +html: "

    a

    \n

    b

    \n" +## setext_h1 "Title\n===\n" +nodes: [Heading(Heading { depth: 1, values: [Text(Text { value: "Title", @1:1-1:6 })], @1:1-2:4 })] +html: "

    Title

    \n" +## setext_h2 "a\n ---\n" +nodes: [Heading(Heading { depth: 2, values: [Text(Text { value: "a", @1:1-1:2 })], @1:1-2:6 })] +html: "

    a

    \n" +## setext_multiline "a\nb\n---\n" +nodes: [Heading(Heading { depth: 2, values: [Text(Text { value: "a\nb", @1:1-2:2 })], @1:1-3:4 })] +html: "

    a\nb

    \n" +## thematic_star "***\n" +nodes: [HorizontalRule(HorizontalRule { @1:1-1:4 })] +html: "
    \n" +## thematic_dash_spaced "- - -\n" +nodes: [HorizontalRule(HorizontalRule { @1:1-1:6 })] +html: "
    \n" +## thematic_underscore " ___ \n" +nodes: [HorizontalRule(HorizontalRule { @1:1-1:8 })] +html: "
    \n" +## thematic_two "**\n" +nodes: [Text(Text { value: "**", @1:1-1:3 })] +html: "

    **

    \n" +## fence_lang_meta "```rust title\nlet a;\n```\n" +nodes: [Code(Code { value: "let a;", lang: Some("rust"), @1:1-3:4, meta: Some("title"), fence: true })] +html: "
    let a;\n
    \n" +## fence_indented " ```\n a\n b\n ```\n" +nodes: [Code(Code { value: "a\n b", lang: None, @1:3-4:6, meta: None, fence: true })] +html: "
    a\n b\n
    \n" +## fence_tilde "~~~\na\n~~~\n" +nodes: [Code(Code { value: "a", lang: None, @1:1-3:4, meta: None, fence: true })] +html: "
    a\n
    \n" +## fence_unclosed "```\na\n" +nodes: [Code(Code { value: "a", lang: None, @1:1-3:1, meta: None, fence: true })] +html: "
    a\n
    \n" +## fence_empty "para\n\n\n```\n```\n" +nodes: [Text(Text { value: "para", @1:1-1:5 }), Code(Code { value: "", lang: None, @4:1-5:4, meta: None, fence: true })] +html: "

    para

    \n
    \n" +## fence_longer_close "```\na\n`````\n" +nodes: [Code(Code { value: "a", lang: None, @1:1-3:6, meta: None, fence: true })] +html: "
    a\n
    \n" +## fence_shorter_close "````\na\n```\nb\n````\n" +nodes: [Code(Code { value: "a\n```\nb", lang: None, @1:1-5:5, meta: None, fence: true })] +html: "
    a\n```\nb\n
    \n" +## fence_blank_inside "```\na\n\nb\n```\n" +nodes: [Code(Code { value: "a\n\nb", lang: None, @1:1-5:4, meta: None, fence: true })] +html: "
    a\n\nb\n
    \n" +## fence_backtick_info "``` a`b\nc\n" +nodes: [Text(Text { value: "``` a`b\nc", @1:1-2:2 })] +html: "

    ``` a`b\nc

    \n" +## fence_interrupts_paragraph "a\n```\nb\n```\n" +nodes: [Text(Text { value: "a", @1:1-1:2 }), Code(Code { value: "b", lang: None, @2:1-4:4, meta: None, fence: true })] +html: "

    a

    \n
    b\n
    \n" +## indented_code " code\n\n more\n\nx\n" +nodes: [Code(Code { value: "code\n\n more", lang: None, @1:1-3:11, meta: None, fence: false }), Text(Text { value: "x", @5:1-5:2 })] +html: "
    code\n\n  more\n
    \n

    x

    \n" +## quote "> a\n> b\n" +nodes: [Blockquote(Blockquote { values: [Text(Text { value: "a\nb", @1:3-2:4 })], @1:1-2:4 })] +html: "
    \n

    a\nb

    \n
    \n" +## quote_indented_lazy " > a\nb\n" +nodes: [Blockquote(Blockquote { values: [Text(Text { value: "a\nb", @1:5-2:2 })], @1:1-2:2 })] +html: "
    \n

    a\nb

    \n
    \n" +## quote_heading "> # h\n>\n> c\n" +nodes: [Blockquote(Blockquote { values: [Heading(Heading { depth: 1, values: [Text(Text { value: "h", @1:5-1:6 })], @1:3-1:6 }), Text(Text { value: "c", @3:3-3:4 })], @1:1-3:4 })] +html: "
    \n

    h

    \n

    c

    \n
    \n" +## quote_no_space ">a\n" +nodes: [Blockquote(Blockquote { values: [Text(Text { value: "a", @1:2-1:3 })], @1:1-1:3 })] +html: "
    \n

    a

    \n
    \n" +## quote_empty ">\n" +nodes: [Blockquote(Blockquote { values: [], @1:1-1:2 })] +html: "
    \n
    \n" +## quote_nested "> > a\n> b\n" +nodes: [Blockquote(Blockquote { values: [Blockquote(Blockquote { values: [Text(Text { value: "a\nb", @1:5-2:4 })], @1:3-2:4 })], @1:1-2:4 })] +html: "
    \n
    \n

    a\nb

    \n
    \n
    \n" +## quote_lazy_nested "> > a\nb\n" +nodes: [Blockquote(Blockquote { values: [Blockquote(Blockquote { values: [Text(Text { value: "a\nb", @1:5-2:2 })], @1:3-2:2 })], @1:1-2:2 })] +html: "
    \n
    \n

    a\nb

    \n
    \n
    \n" +## quote_blank_ends "> a\n\n> b\n" +nodes: [Blockquote(Blockquote { values: [Text(Text { value: "a", @1:3-1:4 })], @1:1-1:4 }), Blockquote(Blockquote { values: [Text(Text { value: "b", @3:3-3:4 })], @3:1-3:4 })] +html: "
    \n

    a

    \n
    \n
    \n

    b

    \n
    \n" +## quote_fence "> ```\n> a\n> ```\n" +nodes: [Blockquote(Blockquote { values: [Code(Code { value: "a", lang: None, @1:3-3:6, meta: None, fence: true })], @1:1-3:6 })] +html: "
    \n
    a\n
    \n
    \n" +## quote_no_lazy_after_fence "> ```\n> a\nb\n" +nodes: [Blockquote(Blockquote { values: [Code(Code { value: "a", lang: None, @1:3-2:4, meta: None, fence: true })], @1:1-2:4 }), Text(Text { value: "b", @3:1-3:2 })] +html: "
    \n
    a\n
    \n
    \n

    b

    \n" +## quote_interrupts_paragraph "a\n> b\n" +nodes: [Text(Text { value: "a", @1:1-1:2 }), Blockquote(Blockquote { values: [Text(Text { value: "b", @2:3-2:4 })], @2:1-2:4 })] +html: "

    a

    \n
    \n

    b

    \n
    \n" +## quote_thematic_not_lazy "> a\n---\n" +nodes: [Blockquote(Blockquote { values: [Text(Text { value: "a", @1:3-1:4 })], @1:1-1:4 }), HorizontalRule(HorizontalRule { @2:1-2:4 })] +html: "
    \n

    a

    \n
    \n
    \n" +## quote_multibyte "> あ\n> い\n" +nodes: [Blockquote(Blockquote { values: [Text(Text { value: "あ\nい", @1:3-2:6 })], @1:1-2:6 })] +html: "
    \n

    あ\nい

    \n
    \n" +## bullet "- a\n- b\n" +nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-1:4 }), List(List { values: [Text(Text { value: "b", @2:3-2:4 })], index: 1, level: 0, ordered: false, checked: None, spread: false, start: None, @2:3-2:4 })] +html: "
      \n
    • a
    • \n
    • b
    • \n
    \n" +## bullet_loose "- a\n\n- b\n" +nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 })], index: 0, level: 0, ordered: false, checked: None, spread: true, start: None, @1:3-1:4 }), List(List { values: [Text(Text { value: "b", @3:3-3:4 })], index: 1, level: 0, ordered: false, checked: None, spread: true, start: None, @3:3-3:4 })] +html: "
      \n
    • \n

      a

      \n
    • \n
    • \n

      b

      \n
    • \n
    \n" +## bullet_indent " - a\n" +nodes: [List(List { values: [Text(Text { value: "a", @1:4-1:5 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:4-1:5 })] +html: "
      \n
    • a
    • \n
    \n" +## bullet_mixed_markers "* a\n+ b\n" +nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-1:4 }), List(List { values: [Text(Text { value: "b", @2:3-2:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @2:3-2:4 })] +html: "
      \n
    • a
    • \n
    \n
      \n
    • b
    • \n
    \n" +## ordered "1. a\n2. b\n" +nodes: [List(List { values: [Text(Text { value: "a", @1:4-1:5 })], index: 0, level: 0, ordered: true, checked: None, spread: false, start: Some(1), @1:4-1:5 }), List(List { values: [Text(Text { value: "b", @2:4-2:5 })], index: 1, level: 0, ordered: true, checked: None, spread: false, start: Some(1), @2:4-2:5 })] +html: "
      \n
    1. a
    2. \n
    3. b
    4. \n
    \n" +## ordered_paren "5) a\n" +nodes: [List(List { values: [Text(Text { value: "a", @1:4-1:5 })], index: 0, level: 0, ordered: true, checked: None, spread: false, start: Some(5), @1:4-1:5 })] +html: "
      \n
    1. a
    2. \n
    \n" +## ordered_zero "0. a\n" +nodes: [List(List { values: [Text(Text { value: "a", @1:4-1:5 })], index: 0, level: 0, ordered: true, checked: None, spread: false, start: Some(0), @1:4-1:5 })] +html: "
      \n
    1. a
    2. \n
    \n" +## ordered_too_long "1234567890. a\n" +nodes: [Text(Text { value: "1234567890. a", @1:1-1:14 })] +html: "

    1234567890. a

    \n" +## ordered_mixed_delimiters "1. a\n2) b\n" +nodes: [List(List { values: [Text(Text { value: "a", @1:4-1:5 })], index: 0, level: 0, ordered: true, checked: None, spread: false, start: Some(1), @1:4-1:5 }), List(List { values: [Text(Text { value: "b", @2:4-2:5 })], index: 0, level: 0, ordered: true, checked: None, spread: false, start: Some(2), @2:4-2:5 })] +html: "
      \n
    1. a
    2. \n
    \n
      \n
    1. b
    2. \n
    \n" +## task "- [ ] x\n- [x] y\n- [X] z\n" +nodes: [List(List { values: [Text(Text { value: "x", @1:7-1:8 })], index: 0, level: 0, ordered: false, checked: Some(false), spread: false, start: None, @1:7-1:8 }), List(List { values: [Text(Text { value: "y", @2:7-2:8 })], index: 1, level: 0, ordered: false, checked: Some(true), spread: false, start: None, @2:7-2:8 }), List(List { values: [Text(Text { value: "z", @3:7-3:8 })], index: 2, level: 0, ordered: false, checked: Some(true), spread: false, start: None, @3:7-3:8 })] +html: "
      \n
    • x
    • \n
    • y
    • \n
    • z
    • \n
    \n" +## task_empty "- [ ]\n" +nodes: [List(List { values: [Text(Text { value: "[ ]", @1:3-1:6 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-1:6 })] +html: "
      \n
    • [ ]
    • \n
    \n" +## task_no_space "- [ ]x\n" +nodes: [List(List { values: [Text(Text { value: "[ ]x", @1:3-1:7 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-1:7 })] +html: "
      \n
    • [ ]x
    • \n
    \n" +## nested "- a\n - b\n c\n" +nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-1:4 }), List(List { values: [Text(Text { value: "b\nc", @2:5-3:6 })], index: 0, level: 1, ordered: false, checked: None, spread: false, start: None, @2:5-3:6 })] +html: "
      \n
    • a\n
        \n
      • b\nc
      • \n
      \n
    • \n
    \n" +## nested_ordered "1. a\n 1. b\n 2. c\n2. d\n" +nodes: [List(List { values: [Text(Text { value: "a", @1:4-1:5 })], index: 0, level: 0, ordered: true, checked: None, spread: false, start: Some(1), @1:4-1:5 }), List(List { values: [Text(Text { value: "b", @2:7-2:8 })], index: 0, level: 1, ordered: true, checked: None, spread: false, start: Some(1), @2:7-2:8 }), List(List { values: [Text(Text { value: "c", @3:7-3:8 })], index: 1, level: 1, ordered: true, checked: None, spread: false, start: Some(1), @3:7-3:8 }), List(List { values: [Text(Text { value: "d", @4:4-4:5 })], index: 1, level: 0, ordered: true, checked: None, spread: false, start: Some(1), @4:4-4:5 })] +html: "
      \n
    1. a\n
        \n
      1. b
      2. \n
      3. c
      4. \n
      \n
    2. \n
    3. d
    4. \n
    \n" +## nested_dedent "- a\n - b\n- c\n" +nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-1:4 }), List(List { values: [Text(Text { value: "b", @2:5-2:6 })], index: 0, level: 1, ordered: false, checked: None, spread: false, start: None, @2:5-2:6 }), List(List { values: [Text(Text { value: "c", @3:3-3:4 })], index: 1, level: 0, ordered: false, checked: None, spread: false, start: None, @3:3-3:4 })] +html: "
      \n
    • a\n
        \n
      • b
      • \n
      \n
    • \n
    • c
    • \n
    \n" +## two_paragraphs "- a\n\n b\n" +nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 }), Text(Text { value: "b", @3:3-3:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-3:4 })] +html: "
      \n
    • \n

      a

      \n

      b

      \n
    • \n
    \n" +## two_paragraphs_then_item "- a\n\n b\n- c\n" +nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 }), Text(Text { value: "b", @3:3-3:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-3:4 }), List(List { values: [Text(Text { value: "c", @4:3-4:4 })], index: 1, level: 0, ordered: false, checked: None, spread: false, start: None, @4:3-4:4 })] +html: "
      \n
    • \n

      a

      \n

      b

      \n
    • \n
    • \n

      c

      \n
    • \n
    \n" +## empty_item "-\n" +nodes: [List(List { values: [], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:1-1:2 })] +html: "
      \n
    • \n
    \n" +## empty_item_content "-\n foo\n" +nodes: [List(List { values: [Text(Text { value: "foo", @2:3-2:6 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @2:3-2:6 })] +html: "
      \n
    • foo
    • \n
    \n" +## empty_item_blank_then_text "-\n\n foo\n" +nodes: [List(List { values: [], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:1-1:2 }), Text(Text { value: "foo", @3:3-3:6 })] +html: "
      \n
    • \n
    \n

    foo

    \n" +## empty_items "-\n-\n" +nodes: [List(List { values: [], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:1-1:2 }), List(List { values: [], index: 1, level: 0, ordered: false, checked: None, spread: false, start: None, @2:1-2:2 })] +html: "
      \n
    • \n
    • \n
    \n" +## wide_marker_gap "- a\n b\n" +nodes: [List(List { values: [Text(Text { value: "a\nb", @1:5-2:6 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:5-2:6 })] +html: "
      \n
    • a\nb
    • \n
    \n" +## code_in_item "- a\n" +nodes: [List(List { values: [Code(Code { value: "a", lang: None, @1:3-1:8, meta: None, fence: false })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-1:8 })] +html: "
      \n
    • \n
      a\n
      \n
    • \n
    \n" +## lazy "- a\nb\n" +nodes: [List(List { values: [Text(Text { value: "a\nb", @1:3-2:2 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-2:2 })] +html: "
      \n
    • a\nb
    • \n
    \n" +## lazy_after_blank "- a\n\nb\n" +nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-1:4 }), Text(Text { value: "b", @3:1-3:2 })] +html: "
      \n
    • a
    • \n
    \n

    b

    \n" +## list_interrupts_paragraph "a\n- b\n" +nodes: [Text(Text { value: "a", @1:1-1:2 }), List(List { values: [Text(Text { value: "b", @2:3-2:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @2:3-2:4 })] +html: "

    a

    \n
      \n
    • b
    • \n
    \n" +## ordered_two_no_interrupt "a\n2. b\n" +nodes: [Text(Text { value: "a\n2. b", @1:1-2:5 })] +html: "

    a\n2. b

    \n" +## ordered_one_interrupts "a\n1. b\n" +nodes: [Text(Text { value: "a", @1:1-1:2 }), List(List { values: [Text(Text { value: "b", @2:4-2:5 })], index: 0, level: 0, ordered: true, checked: None, spread: false, start: Some(1), @2:4-2:5 })] +html: "

    a

    \n
      \n
    1. b
    2. \n
    \n" +## empty_item_no_interrupt "a\n-\n" +nodes: [Heading(Heading { depth: 2, values: [Text(Text { value: "a", @1:1-1:2 })], @1:1-2:2 })] +html: "

    a

    \n" +## thematic_over_list "- - -\n" +nodes: [HorizontalRule(HorizontalRule { @1:1-1:6 })] +html: "
    \n" +## list_in_quote "> - a\n> - b\n" +nodes: [Blockquote(Blockquote { values: [List(List { values: [Text(Text { value: "a", @1:5-1:6 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:5-1:6 }), List(List { values: [Text(Text { value: "b", @2:5-2:6 })], index: 1, level: 0, ordered: false, checked: None, spread: false, start: None, @2:5-2:6 })], @1:1-2:6 })] +html: "
    \n
      \n
    • a
    • \n
    • b
    • \n
    \n
    \n" +## quote_in_list "- > a\n > b\n" +nodes: [List(List { values: [Blockquote(Blockquote { values: [Text(Text { value: "a\nb", @1:5-2:6 })], @1:3-2:6 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-2:6 })] +html: "
      \n
    • \n
      \n

      a\nb

      \n
      \n
    • \n
    \n" +## quote_in_list_level "- a\n > - b\n" +nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 }), Blockquote(Blockquote { values: [List(List { values: [Text(Text { value: "b", @2:7-2:8 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @2:7-2:8 })], @2:3-2:8 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-2:8 })] +html: "
      \n
    • a\n
      \n
        \n
      • b
      • \n
      \n
      \n
    • \n
    \n" +## heading_in_item "- # h\n text\n" +nodes: [List(List { values: [Heading(Heading { depth: 1, values: [Text(Text { value: "h", @1:5-1:6 })], @1:3-1:6 }), Text(Text { value: "text", @2:3-2:7 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-2:7 })] +html: "
      \n
    • \n

      h

      \ntext
    • \n
    \n" +## fence_in_item "- ```\n a\n ```\n" +nodes: [List(List { values: [Code(Code { value: "a", lang: None, @1:3-3:6, meta: None, fence: true })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-3:6 })] +html: "
      \n
    • \n
      a\n
      \n
    • \n
    \n" +## multibyte_item "- あ\n- い\n" +nodes: [List(List { values: [Text(Text { value: "あ", @1:3-1:6 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-1:6 }), List(List { values: [Text(Text { value: "い", @2:3-2:6 })], index: 1, level: 0, ordered: false, checked: None, spread: false, start: None, @2:3-2:6 })] +html: "
      \n
    • あ
    • \n
    • い
    • \n
    \n" +## list_after_paragraph_blank "a\n\n- b\n" +nodes: [Text(Text { value: "a", @1:1-1:2 }), List(List { values: [Text(Text { value: "b", @3:3-3:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @3:3-3:4 })] +html: "

    a

    \n
      \n
    • b
    • \n
    \n" +## two_lists_split "- a\n\n\n- b\n" +nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 })], index: 0, level: 0, ordered: false, checked: None, spread: true, start: None, @1:3-1:4 }), List(List { values: [Text(Text { value: "b", @4:3-4:4 })], index: 1, level: 0, ordered: false, checked: None, spread: true, start: None, @4:3-4:4 })] +html: "
      \n
    • \n

      a

      \n
    • \n
    • \n

      b

      \n
    • \n
    \n" +## deep_quote ">>>>>> a\n" +nodes: [Blockquote(Blockquote { values: [Blockquote(Blockquote { values: [Blockquote(Blockquote { values: [Blockquote(Blockquote { values: [Blockquote(Blockquote { values: [Blockquote(Blockquote { values: [Text(Text { value: "a", @1:8-1:9 })], @1:6-1:9 })], @1:5-1:9 })], @1:4-1:9 })], @1:3-1:9 })], @1:2-1:9 })], @1:1-1:9 })] +html: "
    \n
    \n
    \n
    \n
    \n
    \n

    a

    \n
    \n
    \n
    \n
    \n
    \n
    \n" +## fence_unclosed_no_eol "```\na" +nodes: [Code(Code { value: "a", lang: None, @1:1-2:2, meta: None, fence: true })] +html: "
    a\n
    \n" +## fence_unclosed_in_item "- ```\n a\n" +nodes: [List(List { values: [Code(Code { value: "a", lang: None, @1:3-3:1, meta: None, fence: true })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-3:1 })] +html: "
      \n
    • \n
      a\n
      \n
    • \n
    " +## fence_unclosed_in_item_no_eol "- ```\n a" +nodes: [List(List { values: [Code(Code { value: "a", lang: None, @1:3-2:4, meta: None, fence: true })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-2:4 })] +html: "
      \n
    • \n
      a\n
      \n
    • \n
    " +## fence_unclosed_in_quote_no_eol "> ```\n> a" +nodes: [Blockquote(Blockquote { values: [Code(Code { value: "a", lang: None, @1:3-2:4, meta: None, fence: true })], @1:1-2:4 })] +html: "
    \n
    a\n
    \n
    " +## fence_unclosed_in_quote_eof "> ```\n> a\n" +nodes: [Blockquote(Blockquote { values: [Code(Code { value: "a", lang: None, @1:3-3:1, meta: None, fence: true })], @1:1-3:1 })] +html: "
    \n
    a\n
    \n
    " +## fence_unclosed_trailing_blank "```\na\n\n" +nodes: [Code(Code { value: "a\n", lang: None, @1:1-4:1, meta: None, fence: true })] +html: "
    a\n\n
    \n" +## fence_unclosed_item_then_para "- ```\n a\nb\n" +nodes: [List(List { values: [Code(Code { value: "a", lang: None, @1:3-2:4, meta: None, fence: true })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-2:4 }), Text(Text { value: "b", @3:1-3:2 })] +html: "
      \n
    • \n
      a\n
      \n
    • \n
    \n

    b

    \n" +## item_code_then_unindented_text "- ===\n\n a\n===" +nodes: [List(List { values: [Text(Text { value: "===", @1:3-1:6 }), Code(Code { value: "a", lang: None, @3:3-3:8, meta: None, fence: false })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-3:8 }), Text(Text { value: "===", @4:1-4:4 })] +html: "
      \n
    • \n

      ===

      \n
      a\n
      \n
    • \n
    \n

    ===

    " +## quote_setext_dashes "> a\n> ---\n> ---\n> a" +nodes: [Blockquote(Blockquote { values: [Heading(Heading { depth: 2, values: [Text(Text { value: "a", @1:3-1:4 })], @1:3-2:6 }), Text(Text { value: "---", @3:3-3:6 }), Text(Text { value: "a", @4:3-4:4 })], @1:1-4:4 })] +html: "
    \n

    a

    \n

    ---

    \n

    a

    \n
    " +## table_basic "| a | b |\n|---|:-:|\n| 1 | 2 |\n" +nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:3-1:4 })]:0, row: 0, @1:1-1:5 }), TableCell(TableCell { values: [Text(Text { value: "b", @1:7-1:8 })]:1, row: 0, @1:5-1:10 }), TableAlign(TableAlign { align: [None, Center], @2:1-2:1 }), TableCell(TableCell { values: [Text(Text { value: "1", @3:3-3:4 })]:0, row: 1, @3:1-3:5 }), TableCell(TableCell { values: [Text(Text { value: "2", @3:7-3:8 })]:1, row: 1, @3:5-3:10 })] +html: "\n\n\n\n\n\n\n\n\n\n\n\n\n
    ab
    12
    \n" +## table_no_edge_pipes "a|b\n-|-\n1|2|3\n" +nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:1-1:2 })]:0, row: 0, @1:1-1:2 }), TableCell(TableCell { values: [Text(Text { value: "b", @1:3-1:4 })]:1, row: 0, @1:2-1:4 }), TableAlign(TableAlign { align: [None, None], @2:1-2:1 }), TableCell(TableCell { values: [Text(Text { value: "1", @3:1-3:2 })]:0, row: 1, @3:1-3:2 }), TableCell(TableCell { values: [Text(Text { value: "2", @3:3-3:4 })]:1, row: 1, @3:2-3:4 }), TableCell(TableCell { values: [Text(Text { value: "3", @3:5-3:6 })]:2, row: 1, @3:4-3:6 })] +html: "\n\n\n\n\n\n\n\n\n\n\n\n\n
    ab
    12
    \n" +## table_short_row "|a|b|\n|-|-|\n|1|\n" +nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:2-1:3 })]:0, row: 0, @1:1-1:3 }), TableCell(TableCell { values: [Text(Text { value: "b", @1:4-1:5 })]:1, row: 0, @1:3-1:6 }), TableAlign(TableAlign { align: [None, None], @2:1-2:1 }), TableCell(TableCell { values: [Text(Text { value: "1", @3:2-3:3 })]:0, row: 1, @3:1-3:4 })] +html: "\n\n\n\n\n\n\n\n\n\n\n\n\n
    ab
    1
    \n" +## table_escaped_pipe "| a \\| b | c |\n|--|--|\n" +nodes: [TableCell(TableCell { values: [Text(Text { value: "a | b", @1:3-1:9 })]:0, row: 0, @1:1-1:10 }), TableCell(TableCell { values: [Text(Text { value: "c", @1:12-1:13 })]:1, row: 0, @1:10-1:15 }), TableAlign(TableAlign { align: [None, None], @2:1-2:1 })] +html: "\n\n\n\n\n\n\n
    a | bc
    \n" +## table_indented " | a |\n |---|\n | b |\n" +nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:5-1:6 })]:0, row: 0, @1:1-1:8 }), TableAlign(TableAlign { align: [None], @2:1-2:1 }), TableCell(TableCell { values: [Text(Text { value: "b", @3:5-3:6 })]:0, row: 1, @3:1-3:8 })] +html: "\n\n\n\n\n\n\n\n\n\n\n
    a
    b
    \n" +## table_text_after "| a |\n|---|\ntext after\n" +nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:3-1:4 })]:0, row: 0, @1:1-1:6 }), TableAlign(TableAlign { align: [None], @2:1-2:1 }), TableCell(TableCell { values: [Text(Text { value: "text after", @3:1-3:11 })]:0, row: 1, @3:1-3:11 })] +html: "\n\n\n\n\n\n\n\n\n\n\n
    a
    text after
    \n" +## table_in_quote "> | a |\n> |---|\n> | b |\n" +nodes: [Blockquote(Blockquote { values: [TableCell(TableCell { values: [Text(Text { value: "a", @1:5-1:6 })]:0, row: 0, @1:3-1:8 }), TableAlign(TableAlign { align: [None], @2:1-2:1 }), TableCell(TableCell { values: [Text(Text { value: "b", @3:5-3:6 })]:0, row: 1, @3:3-3:8 })], @1:1-3:8 })] +html: "
    \n\n\n\n\n\n\n\n\n\n\n\n
    a
    b
    \n
    \n" +## table_after_paragraph "x\n| a |\n|---|\n" +nodes: [Text(Text { value: "x", @1:1-1:2 }), TableCell(TableCell { values: [Text(Text { value: "a", @2:3-2:4 })]:0, row: 0, @2:1-2:6 }), TableAlign(TableAlign { align: [None], @3:1-3:1 })] +html: "

    x

    \n\n\n\n\n\n\n
    a
    \n" +## table_column_mismatch "| a | b |\n|---|\n" +nodes: [Text(Text { value: "| a | b |\n|---|", @1:1-2:6 })] +html: "

    | a | b |\n|---|

    \n" +## table_bad_delimiter "|a|\n|:|\n" +nodes: [Text(Text { value: "|a|\n|:|", @1:1-2:4 })] +html: "

    |a|\n|:|

    \n" +## table_then_blank_para "|a|\n|-|\n\nnext\n" +nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:2-1:3 })]:0, row: 0, @1:1-1:4 }), TableAlign(TableAlign { align: [None], @2:1-2:1 }), Text(Text { value: "next", @4:1-4:5 })] +html: "\n\n\n\n\n\n
    a
    \n

    next

    \n" +## table_then_quote "|a|\n|-|\n> q\n" +nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:2-1:3 })]:0, row: 0, @1:1-1:4 }), TableAlign(TableAlign { align: [None], @2:1-2:1 }), Blockquote(Blockquote { values: [Text(Text { value: "q", @3:3-3:4 })], @3:1-3:4 })] +html: "\n\n\n\n\n\n
    a
    \n
    \n

    q

    \n
    \n" +## table_then_heading "|a|\n|-|\n# h\n" +nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:2-1:3 })]:0, row: 0, @1:1-1:4 }), TableAlign(TableAlign { align: [None], @2:1-2:1 }), Heading(Heading { depth: 1, values: [Text(Text { value: "h", @3:3-3:4 })], @3:1-3:4 })] +html: "\n\n\n\n\n\n
    a
    \n

    h

    \n" +## table_then_fence "|a|\n|-|\n```\nx\n```\n" +nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:2-1:3 })]:0, row: 0, @1:1-1:4 }), TableAlign(TableAlign { align: [None], @2:1-2:1 }), Code(Code { value: "x", lang: None, @3:1-5:4, meta: None, fence: true })] +html: "\n\n\n\n\n\n
    a
    \n
    x\n
    \n" +## table_then_list "|a|\n|-|\n- x\n" +nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:2-1:3 })]:0, row: 0, @1:1-1:4 }), TableAlign(TableAlign { align: [None], @2:1-2:1 }), List(List { values: [Text(Text { value: "x", @3:3-3:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @3:3-3:4 })] +html: "\n\n\n\n\n\n
    a
    \n
      \n
    • x
    • \n
    \n" +## table_two "|a|\n|-|\n\n|b|\n|-|\n" +nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:2-1:3 })]:0, row: 0, @1:1-1:4 }), TableAlign(TableAlign { align: [None], @2:1-2:1 }), TableCell(TableCell { values: [Text(Text { value: "b", @4:2-4:3 })]:0, row: 0, @4:1-4:4 }), TableAlign(TableAlign { align: [None], @5:1-5:1 })] +html: "\n\n\n\n\n\n
    a
    \n\n\n\n\n\n\n
    b
    \n" +## table_align_spaces "|a|b|\n|:-|-:|\n| x | y|\n" +nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:2-1:3 })]:0, row: 0, @1:1-1:3 }), TableCell(TableCell { values: [Text(Text { value: "b", @1:4-1:5 })]:1, row: 0, @1:3-1:6 }), TableAlign(TableAlign { align: [Left, Right], @2:1-2:1 }), TableCell(TableCell { values: [Text(Text { value: "x", @3:4-3:5 })]:0, row: 1, @3:1-3:7 }), TableCell(TableCell { values: [Text(Text { value: "y", @3:10-3:11 })]:1, row: 1, @3:7-3:12 })] +html: "\n\n\n\n\n\n\n\n\n\n\n\n\n
    ab
    xy
    \n" +## table_indented_code_row "|a|\n|-|\n|b|\n |c|\n" +nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:2-1:3 })]:0, row: 0, @1:1-1:4 }), TableAlign(TableAlign { align: [None], @2:1-2:1 }), TableCell(TableCell { values: [Text(Text { value: "b", @3:2-3:3 })]:0, row: 1, @3:1-3:4 }), Code(Code { value: "|c|", lang: None, @4:1-4:8, meta: None, fence: false })] +html: "\n\n\n\n\n\n\n\n\n\n\n
    a
    b
    \n
    |c|\n
    \n" +## table_no_body "| a |\n| - |\n" +nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:3-1:4 })]:0, row: 0, @1:1-1:6 }), TableAlign(TableAlign { align: [None], @2:1-2:1 })] +html: "\n\n\n\n\n\n
    a
    \n" +## table_empty_cells "||\n|-|\n" +nodes: [TableCell(TableCell { values: []:0, row: 0, @1:1-1:3 }), TableAlign(TableAlign { align: [None], @2:1-2:1 })] +html: "\n\n\n\n\n\n
    \n" +## table_lone_pipe_row "|a|\n|-|\n|\n" +nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:2-1:3 })]:0, row: 0, @1:1-1:4 }), TableAlign(TableAlign { align: [None], @2:1-2:1 }), TableCell(TableCell { values: []:0, row: 1, @3:1-3:2 })] +html: "\n\n\n\n\n\n\n\n\n\n\n
    a
    \n" +## table_no_trailing_pipe "| a | b\n|-|-\n| c | d\n" +nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:3-1:4 })]:0, row: 0, @1:1-1:5 }), TableCell(TableCell { values: [Text(Text { value: "b", @1:7-1:8 })]:1, row: 0, @1:5-1:8 }), TableAlign(TableAlign { align: [None, None], @2:1-2:1 }), TableCell(TableCell { values: [Text(Text { value: "c", @3:3-3:4 })]:0, row: 1, @3:1-3:5 }), TableCell(TableCell { values: [Text(Text { value: "d", @3:7-3:8 })]:1, row: 1, @3:5-3:8 })] +html: "\n\n\n\n\n\n\n\n\n\n\n\n\n
    ab
    cd
    \n" +## table_multibyte "| あ | い |\n|---|---|\n| う | え |\n" +nodes: [TableCell(TableCell { values: [Text(Text { value: "あ", @1:3-1:6 })]:0, row: 0, @1:1-1:7 }), TableCell(TableCell { values: [Text(Text { value: "い", @1:9-1:12 })]:1, row: 0, @1:7-1:14 }), TableAlign(TableAlign { align: [None, None], @2:1-2:1 }), TableCell(TableCell { values: [Text(Text { value: "う", @3:3-3:6 })]:0, row: 1, @3:1-3:7 }), TableCell(TableCell { values: [Text(Text { value: "え", @3:9-3:12 })]:1, row: 1, @3:7-3:14 })] +html: "\n\n\n\n\n\n\n\n\n\n\n\n\n
    あい
    うえ
    \n" +## not_table_setext "Title\n---\n" +nodes: [Heading(Heading { depth: 2, values: [Text(Text { value: "Title", @1:1-1:6 })], @1:1-2:4 })] +html: "

    Title

    \n" +## not_table_no_pipe_delim "a|b\n---\n" +nodes: [Heading(Heading { depth: 2, values: [Text(Text { value: "a|b", @1:1-1:4 })], @1:1-2:4 })] +html: "

    a|b

    \n" +## table_header_no_pipe "a\n|-|\n" +nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:1-1:2 })]:0, row: 0, @1:1-1:2 }), TableAlign(TableAlign { align: [None], @2:1-2:1 })] +html: "\n\n\n\n\n\n
    a
    \n" +## table_one_col_dash "|a|\n-\n" +nodes: [Heading(Heading { depth: 2, values: [Text(Text { value: "|a|", @1:1-1:4 })], @1:1-2:2 })] +html: "

    |a|

    \n" +## inline_escape "a\\*b" +nodes: [Text(Text { value: "a*b", @1:1-1:5 })] +html: "

    a*b

    " +## inline_escape_nonpunct "a\\qb" +nodes: [Text(Text { value: "a\\qb", @1:1-1:5 })] +html: "

    a\\qb

    " +## inline_entities "a&b © # A &unknown; &" +nodes: [Text(Text { value: "a&b © # A &unknown; &", @1:1-1:40 })] +html: "

    a&b © # A &unknown; &

    " +## inline_emphasis_all "a *b* **c** _d_ __e__ ***f***" +nodes: [Text(Text { value: "a ", @1:1-1:3 }), Emphasis(Emphasis { values: [Text(Text { value: "b", @1:4-1:5 })], @1:3-1:6 }), Text(Text { value: " ", @1:6-1:7 }), Strong(Strong { values: [Text(Text { value: "c", @1:9-1:10 })], @1:7-1:12 }), Text(Text { value: " ", @1:12-1:13 }), Emphasis(Emphasis { values: [Text(Text { value: "d", @1:14-1:15 })], @1:13-1:16 }), Text(Text { value: " ", @1:16-1:17 }), Strong(Strong { values: [Text(Text { value: "e", @1:19-1:20 })], @1:17-1:22 }), Text(Text { value: " ", @1:22-1:23 }), Emphasis(Emphasis { values: [Strong(Strong { values: [Text(Text { value: "f", @1:26-1:27 })], @1:24-1:29 })], @1:23-1:30 })] +html: "

    a b c d e f

    " +## inline_code "`code` `` a`b `` ` a `" +nodes: [CodeInline(CodeInline { value: "code", @1:1-1:7 }), Text(Text { value: " ", @1:7-1:8 }), CodeInline(CodeInline { value: "a`b", @1:8-1:17 }), Text(Text { value: " ", @1:17-1:18 }), CodeInline(CodeInline { value: "a", @1:18-1:23 })] +html: "

    code a`b a

    " +## inline_code_unclosed "`a ``b`" +nodes: [CodeInline(CodeInline { value: "a ``b", @1:1-1:8 })] +html: "

    a ``b

    " +## inline_hard_break_spaces "a \nb" +nodes: [Text(Text { value: "a", @1:1-1:2 }), Break(Break { @1:2-2:1 }), Text(Text { value: "b", @2:1-2:2 })] +html: "

    a
    \nb

    " +## inline_hard_break_backslash "a\\\nb" +nodes: [Text(Text { value: "a", @1:1-1:2 }), Break(Break { @1:2-2:1 }), Text(Text { value: "b", @2:1-2:2 })] +html: "

    a
    \nb

    " +## inline_soft_break "a\nb" +nodes: [Text(Text { value: "a\nb", @1:1-2:2 })] +html: "

    a\nb

    " +## inline_soft_break_trailing_space "a \nb" +nodes: [Text(Text { value: "a\nb", @1:1-2:2 })] +html: "

    a\nb

    " +## inline_emphasis_multiline "*a\nb*" +nodes: [Emphasis(Emphasis { values: [Text(Text { value: "a\nb", @1:2-2:2 })], @1:1-2:3 })] +html: "

    a\nb

    " +## inline_strike "a ~b~ ~~c~~ ~~~d~~~" +nodes: [Text(Text { value: "a ", @1:1-1:3 }), Delete(Delete { values: [Text(Text { value: "b", @1:4-1:5 })], @1:3-1:6 }), Text(Text { value: " ", @1:6-1:7 }), Delete(Delete { values: [Text(Text { value: "c", @1:9-1:10 })], @1:7-1:12 }), Text(Text { value: " ~~~d~~~", @1:12-1:20 })] +html: "

    a b c ~~~d~~~

    " +## inline_math "$x$ $$y$$ $ a" +nodes: [MathInline(MathInline { value: "x", @1:1-1:4 }), Text(Text { value: " ", @1:4-1:5 }), MathInline(MathInline { value: "y", @1:5-1:10 }), Text(Text { value: " $ a", @1:10-1:14 })] +html: "

    x y $ a

    " +## inline_autolinks "
    " +nodes: [Link(Link { url: Url("http://a.b"), title: None, values: [Text(Text { value: "http://a.b", @1:2-1:12 })], @1:1-1:13 }), Text(Text { value: " ", @1:13-1:14 }), Link(Link { url: Url("mailto:a@b.c"), title: None, values: [Text(Text { value: "a@b.c", @1:15-1:20 })], @1:14-1:21 }), Text(Text { value: " ", @1:21-1:22 }), Html(Html { value: "", @1:22-1:28 }), Text(Text { value: " ", @1:28-1:29 }), Html(Html { value: "", @1:29-1:39 }), Text(Text { value: " ", @1:39-1:40 }), Html(Html { value: "
    ", @1:40-1:45 })] +html: "

    http://a.b a@b.c

    " +## inline_links "[a](http://x \"t\") [b][c] [d][] [e] ![f](g) ![h][i]" +nodes: [Link(Link { url: Url("http://x"), title: Some(Title("t")), values: [Text(Text { value: "a", @1:2-1:3 })], @1:1-1:18 }), Text(Text { value: " [b][c] [d][] [e] ", @1:18-1:36 }), Image(Image { alt: "f", url: "g", title: None, @1:36-1:43 }), Text(Text { value: " ![h][i]", @1:43-1:51 })] +html: "

    a [b][c] [d][] [e] \"f\" ![h][i]

    " +## inline_link_forms "[a]( 'x') [d]( e ) [f](g (h)) [i](j \"k\\\"l\") [m](n&o)" +nodes: [Link(Link { url: Url("b c"), title: Some(Title("x")), values: [Text(Text { value: "a", @1:2-1:3 })], @1:1-1:15 }), Text(Text { value: " ", @1:15-1:16 }), Link(Link { url: Url("e"), title: None, values: [Text(Text { value: "d", @1:17-1:18 })], @1:16-1:24 }), Text(Text { value: " ", @1:24-1:25 }), Link(Link { url: Url("g"), title: Some(Title("h")), values: [Text(Text { value: "f", @1:26-1:27 })], @1:25-1:35 }), Text(Text { value: " ", @1:35-1:36 }), Link(Link { url: Url("j"), title: Some(Title("k\"l")), values: [Text(Text { value: "i", @1:37-1:38 })], @1:36-1:49 }), Text(Text { value: " ", @1:49-1:50 }), Link(Link { url: Url("n&o"), title: None, values: [Text(Text { value: "m", @1:51-1:52 })], @1:50-1:62 })] +html: "

    a d f i m

    " +## inline_link_multiline "[a](b\n\"t\") [c](d" +nodes: [Link(Link { url: Url("b"), title: Some(Title("t")), values: [Text(Text { value: "a", @1:2-1:3 })], @1:1-2:5 }), Text(Text { value: " [c](d", @2:5-2:11 })] +html: "

    a [c](d

    " +## inline_image_alt "![a *b* `c`](d) ![e ![f](g)](h)" +nodes: [Image(Image { alt: "a b c", url: "d", title: None, @1:1-1:16 }), Text(Text { value: " ", @1:16-1:17 }), Image(Image { alt: "e ", url: "h", title: None, @1:17-1:32 })] +html: "

    \"a \"e

    " +## inline_link_in_link "[a [b](c) d](e)" +nodes: [Text(Text { value: "[a ", @1:1-1:4 }), Link(Link { url: Url("c"), title: None, values: [Text(Text { value: "b", @1:5-1:6 })], @1:4-1:10 }), Text(Text { value: " d](e)", @1:10-1:16 })] +html: "

    [a b d](e)

    " +## inline_emphasis_link "*[a](b)* **[c][a]**" +nodes: [Emphasis(Emphasis { values: [Link(Link { url: Url("b"), title: None, values: [Text(Text { value: "a", @1:3-1:4 })], @1:2-1:8 })], @1:1-1:9 }), Text(Text { value: " ", @1:9-1:10 }), Strong(Strong { values: [Text(Text { value: "[c][a]", @1:12-1:18 })], @1:10-1:20 })] +html: "

    a [c][a]

    " +## inline_html "t " +nodes: [Html(Html { value: "", @1:1-1:13 }), Text(Text { value: "t", @1:13-1:14 }), Html(Html { value: "", @1:14-1:18 }), Text(Text { value: " ", @1:18-1:19 }), Html(Html { value: "", @1:19-2:8 })] +html: "

    t

    " +## inline_html_misc " " +nodes: [Text(Text { value: "<", @1:1-1:2 }), Link(Link { url: Url("http://a"), title: None, values: [Text(Text { value: "http://a", @1:2-1:10 })], @1:2-1:10 }), Text(Text { value: " b> ", @1:10-1:14 }), Link(Link { url: Url("mailto:a+b@c"), title: None, values: [Text(Text { value: "a+b@c", @1:15-1:20 })], @1:14-1:21 }), Text(Text { value: " ", @1:21-1:22 }), Html(Html { value: "", @1:22-1:26 }), Text(Text { value: " ", @1:26-1:27 }), Html(Html { value: "", @1:27-1:31 }), Text(Text { value: " ", @1:31-1:32 }), Html(Html { value: "", @1:32-1:37 }), Text(Text { value: " ", @1:37-1:38 }), Html(Html { value: "", @1:38-1:51 }), Text(Text { value: " ", @1:51-1:52 }), Html(Html { value: "", @1:52-1:58 })] +html: "

    <http://a b> a+b@c

    " +## inline_nested_emphasis "*a **b** c* **a *b* c**" +nodes: [Emphasis(Emphasis { values: [Text(Text { value: "a ", @1:2-1:4 }), Strong(Strong { values: [Text(Text { value: "b", @1:6-1:7 })], @1:4-1:9 }), Text(Text { value: " c", @1:9-1:11 })], @1:1-1:12 }), Text(Text { value: " ", @1:12-1:13 }), Strong(Strong { values: [Text(Text { value: "a ", @1:15-1:17 }), Emphasis(Emphasis { values: [Text(Text { value: "b", @1:18-1:19 })], @1:17-1:20 }), Text(Text { value: " c", @1:20-1:22 })], @1:13-1:24 })] +html: "

    a b c a b c

    " +## inline_intraword "a_b_c a*b*c _a_b" +nodes: [Text(Text { value: "a_b_c a", @1:1-1:8 }), Emphasis(Emphasis { values: [Text(Text { value: "b", @1:9-1:10 })], @1:8-1:11 }), Text(Text { value: "c _a_b", @1:11-1:17 })] +html: "

    a_b_c abc _a_b

    " +## inline_unmatched "*a **b _c ~d [e ![f" +nodes: [Text(Text { value: "*a **b _c ~d [e ![f", @1:1-1:20 })] +html: "

    *a **b _c ~d [e ![f

    " +## inline_rule_of_three "*foo**bar**baz* ***a** b*" +nodes: [Emphasis(Emphasis { values: [Text(Text { value: "foo", @1:2-1:5 }), Strong(Strong { values: [Text(Text { value: "bar", @1:7-1:10 })], @1:5-1:12 }), Text(Text { value: "baz", @1:12-1:15 })], @1:1-1:16 }), Text(Text { value: " ", @1:16-1:17 }), Emphasis(Emphasis { values: [Strong(Strong { values: [Text(Text { value: "a", @1:20-1:21 })], @1:18-1:23 }), Text(Text { value: " b", @1:23-1:25 })], @1:17-1:26 })] +html: "

    foobarbaz a b

    " +## inline_heading "# a *b* `c`\n" +nodes: [Heading(Heading { depth: 1, values: [Text(Text { value: "a ", @1:3-1:5 }), Emphasis(Emphasis { values: [Text(Text { value: "b", @1:6-1:7 })], @1:5-1:8 }), Text(Text { value: " ", @1:8-1:9 }), CodeInline(CodeInline { value: "c", @1:9-1:12 })], @1:1-1:12 })] +html: "

    a b c

    \n" +## inline_setext_heading "a *b*\nc\n===\n" +nodes: [Heading(Heading { depth: 1, values: [Text(Text { value: "a ", @1:1-1:3 }), Emphasis(Emphasis { values: [Text(Text { value: "b", @1:4-1:5 })], @1:3-1:6 }), Text(Text { value: "\nc", @1:6-2:2 })], @1:1-3:4 })] +html: "

    a b\nc

    \n" +## inline_list_item "- a *b*\n c **d**\n" +nodes: [List(List { values: [Text(Text { value: "a ", @1:3-1:5 }), Emphasis(Emphasis { values: [Text(Text { value: "b", @1:6-1:7 })], @1:5-1:8 }), Text(Text { value: "\nc ", @1:8-2:5 }), Strong(Strong { values: [Text(Text { value: "d", @2:7-2:8 })], @2:5-2:10 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-2:10 })] +html: "
      \n
    • a b\nc d
    • \n
    \n" +## inline_table_cell "| a *b* | `c\\|d` |\n|-|-|\n" +nodes: [TableCell(TableCell { values: [Text(Text { value: "a ", @1:3-1:5 }), Emphasis(Emphasis { values: [Text(Text { value: "b", @1:6-1:7 })], @1:5-1:8 })]:0, row: 0, @1:1-1:9 }), TableCell(TableCell { values: [CodeInline(CodeInline { value: "c|d", @1:11-1:17 })]:1, row: 0, @1:9-1:19 }), TableAlign(TableAlign { align: [None, None], @2:1-2:1 })] +html: "\n\n\n\n\n\n\n
    a bc|d
    \n" +## inline_multibyte "あ*い*う `え` [お](か)" +nodes: [Text(Text { value: "あ", @1:1-1:4 }), Emphasis(Emphasis { values: [Text(Text { value: "い", @1:5-1:8 })], @1:4-1:9 }), Text(Text { value: "う ", @1:9-1:13 }), CodeInline(CodeInline { value: "え", @1:13-1:18 }), Text(Text { value: " ", @1:18-1:19 }), Link(Link { url: Url("か"), title: None, values: [Text(Text { value: "お", @1:20-1:23 })], @1:19-1:29 })] +html: "

    あいう え お

    " +## inline_crlf_break "a \r\nb\r\n" +nodes: [Text(Text { value: "a", @1:1-1:2 }), Break(Break { @1:2-2:1 }), Text(Text { value: "b", @2:1-2:2 })] +html: "

    a
    \r\nb

    \r\n" +## math_multiline "x$\nx$" +nodes: [Text(Text { value: "x", @1:1-1:2 }), MathInline(MathInline { value: "\nx", @1:2-2:3 })] +html: "

    x x

    " +## decl_email "x" +nodes: [Text(Text { value: "x", @1:1-1:2 }), Html(Html { value: "", @1:2-1:11 })] +html: "

    x

    " +## code_multiline "x`\nx`" +nodes: [Text(Text { value: "x", @1:1-1:2 }), CodeInline(CodeInline { value: "\nx", @1:2-2:3 })] +html: "

    x x

    " +## def_basic "[a]: b" +nodes: [Definition(Definition { @1:1-1:7, url: Url("b"), title: None, ident: "a", label: Some("a") })] +html: "" +## def_title "[a]: 'T'" +nodes: [Definition(Definition { @1:1-1:15, url: Url("b c"), title: Some(Title("T")), ident: "a", label: Some("a") })] +html: "" +## def_next_lines "[a]:\nb\n\"t\"" +nodes: [Definition(Definition { @1:1-3:4, url: Url("b"), title: Some(Title("t")), ident: "a", label: Some("a") })] +html: "" +## def_title_junk_next_line "[a]: b\n\"t\" x" +nodes: [Definition(Definition { @1:1-1:7, url: Url("b"), title: None, ident: "a", label: Some("a") }), Text(Text { value: "\"t\" x", @2:1-2:6 })] +html: "

    "t" x

    " +## def_title_junk_same_line "[a]: b \"t\" x" +nodes: [Text(Text { value: "[a]: b \"t\" x", @1:1-1:13 })] +html: "

    [a]: b "t" x

    " +## def_then_paragraph "[a]: b\nc" +nodes: [Definition(Definition { @1:1-1:7, url: Url("b"), title: None, ident: "a", label: Some("a") }), Text(Text { value: "c", @2:1-2:2 })] +html: "

    c

    " +## def_two_then_text "[a]: b\n[c]: d\ntext" +nodes: [Definition(Definition { @1:1-1:7, url: Url("b"), title: None, ident: "a", label: Some("a") }), Definition(Definition { @2:1-2:7, url: Url("d"), title: None, ident: "c", label: Some("c") }), Text(Text { value: "text", @3:1-3:5 })] +html: "

    text

    " +## def_not_at_start "text\n[a]: b" +nodes: [Text(Text { value: "text\n[a]: b", @1:1-2:7 })] +html: "

    text\n[a]: b

    " +## def_then_setext_equals "[a]: b\n=== " +nodes: [Definition(Definition { @1:1-1:7, url: Url("b"), title: None, ident: "a", label: Some("a") }), Text(Text { value: "===", @2:1-2:4 })] +html: "

    ===

    " +## def_then_setext_dashes "[a]: b\n---" +nodes: [Definition(Definition { @1:1-1:7, url: Url("b"), title: None, ident: "a", label: Some("a") }), Text(Text { value: "---", @2:1-2:4 })] +html: "

    ---

    " +## def_indent1 " [a]: b" +nodes: [Definition(Definition { @1:1-1:8, url: Url("b"), title: None, ident: "a", label: Some("a") })] +html: "" +## def_indent3 " [a]: b" +nodes: [Definition(Definition { @1:1-1:10, url: Url("b"), title: None, ident: "a", label: Some("a") })] +html: "" +## def_indent4 " [a]: b" +nodes: [Code(Code { value: "[a]: b", lang: None, @1:1-1:11, meta: None, fence: false })] +html: "
    [a]: b\n
    " +## def_in_quote "> [a]: b" +nodes: [Blockquote(Blockquote { values: [Definition(Definition { @1:3-1:9, url: Url("b"), title: None, ident: "a", label: Some("a") })], @1:1-1:9 })] +html: "
    \n
    " +## def_in_list "- [a]: b" +nodes: [List(List { values: [Definition(Definition { @1:3-1:9, url: Url("b"), title: None, ident: "a", label: Some("a") })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-1:9 })] +html: "
      \n
    • \n
    " +## def_no_dest "[a]:" +nodes: [Text(Text { value: "[a]:", @1:1-1:5 })] +html: "

    [a]:

    " +## def_empty_label "[]: b" +nodes: [Text(Text { value: "[]: b", @1:1-1:6 })] +html: "

    []: b

    " +## def_blank_label "[ ]: b" +nodes: [Text(Text { value: "[ ]: b", @1:1-1:7 })] +html: "

    [ ]: b

    " +## def_escaped_label "[a\\]b]: c" +nodes: [Definition(Definition { @1:1-1:10, url: Url("c"), title: None, ident: "a\\]b", label: Some("a]b") })] +html: "" +## def_multiline_title "[a]: b \"t\nt\"" +nodes: [Definition(Definition { @1:1-2:3, url: Url("b"), title: Some(Title("t\nt")), ident: "a", label: Some("a") })] +html: "" +## def_title_indented "[a]: b\n \"t\"" +nodes: [Definition(Definition { @1:1-2:6, url: Url("b"), title: Some(Title("t")), ident: "a", label: Some("a") })] +html: "" +## def_no_space "[a]:b" +nodes: [Definition(Definition { @1:1-1:6, url: Url("b"), title: None, ident: "a", label: Some("a") })] +html: "" +## def_dest_junk "[a]: b c" +nodes: [Text(Text { value: "[a]: b c", @1:1-1:9 })] +html: "

    [a]: b c

    " +## def_duplicate_labels "[A b]: c\n[a b]: d" +nodes: [Definition(Definition { @1:1-1:10, url: Url("c"), title: None, ident: "ab", label: Some("A b") }), Definition(Definition { @2:1-2:9, url: Url("d"), title: None, ident: "ab", label: Some("a b") })] +html: "" +## def_empty_angle "[a]: <>" +nodes: [Definition(Definition { @1:1-1:8, url: Url(""), title: None, ident: "a", label: Some("a") })] +html: "" +## def_quote_in_dest "[a]: 'x" +nodes: [Definition(Definition { @1:1-1:8, url: Url("'x"), title: None, ident: "a", label: Some("a") })] +html: "" +## def_use_shortcut "[a]: /u\n\n[a] [a][] [b][a] [a][b] [A b]" +nodes: [Definition(Definition { @1:1-1:8, url: Url("/u"), title: None, ident: "a", label: Some("a") }), LinkRef(LinkRef { ident: "a", label: Some("a"), values: [Text(Text { value: "a", @3:2-3:3 })], @3:1-3:4 }), Text(Text { value: " ", @3:4-3:5 }), LinkRef(LinkRef { ident: "a", label: Some("a"), values: [Text(Text { value: "a", @3:6-3:7 })], @3:5-3:10 }), Text(Text { value: " ", @3:10-3:11 }), LinkRef(LinkRef { ident: "a", label: Some("a"), values: [Text(Text { value: "b", @3:12-3:13 })], @3:11-3:17 }), Text(Text { value: " [a][b] [A b]", @3:17-3:30 })] +html: "

    a a b [a][b] [A b]

    " +## def_use_normalized "[ab cd]: /u\n\n[AB CD] [ab\ncd]" +nodes: [Definition(Definition { @1:1-1:12, url: Url("/u"), title: None, ident: "abcd", label: Some("ab cd") }), LinkRef(LinkRef { ident: "abcd", label: Some("AB CD"), values: [Text(Text { value: "AB CD", @3:2-3:9 })], @3:1-3:10 }), Text(Text { value: " ", @3:10-3:11 }), LinkRef(LinkRef { ident: "abcd", label: Some("ab\ncd"), values: [Text(Text { value: "ab\ncd", @3:12-4:3 })], @3:11-4:4 })] +html: "

    AB CD ab\ncd

    " +## def_use_before "[a] and [b][a]\n\n[a]: /u \"T\"" +nodes: [LinkRef(LinkRef { ident: "a", label: Some("a"), values: [Text(Text { value: "a", @1:2-1:3 })], @1:1-1:4 }), Text(Text { value: " and ", @1:4-1:9 }), LinkRef(LinkRef { ident: "a", label: Some("a"), values: [Text(Text { value: "b", @1:10-1:11 })], @1:9-1:15 }), Definition(Definition { @3:1-3:12, url: Url("/u"), title: Some(Title("T")), ident: "a", label: Some("a") })] +html: "

    a and b

    \n" +## def_use_in_link_text "[a]: b\n\n[a](x) [a]" +nodes: [Definition(Definition { @1:1-1:7, url: Url("b"), title: None, ident: "a", label: Some("a") }), Link(Link { url: Url("x"), title: None, values: [Text(Text { value: "a", @3:2-3:3 })], @3:1-3:7 }), Text(Text { value: " ", @3:7-3:8 }), LinkRef(LinkRef { ident: "a", label: Some("a"), values: [Text(Text { value: "a", @3:9-3:10 })], @3:8-3:11 })] +html: "

    a a

    " +## def_use_image "[x]: /u\n\n![x] ![y][x] ![x][]" +nodes: [Definition(Definition { @1:1-1:8, url: Url("/u"), title: None, ident: "x", label: Some("x") }), ImageRef(ImageRef { alt: "x", ident: "x", label: Some("x"), @3:1-3:5 }), Text(Text { value: " ", @3:5-3:6 }), ImageRef(ImageRef { alt: "y", ident: "x", label: Some("x"), @3:6-3:13 }), Text(Text { value: " ", @3:13-3:14 }), ImageRef(ImageRef { alt: "x", ident: "x", label: Some("x"), @3:14-3:20 })] +html: "

    \"x\" \"y\" \"x\"

    " +## def_link_in_emphasis "[a]: b\n\n*[a]* **[c][a]**" +nodes: [Definition(Definition { @1:1-1:7, url: Url("b"), title: None, ident: "a", label: Some("a") }), Emphasis(Emphasis { values: [LinkRef(LinkRef { ident: "a", label: Some("a"), values: [Text(Text { value: "a", @3:3-3:4 })], @3:2-3:5 })], @3:1-3:6 }), Text(Text { value: " ", @3:6-3:7 }), Strong(Strong { values: [LinkRef(LinkRef { ident: "a", label: Some("a"), values: [Text(Text { value: "c", @3:10-3:11 })], @3:9-3:15 })], @3:7-3:17 })] +html: "

    a c

    " +## fn_basic "[^a]: b" +nodes: [Footnote(Footnote { ident: "a", values: [Text(Text { value: "b", @1:7-1:8 })], @1:1-1:8 })] +html: "" +## fn_lazy "[^a]: b\nc" +nodes: [Footnote(Footnote { ident: "a", values: [Text(Text { value: "b\nc", @1:7-2:2 })], @1:1-2:2 })] +html: "" +## fn_indented_continuation "[^a]: b\n c" +nodes: [Footnote(Footnote { ident: "a", values: [Text(Text { value: "b\nc", @1:7-2:6 })], @1:1-2:6 })] +html: "" +## fn_two_paragraphs "[^a]: b\n\n c" +nodes: [Footnote(Footnote { ident: "a", values: [Text(Text { value: "b", @1:7-1:8 }), Text(Text { value: "c", @3:5-3:6 })], @1:1-3:6 })] +html: "" +## fn_two_paragraphs_then_text "[^a]: b\n\n c\n\nd" +nodes: [Footnote(Footnote { ident: "a", values: [Text(Text { value: "b", @1:7-1:8 }), Text(Text { value: "c", @3:5-3:6 })], @1:1-4:1 }), Text(Text { value: "d", @5:1-5:2 })] +html: "

    d

    " +## fn_two_space_continuation "[^a]: b\n c" +nodes: [Footnote(Footnote { ident: "a", values: [Text(Text { value: "b\nc", @1:7-2:4 })], @1:1-2:4 })] +html: "" +## fn_empty "[^a]:" +nodes: [Footnote(Footnote { ident: "a", values: [], @1:1-1:6 })] +html: "" +## fn_empty_space "[^a]: " +nodes: [Footnote(Footnote { ident: "a", values: [], @1:1-1:7 })] +html: "" +## fn_content_next_line "[^a]:\n b" +nodes: [Footnote(Footnote { ident: "a", values: [Text(Text { value: "b", @2:5-2:6 })], @1:1-2:6 })] +html: "" +## fn_two_adjacent "[^a]: b\n[^c]: d" +nodes: [Footnote(Footnote { ident: "a", values: [Text(Text { value: "b", @1:7-1:8 })], @1:1-1:8 }), Footnote(Footnote { ident: "c", values: [Text(Text { value: "d", @2:7-2:8 })], @2:1-2:8 })] +html: "" +## fn_two_separated "[^a]: b\n\n[^c]: d" +nodes: [Footnote(Footnote { ident: "a", values: [Text(Text { value: "b", @1:7-1:8 })], @1:1-2:1 }), Footnote(Footnote { ident: "c", values: [Text(Text { value: "d", @3:7-3:8 })], @3:1-3:8 })] +html: "" +## fn_label_space "[^ a]: b" +nodes: [Definition(Definition { @1:1-1:9, url: Url("b"), title: None, ident: "^a", label: Some("^ a") })] +html: "" +## fn_label_inner_space "[^a b]: b" +nodes: [Definition(Definition { @1:1-1:10, url: Url("b"), title: None, ident: "^ab", label: Some("^a b") })] +html: "" +## fn_label_empty "[^]: b" +nodes: [Definition(Definition { @1:1-1:7, url: Url("b"), title: None, ident: "^", label: Some("^") })] +html: "" +## fn_indent1 " [^a]: b" +nodes: [Footnote(Footnote { ident: "a", values: [Text(Text { value: "b", @1:8-1:9 })], @1:1-1:9 })] +html: "" +## fn_indent4 " [^a]: b" +nodes: [Code(Code { value: "[^a]: b", lang: None, @1:1-1:12, meta: None, fence: false })] +html: "
    [^a]: b\n
    " +## fn_in_quote "> [^a]: b" +nodes: [Blockquote(Blockquote { values: [Footnote(Footnote { ident: "a", values: [Text(Text { value: "b", @1:9-1:10 })], @1:3-1:10 })], @1:1-1:10 })] +html: "
    \n
    " +## fn_in_list "- [^a]: b" +nodes: [List(List { values: [Footnote(Footnote { ident: "a", values: [Text(Text { value: "b", @1:9-1:10 })], @1:3-1:10 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-1:10 })] +html: "
      \n
    • \n
    " +## fn_then_quote "[^a]: b\n> q" +nodes: [Footnote(Footnote { ident: "a", values: [Text(Text { value: "b", @1:7-1:8 })], @1:1-1:8 }), Blockquote(Blockquote { values: [Text(Text { value: "q", @2:3-2:4 })], @2:1-2:4 })] +html: "
    \n

    q

    \n
    " +## fn_then_list "[^a]: b\n- x" +nodes: [Footnote(Footnote { ident: "a", values: [Text(Text { value: "b", @1:7-1:8 })], @1:1-1:8 }), List(List { values: [Text(Text { value: "x", @2:3-2:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @2:3-2:4 })] +html: "
      \n
    • x
    • \n
    " +## fn_heading "[^a]: # h" +nodes: [Footnote(Footnote { ident: "a", values: [Heading(Heading { depth: 1, values: [Text(Text { value: "h", @1:9-1:10 })], @1:7-1:10 })], @1:1-1:10 })] +html: "" +## fn_list "[^a]: - x\n - y" +nodes: [Footnote(Footnote { ident: "a", values: [List(List { values: [Text(Text { value: "x", @1:9-1:10 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:9-1:10 }), List(List { values: [Text(Text { value: "y", @2:7-2:8 })], index: 1, level: 0, ordered: false, checked: None, spread: false, start: None, @2:7-2:8 })], @1:1-2:8 })] +html: "" +## fn_fence "[^a]: b\n ```\n x\n ```" +nodes: [Footnote(Footnote { ident: "a", values: [Text(Text { value: "b", @1:7-1:8 }), Code(Code { value: "x", lang: None, @2:5-4:8, meta: None, fence: true })], @1:1-4:8 })] +html: "" +## fn_ref_case "[^A]: b\n\nx[^a]" +nodes: [Footnote(Footnote { ident: "a", values: [Text(Text { value: "b", @1:7-1:8 })], @1:1-2:1 }), Text(Text { value: "x", @3:1-3:2 }), FootnoteRef(FootnoteRef { ident: "a", label: Some("a"), @3:2-3:6 })] +html: "

    x1

    \n

    Footnotes

    \n
      \n
    1. \n

      b ↩

      \n
    2. \n
    \n
    \n" +## fn_no_space "[^a]:b" +nodes: [Footnote(Footnote { ident: "a", values: [Text(Text { value: "b", @1:6-1:7 })], @1:1-1:7 })] +html: "" +## fn_interrupts_paragraph "text\n[^a]: b" +nodes: [Text(Text { value: "text", @1:1-1:5 }), Footnote(Footnote { ident: "a", values: [Text(Text { value: "b", @2:7-2:8 })], @2:1-2:8 })] +html: "

    text

    \n" +## fn_then_dashes "[^a]: b\n---" +nodes: [Footnote(Footnote { ident: "a", values: [Text(Text { value: "b", @1:7-1:8 })], @1:1-1:8 }), HorizontalRule(HorizontalRule { @2:1-2:4 })] +html: "
    " +## fn_then_equals "[^a]: b\n===" +nodes: [Footnote(Footnote { ident: "a", values: [Text(Text { value: "b\n===", @1:7-2:4 })], @1:1-2:4 })] +html: "" +## fn_ref_basic "[^a]\n\n[^a]: note" +nodes: [FootnoteRef(FootnoteRef { ident: "a", label: Some("a"), @1:1-1:5 }), Footnote(Footnote { ident: "a", values: [Text(Text { value: "note", @3:7-3:11 })], @3:1-3:11 })] +html: "

    1

    \n

    Footnotes

    \n
      \n
    1. \n

      note ↩

      \n
    2. \n
    \n
    \n" +## fn_ref_no_def "[^b] no def" +nodes: [Text(Text { value: "[^b] no def", @1:1-1:12 })] +html: "

    [^b] no def

    " +## fn_ref_inline "x[^a] and [^a]!\n\n[^a]: note" +nodes: [Text(Text { value: "x", @1:1-1:2 }), FootnoteRef(FootnoteRef { ident: "a", label: Some("a"), @1:2-1:6 }), Text(Text { value: " and ", @1:6-1:11 }), FootnoteRef(FootnoteRef { ident: "a", label: Some("a"), @1:11-1:15 }), Text(Text { value: "!", @1:15-1:16 }), Footnote(Footnote { ident: "a", values: [Text(Text { value: "note", @3:7-3:11 })], @3:1-3:11 })] +html: "

    x1 and 1!

    \n

    Footnotes

    \n
      \n
    1. \n

      note ↩ ↩2

      \n
    2. \n
    \n
    \n" +## fn_ref_in_link_text "[a[^a]](b)\n\n[^a]: note" +nodes: [Text(Text { value: "[a", @1:1-1:3 }), FootnoteRef(FootnoteRef { ident: "a", label: Some("a"), @1:3-1:7 }), Text(Text { value: "](b)", @1:7-1:11 }), Footnote(Footnote { ident: "a", values: [Text(Text { value: "note", @3:7-3:11 })], @3:1-3:11 })] +html: "

    [a1](b)

    \n

    Footnotes

    \n
      \n
    1. \n

      note ↩

      \n
    2. \n
    \n
    \n" +## html_div "
    \nx\n
    \n\ny" +nodes: [Html(Html { value: "
    \nx\n
    ", @1:1-3:7 }), Text(Text { value: "y", @5:1-5:2 })] +html: "
    \nx\n
    \n

    y

    " +## html_indented "
    \n x\n" +nodes: [Html(Html { value: "
    \n x", @1:1-2:4 })] +html: "
    \n x\n" +## html_oneline "
    x
    " +nodes: [Html(Html { value: "
    x
    ", @1:1-1:13 })] +html: "
    x
    " +## html_script "\nz" +nodes: [Html(Html { value: "", @1:1-5:10 }), Text(Text { value: "z", @6:1-6:2 })] +html: "\n

    z

    " +## html_pre "
    \na\n\nb
    \nc" +nodes: [Html(Html { value: "
    \na\n\nb
    ", @1:1-4:8 }), Text(Text { value: "c", @5:1-5:2 })] +html: "
    \na\n\nb
    \n

    c

    " +## html_style_inline_end " y\nz" +nodes: [Html(Html { value: " y", @1:1-1:19 }), Text(Text { value: "z", @2:1-2:2 })] +html: " y\n

    z

    " +## html_comment "\nx" +nodes: [Html(Html { value: "", @1:1-3:7 }), Text(Text { value: "x", @4:1-4:2 })] +html: "\n

    x

    " +## html_instruction "\ny" +nodes: [Html(Html { value: "", @1:1-2:5 }), Text(Text { value: "y", @3:1-3:2 })] +html: "\n

    y

    " +## html_declaration "\nx" +nodes: [Html(Html { value: "", @1:1-1:16 }), Text(Text { value: "x", @2:1-2:2 })] +html: "\n

    x

    " +## html_cdata "\ny" +nodes: [Html(Html { value: "", @1:1-3:5 }), Text(Text { value: "y", @4:1-4:2 })] +html: "\n

    y

    " +## html_complete_tag "\ny" +nodes: [Html(Html { value: "\ny", @1:1-2:2 })] +html: "\ny" +## html_tag_then_text "y" +nodes: [Html(Html { value: "", @1:1-1:13 }), Text(Text { value: "y", @1:13-1:14 })] +html: "

    y

    " +## html_interrupt_basic "text\n\nz" +## html_multiline_open "\nz" +nodes: [Html(Html { value: "\nz", @1:1-3:2 })] +html: "\nz" +## html_in_quote ">
    \n> x" +nodes: [Blockquote(Blockquote { values: [Html(Html { value: "
    \nx", @1:3-2:4 })], @1:1-2:4 })] +html: "
    \n
    \nx\n
    " +## html_in_list "-
    \n x" +nodes: [List(List { values: [Html(Html { value: "
    \nx", @1:3-2:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-2:4 })] +html: "
      \n
    • \n
      \nx\n
    • \n
    " +## html_uppercase "
    \nx" +nodes: [Html(Html { value: "
    \nx", @1:1-2:2 })] +html: "
    \nx" +## html_self_closing "
    \nx" +nodes: [Html(Html { value: "
    \nx", @1:1-2:2 })] +html: "
    \nx" +## html_ins "\nx" +nodes: [Html(Html { value: "\nx", @1:1-2:2 })] +html: "\nx" +## html_br "
    \nx" +nodes: [Html(Html { value: "
    \nx", @1:1-2:2 })] +html: "
    \nx" +## html_textarea "" +nodes: [Html(Html { value: "", @1:1-4:13 })] +html: "" +## html_table_two_blocks "\n\n\n
    " +nodes: [Html(Html { value: "\n", @1:1-2:5 }), Html(Html { value: "
    ", @4:1-4:5 })] +html: "\n\n"), rows > 0); + } + + #[test] + fn footnotes_are_numbered_from_one_in_the_order_of_their_references(order in Just(vec![0usize, 1, 2, 3]).prop_shuffle()) { + let references = order.iter().map(|id| format!("r[^n{id}]")).collect::>().join(" "); + let definitions = (0..4).map(|id| format!("[^n{id}]: d{id}")).collect::>().join("\n"); + let html = to_html(&format!("{references}\n\n{definitions}\n")); + for (number, id) in order.iter().enumerate() { + let call = format!("href=\"#user-content-fn-n{id}\" id=\"user-content-fnref-n{id}\" data-footnote-ref=\"\" aria-describedby=\"footnote-label\">{}", number + 1); + prop_assert!(html.contains(&call), "{html}"); + } + let positions = order.iter().map(|id| html.find(&format!("
  • ")).unwrap()).collect::>(); + prop_assert!(positions.windows(2).all(|pair| pair[0] < pair[1]), "{html}"); + } +} From 7c73078a424fc80122a0935fe354140ab15cb45d Mon Sep 17 00:00:00 2001 From: harehare Date: Thu, 8 Oct 2026 13:05:37 +0900 Subject: [PATCH 07/88] =?UTF-8?q?=E2=9C=A8=20feat(markdown):=20improve=20t?= =?UTF-8?q?he=20native=20parser's=20conformance,=20HTML=20output=20and=20p?= =?UTF-8?q?erformance?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Check the parser against the CommonMark 0.31.2 spec, the GFM spec and its extensions, and reference implementations (commonmark.js, cmark-gfm, micromark), on generated input and about 99k real documents, and fix what differed. Source style: - keep the marker of a list item (-, +, *, . or )) and the character of a thematic break, and write them back - RenderOptions.list_style is now optional, and None keeps the original bullet. --list-style no longer defaults to dash Performance (about 30% faster on real documents): - cache whether a line has a tab, check blank lines by bytes, skip the tab scan for inline positions, and skip plain text in the inline scanner with a table - add benchmarks for real documents and for many short lines CommonMark: - emphasis opens and closes by the flanking rules, and the rule of three goes by the whole run - a heading keeps a # that starts its text, and the lines of a code span lose their leading whitespace - a paragraph is continued lazily by a line of backticks, tildes, underscores or an angle bracket at the end of the document - drop the restrictions on the first line of a container, on markers on the second line of a paragraph, on dashes after a setext heading, on empty nested items in loose lists, and on the first line of indented code - tabs that a container consumes in part stay in indented code as spaces, and the alt text of an image includes the alt text of an image in it - reference labels have their whitespace collapsed, and links in link text are unwrapped GFM: - write the language of a code block as lang, the checkbox attributes in the order of the spec, and filter the tags script, style, textarea and a few more - read the checkbox of a task as the start of its text, so an empty task and what follows the checkbox read as they do in GitHub - link an email address after a slash, allow emoji in a domain, and do not link an email address followed by @ - a footnote reference can follow a !, and a footnote definition goes on after a blank line that follows its marker - test the examples of the CommonMark and GFM extensions specs Obsidian: - read a wikilink and an embed when the text is scanned, so emphasis cannot split them, an escape stops them, and their position is their own - a callout has the fold marker as its own field, a title with markup, and a type of letters, digits, - and _ MDX: - read import and export blocks as ESM Rendering: - do not panic on nodes with overlapping positions - keep tables in quotes and list items, nested items in order, the checkbox, links, references, emphasis in strong, footnotes and empty fenced code intact - write a rule of dashes at the start as *** when another one would read as frontmatter, and escape pipes in table cells --- crates/mq-lang/src/runtime/builtin.rs | 78 +- .../mq-lang/src/runtime/builtin/compress.rs | 1 + crates/mq-lang/src/runtime/builtin/convert.rs | 2 + crates/mq-lang/tests/integration_tests.rs | 16 +- crates/mq-markdown/benches/benchmark.rs | 90 ++- crates/mq-markdown/src/lib.rs | 2 +- crates/mq-markdown/src/markdown.rs | 119 +-- crates/mq-markdown/src/markdown/list_order.rs | 124 +++ .../mq-markdown/src/markdown/table_layout.rs | 18 +- crates/mq-markdown/src/node.rs | 710 ++++++++++-------- crates/mq-markdown/src/node/attr_value.rs | 2 + crates/mq-markdown/src/parser.rs | 148 ++-- crates/mq-markdown/src/parser/block.rs | 259 ++----- crates/mq-markdown/src/parser/code.rs | 23 +- crates/mq-markdown/src/parser/inline.rs | 107 ++- .../mq-markdown/src/parser/inline/emphasis.rs | 27 +- crates/mq-markdown/src/parser/inline/link.rs | 58 +- .../mq-markdown/src/parser/inline/literal.rs | 13 +- crates/mq-markdown/src/parser/line.rs | 15 +- crates/mq-markdown/src/parser/render_html.rs | 52 +- crates/mq-markdown/src/parser/resolve.rs | 1 + crates/mq-markdown/src/parser/snapshots.txt | 226 +++--- crates/mq-markdown/src/parser/tree.rs | 28 +- .../mq-markdown/tests/html_rendering_tests.rs | 76 +- crates/mq-markdown/tests/obsidian_tests.rs | 129 ++++ .../tests/parser_structure_tests.rs | 85 +++ crates/mq-run/src/cli.rs | 12 +- crates/mq-run/tests/integration_tests.rs | 7 +- crates/mq-wasm/src/script.rs | 13 +- 29 files changed, 1542 insertions(+), 899 deletions(-) create mode 100644 crates/mq-markdown/src/markdown/list_order.rs create mode 100644 crates/mq-markdown/tests/obsidian_tests.rs diff --git a/crates/mq-lang/src/runtime/builtin.rs b/crates/mq-lang/src/runtime/builtin.rs index 2170e6975..01445af2c 100644 --- a/crates/mq-lang/src/runtime/builtin.rs +++ b/crates/mq-lang/src/runtime/builtin.rs @@ -3200,7 +3200,11 @@ fn to_h_impl(_: &Ident, _: &RuntimeValue, args: Args, _: &SharedEnv) -> Result Result { - Ok(mq_markdown::Node::HorizontalRule(mq_markdown::HorizontalRule { position: None }).into()) + Ok(mq_markdown::Node::HorizontalRule(mq_markdown::HorizontalRule { + marker: None, + position: None, + }) + .into()) } #[mq_macros::mq_fn(name = "to_link", params = Fixed(3))] @@ -3386,6 +3390,7 @@ fn to_callout_impl(_: &Ident, _: &RuntimeValue, args: Args, _: &SharedEnv) -> Re RuntimeValue::String(kind), RuntimeValue::String(title), ] => Ok(mq_markdown::Node::Callout(mq_markdown::Callout { + fold: None, kind: kind.to_uppercase(), title: if title.is_empty() { None @@ -3398,6 +3403,7 @@ fn to_callout_impl(_: &Ident, _: &RuntimeValue, args: Args, _: &SharedEnv) -> Re .into()), [a, RuntimeValue::String(kind), RuntimeValue::String(title)] if !a.is_none() => { Ok(mq_markdown::Node::Callout(mq_markdown::Callout { + fold: None, kind: kind.to_uppercase(), title: if title.is_empty() { None @@ -3504,6 +3510,7 @@ fn to_md_list_impl(_: &Ident, _: &RuntimeValue, args: Args, _: &SharedEnv) -> Re match args.as_slice() { [RuntimeValue::Markdown(node, _), RuntimeValue::Number(level)] => { Ok(mq_markdown::Node::List(mq_markdown::List { + marker: None, values: node.node_values(), index: 0, ordered: false, @@ -3516,6 +3523,7 @@ fn to_md_list_impl(_: &Ident, _: &RuntimeValue, args: Args, _: &SharedEnv) -> Re .into()) } [a, RuntimeValue::Number(level)] if !a.is_none() => Ok(mq_markdown::Node::List(mq_markdown::List { + marker: None, values: vec![a.to_string().into()], index: 0, ordered: false, @@ -7709,62 +7717,62 @@ mod tests { false )] #[case::list_with_matching_index_checked( - Node::List(mq_markdown::List { start: None, spread: false, values: vec!["test".to_string().into()], ordered: false, index: 1, level: 1, checked: Some(true), position: None }), + Node::List(mq_markdown::List { marker: None, start: None, spread: false, values: vec!["test".to_string().into()], ordered: false, index: 1, level: 1, checked: Some(true), position: None }), Selector::List(Some(1), Some(true)), true )] #[case::list_with_wrong_index( - Node::List(mq_markdown::List { start: None, spread: false, values: vec!["test".to_string().into()], ordered: false, index: 1, level: 1, checked: Some(true), position: None }), + Node::List(mq_markdown::List { marker: None, start: None, spread: false, values: vec!["test".to_string().into()], ordered: false, index: 1, level: 1, checked: Some(true), position: None }), Selector::List(Some(2), Some(true)), false )] #[case::list_without_index( - Node::List(mq_markdown::List { start: None, spread: false, values: vec!["test".to_string().into()], ordered: false, index: 1, level: 1, checked: Some(true), position: None }), + Node::List(mq_markdown::List { marker: None, start: None, spread: false, values: vec!["test".to_string().into()], ordered: false, index: 1, level: 1, checked: Some(true), position: None }), Selector::List(None, None), true )] #[case::task_list( - Node::List(mq_markdown::List { start: None, spread: false, values: vec!["test".to_string().into()], ordered: false, index: 1, level: 1, checked: Some(true), position: None }), + Node::List(mq_markdown::List { marker: None, start: None, spread: false, values: vec!["test".to_string().into()], ordered: false, index: 1, level: 1, checked: Some(true), position: None }), Selector::Task, true )] #[case::task_list( - Node::List(mq_markdown::List { start: None, spread: false, values: vec!["test".to_string().into()], ordered: false, index: 1, level: 1, checked: Some(false), position: None }), + Node::List(mq_markdown::List { marker: None, start: None, spread: false, values: vec!["test".to_string().into()], ordered: false, index: 1, level: 1, checked: Some(false), position: None }), Selector::Task, true )] #[case::task_list( - Node::List(mq_markdown::List { start: None, spread: false, values: vec!["test".to_string().into()], ordered: false, index: 1, level: 1, checked: None, position: None }), + Node::List(mq_markdown::List { marker: None, start: None, spread: false, values: vec!["test".to_string().into()], ordered: false, index: 1, level: 1, checked: None, position: None }), Selector::Task, false )] #[case::todo_list( - Node::List(mq_markdown::List { start: None, spread: false, values: vec!["test".to_string().into()], ordered: false, index: 1, level: 1, checked: Some(false), position: None }), + Node::List(mq_markdown::List { marker: None, start: None, spread: false, values: vec!["test".to_string().into()], ordered: false, index: 1, level: 1, checked: Some(false), position: None }), Selector::Todo, true )] #[case::todo_list( - Node::List(mq_markdown::List { start: None, spread: false, values: vec!["test".to_string().into()], ordered: false, index: 1, level: 1, checked: Some(true), position: None }), + Node::List(mq_markdown::List { marker: None, start: None, spread: false, values: vec!["test".to_string().into()], ordered: false, index: 1, level: 1, checked: Some(true), position: None }), Selector::Todo, false )] #[case::todo_list( - Node::List(mq_markdown::List { start: None, spread: false, values: vec!["test".to_string().into()], ordered: false, index: 1, level: 1, checked: None, position: None }), + Node::List(mq_markdown::List { marker: None, start: None, spread: false, values: vec!["test".to_string().into()], ordered: false, index: 1, level: 1, checked: None, position: None }), Selector::Todo, false )] #[case::done_list( - Node::List(mq_markdown::List { start: None, spread: false, values: vec!["test".to_string().into()], ordered: false, index: 1, level: 1, checked: Some(true), position: None }), + Node::List(mq_markdown::List { marker: None, start: None, spread: false, values: vec!["test".to_string().into()], ordered: false, index: 1, level: 1, checked: Some(true), position: None }), Selector::Done, true )] #[case::done_list( - Node::List(mq_markdown::List { start: None, spread: false, values: vec!["test".to_string().into()], ordered: false, index: 1, level: 1, checked: Some(false), position: None }), + Node::List(mq_markdown::List { marker: None, start: None, spread: false, values: vec!["test".to_string().into()], ordered: false, index: 1, level: 1, checked: Some(false), position: None }), Selector::Done, false )] #[case::done_list( - Node::List(mq_markdown::List { start: None, spread: false, values: vec!["test".to_string().into()], ordered: false, index: 1, level: 1, checked: None, position: None }), + Node::List(mq_markdown::List { marker: None, start: None, spread: false, values: vec!["test".to_string().into()], ordered: false, index: 1, level: 1, checked: None, position: None }), Selector::Done, false )] @@ -7819,7 +7827,7 @@ mod tests { true )] #[case::horizontal_rule( - Node::HorizontalRule(mq_markdown::HorizontalRule{ position: None }), + Node::HorizontalRule(mq_markdown::HorizontalRule{ marker: None, position: None }), Selector::HorizontalRule, true )] @@ -9800,31 +9808,31 @@ mod tests { true )] #[case::callout_kind_match( - Node::Callout(mq_markdown::Callout { kind: "NOTE".to_string(), title: None, values: vec![], position: None }), + Node::Callout(mq_markdown::Callout { fold: None, kind: "NOTE".to_string(), title: None, values: vec![], position: None }), Selector::Callout, vec![RuntimeValue::String(Shared::new("NOTE".into()))], true )] #[case::callout_kind_no_match( - Node::Callout(mq_markdown::Callout { kind: "WARNING".to_string(), title: None, values: vec![], position: None }), + Node::Callout(mq_markdown::Callout { fold: None, kind: "WARNING".to_string(), title: None, values: vec![], position: None }), Selector::Callout, vec![RuntimeValue::String(Shared::new("NOTE".into()))], false )] #[case::callout_multi_kind_match( - Node::Callout(mq_markdown::Callout { kind: "TIP".to_string(), title: None, values: vec![], position: None }), + Node::Callout(mq_markdown::Callout { fold: None, kind: "TIP".to_string(), title: None, values: vec![], position: None }), Selector::Callout, vec![RuntimeValue::String(Shared::new("NOTE".into())), RuntimeValue::String(Shared::new("TIP".into()))], true )] #[case::callout_no_args_fallback( - Node::Callout(mq_markdown::Callout { kind: "NOTE".to_string(), title: None, values: vec![], position: None }), + Node::Callout(mq_markdown::Callout { fold: None, kind: "NOTE".to_string(), title: None, values: vec![], position: None }), Selector::Callout, vec![], true )] #[case::callout_non_callout_node( - Node::HorizontalRule(mq_markdown::HorizontalRule { position: None }), + Node::HorizontalRule(mq_markdown::HorizontalRule { marker: None, position: None }), Selector::Callout, vec![RuntimeValue::String(Shared::new("NOTE".into()))], false @@ -9854,7 +9862,7 @@ mod tests { true )] #[case::wikilink_non_wikilink_node( - Node::HorizontalRule(mq_markdown::HorizontalRule { position: None }), + Node::HorizontalRule(mq_markdown::HorizontalRule { marker: None, position: None }), Selector::WikiLink, vec![RuntimeValue::String(Shared::new("Some Page".into()))], false @@ -9884,7 +9892,7 @@ mod tests { true )] #[case::embed_non_embed_node( - Node::HorizontalRule(mq_markdown::HorizontalRule { position: None }), + Node::HorizontalRule(mq_markdown::HorizontalRule { marker: None, position: None }), Selector::Embed, vec![RuntimeValue::String(Shared::new("image.png".into()))], false @@ -9914,7 +9922,7 @@ mod tests { true )] #[case::link_ref_non_link_ref_node( - Node::HorizontalRule(mq_markdown::HorizontalRule { position: None }), + Node::HorizontalRule(mq_markdown::HorizontalRule { marker: None, position: None }), Selector::LinkRef, vec![RuntimeValue::String(Shared::new("ref".into()))], false @@ -9944,7 +9952,7 @@ mod tests { true )] #[case::image_ref_non_image_ref_node( - Node::HorizontalRule(mq_markdown::HorizontalRule { position: None }), + Node::HorizontalRule(mq_markdown::HorizontalRule { marker: None, position: None }), Selector::ImageRef, vec![RuntimeValue::String(Shared::new("ref".into()))], false @@ -9974,7 +9982,7 @@ mod tests { true )] #[case::footnote_ref_non_footnote_ref_node( - Node::HorizontalRule(mq_markdown::HorizontalRule { position: None }), + Node::HorizontalRule(mq_markdown::HorizontalRule { marker: None, position: None }), Selector::FootnoteRef, vec![RuntimeValue::String(Shared::new("1".into()))], false @@ -10004,7 +10012,7 @@ mod tests { true )] #[case::footnote_non_footnote_node( - Node::HorizontalRule(mq_markdown::HorizontalRule { position: None }), + Node::HorizontalRule(mq_markdown::HorizontalRule { marker: None, position: None }), Selector::Footnote, vec![RuntimeValue::String(Shared::new("1".into()))], false @@ -10034,7 +10042,7 @@ mod tests { true )] #[case::definition_non_definition_node( - Node::HorizontalRule(mq_markdown::HorizontalRule { position: None }), + Node::HorizontalRule(mq_markdown::HorizontalRule { marker: None, position: None }), Selector::Definition, vec![RuntimeValue::String(Shared::new("ref".into()))], false @@ -10064,7 +10072,7 @@ mod tests { true )] #[case::mdx_jsx_flow_element_non_matching_node( - Node::HorizontalRule(mq_markdown::HorizontalRule { position: None }), + Node::HorizontalRule(mq_markdown::HorizontalRule { marker: None, position: None }), Selector::MdxJsxFlowElement, vec![RuntimeValue::String(Shared::new("Alert".into()))], false @@ -10094,43 +10102,43 @@ mod tests { true )] #[case::mdx_jsx_text_element_non_matching_node( - Node::HorizontalRule(mq_markdown::HorizontalRule { position: None }), + Node::HorizontalRule(mq_markdown::HorizontalRule { marker: None, position: None }), Selector::MdxJsxTextElement, vec![RuntimeValue::String(Shared::new("Alert".into()))], false )] #[case::non_heading_node( - Node::HorizontalRule(mq_markdown::HorizontalRule { position: None }), + Node::HorizontalRule(mq_markdown::HorizontalRule { marker: None, position: None }), Selector::Heading(None), vec![RuntimeValue::Number(1.into())], false )] #[case::list_index_match( - Node::List(mq_markdown::List { index: 2, level: 0, checked: None, ordered: false, start: None, spread: false, values: vec![], position: None }), + Node::List(mq_markdown::List { marker: None, index: 2, level: 0, checked: None, ordered: false, start: None, spread: false, values: vec![], position: None }), Selector::List(None, None), vec![RuntimeValue::Number(2.into())], true )] #[case::list_index_no_match( - Node::List(mq_markdown::List { index: 0, level: 0, checked: None, ordered: false, start: None, spread: false, values: vec![], position: None }), + Node::List(mq_markdown::List { marker: None, index: 0, level: 0, checked: None, ordered: false, start: None, spread: false, values: vec![], position: None }), Selector::List(None, None), vec![RuntimeValue::Number(1.into())], false )] #[case::list_multi_index_match( - Node::List(mq_markdown::List { index: 3, level: 0, checked: None, ordered: false, start: None, spread: false, values: vec![], position: None }), + Node::List(mq_markdown::List { marker: None, index: 3, level: 0, checked: None, ordered: false, start: None, spread: false, values: vec![], position: None }), Selector::List(None, None), vec![RuntimeValue::Number(1.into()), RuntimeValue::Number(3.into())], true )] #[case::list_no_args_fallback( - Node::List(mq_markdown::List { index: 0, level: 0, checked: None, ordered: false, start: None, spread: false, values: vec![], position: None }), + Node::List(mq_markdown::List { marker: None, index: 0, level: 0, checked: None, ordered: false, start: None, spread: false, values: vec![], position: None }), Selector::List(None, None), vec![], true )] #[case::list_non_list_node( - Node::HorizontalRule(mq_markdown::HorizontalRule { position: None }), + Node::HorizontalRule(mq_markdown::HorizontalRule { marker: None, position: None }), Selector::List(None, None), vec![RuntimeValue::Number(0.into())], false @@ -10166,7 +10174,7 @@ mod tests { true )] #[case::table_non_table_node( - Node::HorizontalRule(mq_markdown::HorizontalRule { position: None }), + Node::HorizontalRule(mq_markdown::HorizontalRule { marker: None, position: None }), Selector::Table(None, None), vec![RuntimeValue::Number(0.into())], false diff --git a/crates/mq-lang/src/runtime/builtin/compress.rs b/crates/mq-lang/src/runtime/builtin/compress.rs index 6f325e51b..8d3b77f2f 100644 --- a/crates/mq-lang/src/runtime/builtin/compress.rs +++ b/crates/mq-lang/src/runtime/builtin/compress.rs @@ -381,6 +381,7 @@ mod tests { fn list_item(value: &str, index: usize) -> Node { Node::List(List { + marker: None, values: vec![text(value)], index, level: 0, diff --git a/crates/mq-lang/src/runtime/builtin/convert.rs b/crates/mq-lang/src/runtime/builtin/convert.rs index 5ed937eb0..5dff169c1 100644 --- a/crates/mq-lang/src/runtime/builtin/convert.rs +++ b/crates/mq-lang/src/runtime/builtin/convert.rs @@ -194,6 +194,7 @@ impl Convert { ), ConvertKind::ListItem => RuntimeValue::Markdown( Shared::new(mq_markdown::Node::List(mq_markdown::List { + marker: None, values: vec![text.into()], index: 0, ordered: false, @@ -230,6 +231,7 @@ impl Convert { ), ConvertKind::HorizontalRule => RuntimeValue::Markdown( Shared::new(mq_markdown::Node::HorizontalRule(mq_markdown::HorizontalRule { + marker: None, position: None, })), None, diff --git a/crates/mq-lang/tests/integration_tests.rs b/crates/mq-lang/tests/integration_tests.rs index dc615eb14..7df52ef79 100644 --- a/crates/mq-lang/tests/integration_tests.rs +++ b/crates/mq-lang/tests/integration_tests.rs @@ -289,7 +289,7 @@ fn to_code_round_trip_preserves_compound_assign_result(mut engine: DefaultEngine | _: \"other\" end ", - vec![RuntimeValue::new_markdown(mq_markdown::Node::List(mq_markdown::List { start: None, spread: false, + vec![RuntimeValue::new_markdown(mq_markdown::Node::List(mq_markdown::List { marker: None, start: None, spread: false, values: vec![], index: 0, level: 1, @@ -308,7 +308,7 @@ fn to_code_round_trip_preserves_compound_assign_result(mut engine: DefaultEngine | _: \"other\" end ", - vec![RuntimeValue::new_markdown(mq_markdown::Node::List(mq_markdown::List { start: None, spread: false, + vec![RuntimeValue::new_markdown(mq_markdown::Node::List(mq_markdown::List { marker: None, start: None, spread: false, values: vec![], index: 0, level: 1, @@ -365,7 +365,7 @@ fn to_code_round_trip_preserves_compound_assign_result(mut engine: DefaultEngine | _: \"other\" end ", - vec![RuntimeValue::new_markdown(mq_markdown::Node::List(mq_markdown::List { start: None, spread: false, + vec![RuntimeValue::new_markdown(mq_markdown::Node::List(mq_markdown::List { marker: None, start: None, spread: false, values: vec![], index: 0, level: 1, @@ -1475,7 +1475,7 @@ fn to_code_round_trip_preserves_compound_assign_result(mut engine: DefaultEngine position: None, }))].into()))] #[case::is_list_true("is_list()", - vec![RuntimeValue::new_markdown(mq_markdown::Node::List(mq_markdown::List { start: None, spread: false, + vec![RuntimeValue::new_markdown(mq_markdown::Node::List(mq_markdown::List { marker: None, start: None, spread: false, values: vec![], position: None, ordered: false, @@ -2375,7 +2375,7 @@ fn to_code_round_trip_preserves_compound_assign_result(mut engine: DefaultEngine }))].into()))] #[case::convert_string_to_list_item_operator("\"Hello\" @ \"-\"", vec![RuntimeValue::None], - Ok(vec![RuntimeValue::new_markdown(mq_markdown::Node::List(mq_markdown::List { start: None, spread: false, + Ok(vec![RuntimeValue::new_markdown(mq_markdown::Node::List(mq_markdown::List { marker: None, start: None, spread: false, values: vec!["Hello".to_string().into()], index: 0, ordered: false, @@ -2397,7 +2397,7 @@ fn to_code_round_trip_preserves_compound_assign_result(mut engine: DefaultEngine }))].into()))] #[case::convert_string_to_horizontal_rule_operator("\"Hello\" @ \"--\"", vec![RuntimeValue::None], - Ok(vec![RuntimeValue::new_markdown(mq_markdown::Node::HorizontalRule(mq_markdown::HorizontalRule { + Ok(vec![RuntimeValue::new_markdown(mq_markdown::Node::HorizontalRule(mq_markdown::HorizontalRule { marker: None, position: None, }))].into()))] #[case::skip_while_basic("skip_while([1,2,3,4,5], fn(x): x < 3;)", @@ -2661,7 +2661,7 @@ fn to_code_round_trip_preserves_compound_assign_result(mut engine: DefaultEngine #[case::to_code_none(r#"to_code(None, "rust")"#, vec![RuntimeValue::None], Ok(vec![RuntimeValue::None].into()))] #[case::to_math_none("to_math(None)", vec![RuntimeValue::None], Ok(vec![RuntimeValue::None].into()))] #[case::to_math_inline_none("to_math_inline(None)", vec![RuntimeValue::None], Ok(vec![RuntimeValue::None].into()))] -#[case::to_hr_simple("to_hr()", vec![RuntimeValue::None], Ok(vec![RuntimeValue::new_markdown(mq_markdown::Node::HorizontalRule(mq_markdown::HorizontalRule{position: None}))].into()))] +#[case::to_hr_simple("to_hr()", vec![RuntimeValue::None], Ok(vec![RuntimeValue::new_markdown(mq_markdown::Node::HorizontalRule(mq_markdown::HorizontalRule{ marker: None,position: None}))].into()))] #[case::to_strong_simple(r##"to_strong("bold")"##, vec![RuntimeValue::None], Ok(vec![RuntimeValue::new_markdown(mq_markdown::Node::Strong(mq_markdown::Strong{values: vec!["bold".to_string().into()], position: None}))].into()))] #[case::to_em_simple(r##"to_em("italic")"##, vec![RuntimeValue::None], Ok(vec![RuntimeValue::new_markdown(mq_markdown::Node::Emphasis(mq_markdown::Emphasis{values: vec!["italic".to_string().into()], position: None}))].into()))] #[case::to_code_inline_simple(r##"to_code_inline("code")"##, vec![RuntimeValue::None], Ok(vec![RuntimeValue::new_markdown(mq_markdown::Node::CodeInline(mq_markdown::CodeInline{value: "code".to_string().into(), position: None}))].into()))] @@ -2688,7 +2688,7 @@ fn to_code_round_trip_preserves_compound_assign_result(mut engine: DefaultEngine #[case::to_link_simple(r##"to_link("url", "text", "title")"##, vec![RuntimeValue::None], Ok(vec![RuntimeValue::new_markdown(mq_markdown::Node::Link(mq_markdown::Link{url: mq_markdown::Url::new("url".to_string()), title: Some(mq_markdown::Title::new("title".to_string())), values: vec!["text".to_string().into()], position: None}))].into()))] #[case::to_math_simple(r##"to_math("E=mc^2")"##, vec![RuntimeValue::None], Ok(vec![RuntimeValue::new_markdown(mq_markdown::Node::Math(mq_markdown::Math{value: "E=mc^2".to_string(), position: None}))].into()))] #[case::to_math_inline_simple(r##"to_math_inline("E=mc^2")"##, vec![RuntimeValue::None], Ok(vec![RuntimeValue::new_markdown(mq_markdown::Node::MathInline(mq_markdown::MathInline{value: "E=mc^2".to_string().into(), position: None}))].into()))] -#[case::to_md_list_simple(r##"to_md_list("item", 1)"##, vec![RuntimeValue::None], Ok(vec![RuntimeValue::new_markdown(mq_markdown::Node::List(mq_markdown::List{ start: None, spread: false,values: vec!["item".to_string().into()], index: 0, ordered: false, level: 1, checked: None, position: None}))].into()))] +#[case::to_md_list_simple(r##"to_md_list("item", 1)"##, vec![RuntimeValue::None], Ok(vec![RuntimeValue::new_markdown(mq_markdown::Node::List(mq_markdown::List{ marker: None, start: None, spread: false,values: vec!["item".to_string().into()], index: 0, ordered: false, level: 1, checked: None, position: None}))].into()))] #[case::to_md_table_row_simple(r##"to_md_table_row("a", "b")"##, vec![RuntimeValue::None], Ok(vec![RuntimeValue::new_markdown(mq_markdown::Node::TableRow(mq_markdown::TableRow{values: vec![ mq_markdown::Node::TableCell(mq_markdown::TableCell{row: 0, column: 0, values: vec!["a".to_string().into()], position: None}), mq_markdown::Node::TableCell(mq_markdown::TableCell{row: 0, column: 1, values: vec!["b".to_string().into()], position: None}), diff --git a/crates/mq-markdown/benches/benchmark.rs b/crates/mq-markdown/benches/benchmark.rs index 4aab2570c..c73382a1b 100644 --- a/crates/mq-markdown/benches/benchmark.rs +++ b/crates/mq-markdown/benches/benchmark.rs @@ -55,53 +55,20 @@ fn render_table(bencher: divan::Bencher) { bencher.bench(|| markdown.to_string()); } -// Full end-to-end: mdast parse + expand_wikilinks (wikilinks present, early-exit skipped). +// Full end-to-end parse of a document with wikilinks. #[cfg(feature = "wikilink")] #[divan::bench(name = "from_markdown_str/with_wikilinks")] fn from_markdown_str_with_wikilinks() -> mq_markdown::Markdown { mq_markdown::Markdown::from_markdown_str(&wikilink_doc()).unwrap() } -// Full end-to-end: mdast parse only (early-exit fires, expand_wikilinks skipped). +// Full end-to-end parse of a document without wikilinks. #[cfg(feature = "wikilink")] #[divan::bench(name = "from_markdown_str/without_wikilinks")] fn from_markdown_str_without_wikilinks() -> mq_markdown::Markdown { mq_markdown::Markdown::from_markdown_str(&plain_doc()).unwrap() } -// Isolate expand_wikilinks alone (wikilinks present). -#[cfg(feature = "wikilink")] -#[divan::bench(name = "expand_wikilinks/with_wikilinks")] -fn expand_wikilinks_with(bencher: divan::Bencher) { - let doc = wikilink_doc(); - let nodes = mq_markdown::Markdown::from_markdown_str_no_expand(&doc).unwrap(); - bencher.bench(|| mq_markdown::Node::expand_wikilinks(nodes.clone())); -} - -// Isolate expand_wikilinks alone (no wikilinks — measures pure traversal cost before early-exit). -#[cfg(feature = "wikilink")] -#[divan::bench(name = "expand_wikilinks/without_wikilinks")] -fn expand_wikilinks_without(bencher: divan::Bencher) { - let doc = plain_doc(); - let nodes = mq_markdown::Markdown::from_markdown_str_no_expand(&doc).unwrap(); - bencher.bench(|| mq_markdown::Node::expand_wikilinks(nodes.clone())); -} - -// Isolate the early-exit check (contains("[[")) vs full expand_wikilinks for plain doc. -#[cfg(feature = "wikilink")] -#[divan::bench(name = "contains_check/with_wikilinks")] -fn contains_check_with() { - let doc = wikilink_doc(); - divan::black_box(doc.contains("[[")); -} - -#[cfg(feature = "wikilink")] -#[divan::bench(name = "contains_check/without_wikilinks")] -fn contains_check_without() { - let doc = plain_doc(); - divan::black_box(doc.contains("[[")); -} - // Callout: end-to-end parse with callout headers (mdast + try_parse_callout per blockquote). #[cfg(feature = "callout")] #[divan::bench(name = "from_markdown_str/with_callouts")] @@ -123,17 +90,44 @@ fn from_markdown_str_with_embeds() -> mq_markdown::Markdown { mq_markdown::Markdown::from_markdown_str(&embed_doc()).unwrap() } -// Early-exit check for embeds. -#[cfg(feature = "embed")] -#[divan::bench(name = "contains_check/with_embeds")] -fn contains_check_embed_with() { - let doc = embed_doc(); - divan::black_box(doc.contains("![[")); -} - -#[cfg(feature = "embed")] -#[divan::bench(name = "contains_check/without_embeds")] -fn contains_check_embed_without() { - let doc = plain_doc(); - divan::black_box(doc.contains("![[")); +// Real-world documents: the README and the book sources, concatenated. +fn real_doc() -> String { + fn walk(dir: &std::path::Path, out: &mut Vec) { + let Ok(entries) = std::fs::read_dir(dir) else { return }; + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() { + walk(&path, out); + } else if path.extension().is_some_and(|ext| ext == "md") { + out.push(path); + } + } + } + + let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let mut paths = vec![root.join("README.md")]; + walk(&root.join("docs/books/src"), &mut paths); + paths.sort(); + paths + .iter() + .filter_map(|path| std::fs::read_to_string(path).ok()) + .collect::>() + .join("\n\n") +} + +#[divan::bench(name = "from_markdown_str/real_docs")] +fn from_markdown_str_real_docs(bencher: divan::Bencher) { + let doc = real_doc(); + bencher + .counter(divan::counter::BytesCount::of_str(&doc)) + .bench(|| mq_markdown::Markdown::from_markdown_str(&doc).unwrap()); +} + +// Many short lines, to stress line splitting. +#[divan::bench(name = "from_markdown_str/many_lines")] +fn from_markdown_str_many_lines(bencher: divan::Bencher) { + let doc = "line of plain text\n".repeat(20_000); + bencher + .counter(divan::counter::BytesCount::of_str(&doc)) + .bench(|| mq_markdown::Markdown::from_markdown_str(&doc).unwrap()); } diff --git a/crates/mq-markdown/src/lib.rs b/crates/mq-markdown/src/lib.rs index 5efa438ef..084c61901 100644 --- a/crates/mq-markdown/src/lib.rs +++ b/crates/mq-markdown/src/lib.rs @@ -46,7 +46,7 @@ //! //! let mut doc = "- Item 1\n- Item 2".parse::().unwrap(); //! doc.set_options(RenderOptions { -//! list_style: ListStyle::Plus, +//! list_style: Some(ListStyle::Plus), //! ..Default::default() //! }); //! diff --git a/crates/mq-markdown/src/markdown.rs b/crates/mq-markdown/src/markdown.rs index e3eeafd6b..6268253c3 100644 --- a/crates/mq-markdown/src/markdown.rs +++ b/crates/mq-markdown/src/markdown.rs @@ -2,15 +2,19 @@ use crate::html_to_markdown; #[cfg(feature = "html-to-markdown")] use crate::html_to_markdown::ConversionOptions; +#[cfg(feature = "html-to-markdown")] +use crate::node::ListStyle; use crate::node::{ - Code, ColorTheme, Node, Position, RenderOptions, TableAlign, TableCell, list_own_prefix_width, reindent_all_lines, - render_values, + Code, ColorTheme, Node, Position, RenderOptions, TableAlign, TableCell, indent_lines, list_own_prefix_width, + reindent_all_lines, render_cell_values, render_values_block, }; +use list_order::CONTINUATION; #[cfg(any(feature = "json", feature = "html-to-markdown"))] use miette::miette; use std::{fmt, str::FromStr}; use table_layout::{TableLayout, write_padded_cell}; +mod list_order; mod table_layout; #[derive(Debug, Clone)] @@ -57,7 +61,7 @@ impl Markdown { self.render_with_theme(theme) } - fn render_with_theme(&self, theme: &ColorTheme<'_>) -> String { + pub(crate) fn render_with_theme(&self, theme: &ColorTheme<'_>) -> String { let mut pre_position: Option = None; let mut is_first = true; let mut current_table_row: Option = None; @@ -67,22 +71,24 @@ impl Markdown { // nested-list indentation from the nearest open ancestor at each level. let mut list_indent_stack: Vec = Vec::new(); - let mut buffer = String::with_capacity(self.nodes.len() * 50); + let reordered = list_order::reorder(&self.nodes); + let nodes: &[Node] = reordered.as_deref().unwrap_or(&self.nodes); + let mut buffer = String::with_capacity(nodes.len() * 50); - for (i, node) in self.nodes.iter().enumerate() { + for (i, node) in nodes.iter().enumerate() { if let Node::TableCell(TableCell { row, column, values, .. }) = node { let is_new_table = current_table.as_ref().is_none_or(|t| i >= t.end); if is_new_table { - current_table = Some(TableLayout::compute(&self.nodes, &self.options, i)); + current_table = Some(TableLayout::compute(nodes, &self.options, i)); } let table = current_table.as_ref().unwrap(); let align = table.align_for(*column); let width = table.width_for(*column); - let value = render_values(values, &self.options, theme); + let value = render_cell_values(values, &self.options, theme); let plain_width = table.plain_width_at(*row, *column); let is_new_row = current_table_row != Some(*row); @@ -115,7 +121,7 @@ impl Markdown { write_padded_cell(&mut buffer, &value, plain_width, width, &align); - let next_node = self.nodes.get(i + 1); + let next_node = nodes.get(i + 1); let next_is_different_row = next_node.is_none_or( |next| !matches!(next, Node::TableCell(TableCell { row: next_row, .. }) if *next_row == *row), ); @@ -133,7 +139,7 @@ impl Markdown { if let Node::TableAlign(TableAlign { .. }) = node { if current_table.as_ref().is_none_or(|t| i >= t.end) { - current_table = Some(TableLayout::compute(&self.nodes, &self.options, i)); + current_table = Some(TableLayout::compute(nodes, &self.options, i)); } let table = current_table.as_ref().unwrap(); buffer.push_str(&table.render_separator()); @@ -148,15 +154,22 @@ impl Markdown { current_table = None; in_table = false; - let prev_node = i.checked_sub(1).and_then(|j| self.nodes.get(j)); + let prev_node = i.checked_sub(1).and_then(|j| nodes.get(j)); - let value = if let Node::List(list) = node { + let value = if let Node::List(list) = node + && list.marker == Some(CONTINUATION) + { + // The rest of an item after its nested items, aligned with the content of the item. + list_indent_stack.truncate(list.level as usize + 1); + let indent = list_indent_stack.last().copied().unwrap_or(0); + indent_lines(&render_values_block(&list.values, &self.options, theme), indent) + } else if let Node::List(list) = node { while list_indent_stack.len() > list.level as usize { list_indent_stack.pop(); } let own_indent = list_indent_stack.last().copied().unwrap_or(0); let delta = own_indent as isize - list.level as isize * 2; - let own_width = own_indent + list_own_prefix_width(list.ordered, list.index, list.start, list.checked); + let own_width = own_indent + list_own_prefix_width(list.ordered, list.index, list.start); list_indent_stack.push(own_width); reindent_all_lines(&node.render_with_theme(&self.options, theme), delta) } else if let Node::Code(code) = node @@ -192,6 +205,23 @@ impl Markdown { new_line_count = 2; } + // A footnote ends where the next line starts, so a block after it needs a blank line. + if new_line_count < 2 + && matches!(prev_node, Some(Node::Footnote(_))) + && !matches!(node, Node::Footnote(_)) + { + new_line_count = 2; + } + + // An item that starts on the line of the marker that holds it follows it directly. + let value = + if new_line_count == 0 && matches!(node, Node::List(_)) && matches!(prev_node, Some(Node::List(_))) + { + value.trim_start().to_string() + } else { + value + }; + // Same-list adjacent items separate by `spread`, not source line gap. if let Node::List(cur_list) = node && let Some(Node::List(prev_list)) = prev_node @@ -224,12 +254,17 @@ impl Markdown { } } - if buffer.is_empty() || buffer.ends_with('\n') { - buffer - } else { + if !buffer.is_empty() && !buffer.ends_with('\n') { buffer.push('\n'); - buffer } + // A rule of dashes first and a line of three dashes later would read as frontmatter. + if matches!(nodes.first(), Some(Node::HorizontalRule(_))) + && buffer.starts_with("---\n") + && buffer[4..].lines().any(|line| line == "---") + { + buffer.replace_range(..3, "***"); + } + buffer } pub fn from_mdx_str(content: &str) -> miette::Result { @@ -272,44 +307,21 @@ impl Markdown { html_to_markdown::convert_html_to_markdown(content, options) .map_err(|e| miette!(e)) .and_then(|md_string| Self::from_markdown_str(&md_string)) + .map(|mut markdown| { + // The converter's `*` bullets are an artifact, not a source style to preserve. + markdown.options.list_style = Some(ListStyle::Dash); + markdown + }) } pub fn from_markdown_str(content: &str) -> miette::Result { let nodes = parse_nodes(content)?; - #[cfg(all(feature = "embed", feature = "wikilink"))] - let nodes = if content.contains("[[") { - Node::expand_inline_links(nodes) - } else { - nodes - }; - - #[cfg(all(feature = "embed", not(feature = "wikilink")))] - let nodes = if content.contains("![[") { - Node::expand_embeds(nodes) - } else { - nodes - }; - - #[cfg(all(feature = "wikilink", not(feature = "embed")))] - let nodes = if content.contains("[[") { - Node::expand_wikilinks(nodes) - } else { - nodes - }; - Ok(Self { nodes, options: RenderOptions::default(), }) } - - /// Parses markdown without running any expand pass. Used in benchmarks to - /// isolate expand cost from the mdast parse cost. - #[cfg(any(feature = "wikilink", feature = "embed"))] - pub fn from_markdown_str_no_expand(content: &str) -> miette::Result> { - parse_nodes(content) - } } /// Parses `content` into nodes. @@ -334,7 +346,7 @@ pub fn to_html(s: &str) -> String { mod tests { use rstest::rstest; - use crate::{ListStyle, TitleSurroundStyle, UrlSurroundStyle}; + use crate::{TitleSurroundStyle, UrlSurroundStyle}; use super::*; @@ -369,7 +381,7 @@ mod tests { #[case::footnote("[^a]: b", 1, "[^a]: b\n")] #[case::definition("[a]: b", 1, "[a]: b\n")] #[case::footnote("[^a]: b", 1, "[^a]: b\n")] - #[case::footnote_ref("[^a]: b\n\n[^a]", 2, "[^a]: b\n[^a]\n")] + #[case::footnote_ref("[^a]: b\n\n[^a]", 2, "[^a]: b\n\n[^a]\n")] #[case::image("![a](b)", 1, "![a](b)\n")] #[case::image_with_title("![a](b \"c\")", 1, "![a](b \"c\")\n")] #[case::image_ref("[a]: b\n\n ![c][a]", 2, "[a]: b\n\n![c][a]\n")] @@ -383,7 +395,12 @@ mod tests { #[case::break_("a\\b", 1, "a\\\\b\n")] #[case::delete("~~a~~", 1, "~~a~~\n")] #[case::emphasis("*a*", 1, "*a*\n")] - #[case::horizontal_rule("---", 1, "***\n")] + #[case::horizontal_rule("---", 1, "---\n")] + #[case::horizontal_rule_star("***", 1, "***\n")] + #[case::horizontal_rule_underscore("___", 1, "___\n")] + #[case::list_star("* a\n* b", 2, "* a\n* b\n")] + #[case::list_plus("+ a\n+ b", 2, "+ a\n+ b\n")] + #[case::list_ordered_paren("3) a\n4) b", 2, "3) a\n4) b\n")] #[case::table( "| Column1 | Column2 | Column3 |\n|:--------|:--------:|---------:|\n| Left | Center | Right |\n", 7, @@ -694,10 +711,10 @@ mod tests { assert_eq!(md.options, RenderOptions::default()); md.set_options(RenderOptions { - list_style: ListStyle::Plus, + list_style: Some(ListStyle::Plus), ..RenderOptions::default() }); - assert_eq!(md.options.list_style, ListStyle::Plus); + assert_eq!(md.options.list_style, Some(ListStyle::Plus)); let pretty = md.to_string(); assert!(pretty.contains("+ Item 1")); @@ -732,7 +749,7 @@ mod tests { let mut md = "- Item 1\n- Item 2".parse::().unwrap(); md.set_options(RenderOptions { - list_style: ListStyle::Star, + list_style: Some(ListStyle::Star), link_title_style: TitleSurroundStyle::default(), link_url_style: UrlSurroundStyle::default(), }); @@ -769,7 +786,7 @@ mod color_tests { #[case::link("[text](url)", "\x1b[4m\x1b[34m[text](url)\x1b[0m\n")] #[case::image("![alt](url)", "\x1b[35m![alt](url)\x1b[0m\n")] #[case::delete("~~deleted~~", "\x1b[31m\x1b[2m~~deleted~~\x1b[0m\n")] - #[case::horizontal_rule("---", "\x1b[2m***\x1b[0m\n")] + #[case::horizontal_rule("---", "\x1b[2m---\x1b[0m\n")] #[case::blockquote("> quote", "\x1b[2m> \x1b[0mquote\n")] #[case::math_inline("$x^2$", "\x1b[32m$x^2$\x1b[0m\n")] #[case::list("- item", "\x1b[33m-\x1b[0m item\n")] diff --git a/crates/mq-markdown/src/markdown/list_order.rs b/crates/mq-markdown/src/markdown/list_order.rs new file mode 100644 index 000000000..969c2bf97 --- /dev/null +++ b/crates/mq-markdown/src/markdown/list_order.rs @@ -0,0 +1,124 @@ +//! Puts the children of a list item back in order with the items nested in it. +//! +//! A flat list holds the items nested in an item right after it, even when some of the children of +//! that item come after them in the source, like a paragraph that follows a nested list. Rendering +//! the nodes as they are would write those children before the nested items. + +use crate::node::{List, Node, Position}; + +/// The marker of a node that holds the children of an item that follow items nested in it. Such a +/// node is rendered as its content, indented to the content of the item, without a marker. +pub(super) const CONTINUATION: char = '\0'; + +/// Returns the nodes in source order, or `None` when they already are. +pub(super) fn reorder(nodes: &[Node]) -> Option> { + // Some child of an item starts after the item nested right after it. + let needed = nodes.windows(2).any(|pair| match (&pair[0], &pair[1]) { + (Node::List(item), Node::List(next)) if next.level > item.level => { + let nested = start_line(&pair[1]); + item.values + .iter() + .filter_map(start_line) + .any(|line| Some(line) > nested) + } + _ => false, + }); + if !needed { + return None; + } + let mut out = Vec::with_capacity(nodes.len() + 4); + reorder_into(nodes, &mut out); + Some(out) +} + +fn start_line(node: &Node) -> Option { + node.position().map(|position| position.start.line) +} + +fn reorder_into(nodes: &[Node], out: &mut Vec) { + let mut index = 0; + while index < nodes.len() { + let node = &nodes[index]; + let Node::List(item) = node else { + out.push(node.clone()); + index += 1; + continue; + }; + let nested = nodes[index + 1..] + .iter() + .take_while(|next| matches!(next, Node::List(next) if next.level > item.level)) + .count(); + let lines: Option> = item.values.iter().map(start_line).collect(); + let (true, Some(lines)) = (!item.values.is_empty() && nested > 0, lines) else { + out.push(node.clone()); + index += 1; + continue; + }; + + let run = &nodes[index + 1..index + 1 + nested]; + let Node::List(first) = &run[0] else { unreachable!() }; + let segment_level = first.level; + // Each item nested directly in this one, with the items nested in those. + let mut segments: Vec<&[Node]> = Vec::new(); + let mut from = 0; + for at in 1..=run.len() { + if at == run.len() || matches!(&run[at], Node::List(next) if next.level <= segment_level) { + segments.push(&run[from..at]); + from = at; + } + } + let segment_lines: Option> = segments.iter().map(|segment| start_line(&segment[0])).collect(); + let Some(segment_lines) = segment_lines else { + out.push(node.clone()); + index += 1; + continue; + }; + + // The children before the first nested item stay with the item. + let head = lines.iter().take_while(|line| **line < segment_lines[0]).count(); + out.push(Node::List(List { + values: item.values[..head].to_vec(), + position: span(&item.values[..head]).or_else(|| { + // Without children it ends where it starts, so the nested item follows the marker. + item.position.clone().map(|position| Position { + end: position.start.clone(), + start: position.start, + }) + }), + ..item.clone() + })); + + let mut child = head; + for (segment, line) in segments.iter().zip(&segment_lines) { + let from = child; + while child < item.values.len() && lines[child] < *line { + child += 1; + } + if child > from { + out.push(continuation(item, &item.values[from..child])); + } + reorder_into(segment, out); + } + if child < item.values.len() { + out.push(continuation(item, &item.values[child..])); + } + index += 1 + nested; + } +} + +fn continuation(item: &List, values: &[Node]) -> Node { + Node::List(List { + values: values.to_vec(), + position: span(values), + marker: Some(CONTINUATION), + ..item.clone() + }) +} + +/// The position from the start of the first node to the end of the last. +fn span(values: &[Node]) -> Option { + Some(Position { + start: values.first()?.position()?.start, + end: values.last()?.position()?.end, + }) +} diff --git a/crates/mq-markdown/src/markdown/table_layout.rs b/crates/mq-markdown/src/markdown/table_layout.rs index c06c6fbfd..83ed0c934 100644 --- a/crates/mq-markdown/src/markdown/table_layout.rs +++ b/crates/mq-markdown/src/markdown/table_layout.rs @@ -1,4 +1,4 @@ -use crate::node::{ColorTheme, Node, RenderOptions, TableAlign, TableAlignKind, TableCell, render_values}; +use crate::node::{ColorTheme, Node, RenderOptions, TableAlign, TableAlignKind, TableCell, render_cell_values}; use unicode_width::UnicodeWidthStr; /// Per-column widths and alignment of a single contiguous table, computed by @@ -8,6 +8,8 @@ use unicode_width::UnicodeWidthStr; pub(super) struct TableLayout { widths: Vec, align: Vec, + /// Number of cells in the header row, which the delimiter row has to match. + header_columns: usize, /// Plain (color-free) display width of every cell, keyed by `(row, column)`, /// computed once during the scan so the main render pass never has to /// re-render a cell just to measure it. @@ -21,6 +23,7 @@ impl TableLayout { let mut widths: Vec = Vec::new(); let mut align: Vec = Vec::new(); let mut cell_widths = rustc_hash::FxHashMap::default(); + let mut header_columns = 0; let mut i = start; while let Some(node) = nodes.get(i) { @@ -32,6 +35,9 @@ impl TableLayout { if *row == 0 && *column == 0 && !align.is_empty() { break; } + if *row == 0 { + header_columns = header_columns.max(*column + 1); + } let width = cell_display_width(values, options); cell_widths.insert((*row, *column), width); if *column >= widths.len() { @@ -60,6 +66,7 @@ impl TableLayout { Self { widths, align, + header_columns, cell_widths, end: i, } @@ -82,9 +89,16 @@ impl TableLayout { } pub(super) fn render_separator(&self) -> String { + // The delimiter row has as many columns as the header, whatever a body row has more. + let columns = if self.header_columns == 0 { + self.widths.len() + } else { + self.header_columns + }; let segments = self .widths .iter() + .take(columns) .enumerate() .map(|(idx, &width)| { let align = self.align_for(idx); @@ -116,7 +130,7 @@ fn column_min_width(align: &TableAlignKind) -> usize { /// content, ignoring any ANSI color codes so padding stays correct with /// `--color` output. fn cell_display_width(values: &[Node], options: &RenderOptions) -> usize { - UnicodeWidthStr::width(render_values(values, options, &ColorTheme::PLAIN).as_str()) + UnicodeWidthStr::width(render_cell_values(values, options, &ColorTheme::PLAIN).as_str()) } /// Writes an already-rendered (possibly colored) cell value padded to `width` columns. diff --git a/crates/mq-markdown/src/node.rs b/crates/mq-markdown/src/node.rs index fd79f252a..a61365e02 100644 --- a/crates/mq-markdown/src/node.rs +++ b/crates/mq-markdown/src/node.rs @@ -148,7 +148,8 @@ pub(crate) type Level = u8; #[derive(Debug, Clone, Default, PartialEq)] pub struct RenderOptions { - pub list_style: ListStyle, + /// Bullet marker for lists. `None` keeps each list's original marker (`-` if unknown). + pub list_style: Option, pub link_url_style: UrlSurroundStyle, pub link_title_style: TitleSurroundStyle, } @@ -289,6 +290,9 @@ pub struct List { /// Starting number for an ordered list (e.g. `5` in `5. foo`); `None` means 1. #[cfg_attr(feature = "json", serde(skip_serializing_if = "Option::is_none"))] pub start: Option, + /// Source marker: `-`, `+` or `*` for bullets, `.` or `)` for ordered lists. + #[cfg_attr(feature = "json", serde(default, skip_serializing_if = "Option::is_none"))] + pub marker: Option, #[cfg_attr(feature = "json", serde(skip_serializing_if = "Option::is_none"))] pub position: Option, } @@ -406,8 +410,11 @@ pub struct Link { serde(rename_all = "camelCase", tag = "type") )] pub struct Callout { - /// The callout type in uppercase (e.g. `"NOTE"`, `"WARNING"`, `"TIP"`). + /// The callout type as written, e.g. `"NOTE"` or `"warning"`. pub kind: String, + /// The fold marker right after `[!TYPE]`: `+` for open and `-` for folded. + #[cfg_attr(feature = "json", serde(default, skip_serializing_if = "Option::is_none"))] + pub fold: Option, /// Optional custom title after the `[!TYPE]` marker. pub title: Option, /// Body content nodes (the lines after the header). @@ -764,6 +771,9 @@ pub struct Break { serde(rename_all = "camelCase", tag = "type") )] pub struct HorizontalRule { + /// Source marker: `*`, `-` or `_`. + #[cfg_attr(feature = "json", serde(default, skip_serializing_if = "Option::is_none"))] + pub marker: Option, #[cfg_attr(feature = "json", serde(skip_serializing_if = "Option::is_none"))] pub position: Option, } @@ -1075,27 +1085,26 @@ impl Node { ordered, index, start, + marker, .. }) => { let marker = if *ordered { - format!("{}.", start.unwrap_or(1) as usize + *index) + let delimiter = marker.filter(|c| *c == ')').unwrap_or('.'); + format!("{}{}", start.unwrap_or(1) as usize + *index, delimiter) + } else if let Some(style) = &options.list_style { + style.to_string() } else { - options.list_style.to_string() + marker + .filter(|c| matches!(c, '-' | '+' | '*')) + .unwrap_or('-') + .to_string() }; let checkbox = (*checked).map(|it| if it { "[x] " } else { "[ ] " }).unwrap_or(""); - let prefix_width = *level as usize * 2 + list_own_prefix_width(*ordered, *index, *start, *checked); - // A block quote/callout child needs a flat prefix_width add, not the delta below. - let delta = if values.first().is_some_and(Self::is_blockquote_like) { - prefix_width as isize - } else { - // Marker width is normalized, shifting continuation lines' source columns. - let original_first_column = values.first().and_then(|v| v.position()).map(|p| p.start.column); - original_first_column - .map(|c| prefix_width as isize - (c as isize - 1)) - .unwrap_or(0) - }; - let content = reindent_continuation(&render_values(values, options, theme), delta); - let content = reindent_first_leaf_block(content, values.first(), options, theme, prefix_width); + let prefix_width = *level as usize * 2 + list_own_prefix_width(*ordered, *index, *start); + // Lines after the first are indented to the content of the item, whatever column they + // had in the source, which is not reliable after an escape. + let content = render_values_block(values, options, theme); + let content = indent_continuation(&content, prefix_width); let (ms, me) = &theme.list_marker; format!( "{}{}{}{} {}{}", @@ -1116,7 +1125,7 @@ impl Node { .join("|"); format!("{}|{}{}|", ts, te, cells) } - Self::TableCell(TableCell { values, .. }) => render_values(values, options, theme), + Self::TableCell(TableCell { values, .. }) => render_cell_values(values, options, theme), Self::TableAlign(TableAlign { align, .. }) => { let (ts, te) = &theme.table_separator; format!("{}|{}|{}", ts, align.iter().map(|a| a.to_string()).join("|"), te) @@ -1130,12 +1139,17 @@ impl Node { } #[cfg(feature = "callout")] Self::Callout(Callout { - kind, title, values, .. + kind, + fold, + title, + values, + .. }) => { let (bs, be) = &theme.blockquote_marker; + let fold = fold.map(String::from).unwrap_or_default(); let header = match title.as_deref() { - Some(t) if !t.is_empty() => format!("[!{}] {}", kind, t), - _ => format!("[!{}]", kind), + Some(t) if !t.is_empty() => format!("[!{}]{} {}", kind, fold, t), + _ => format!("[!{}]{}", kind, fold), }; let header_line = format!("{}> {}{}", bs, be, header); if values.is_empty() { @@ -1162,7 +1176,7 @@ impl Node { lang, fence, meta, - .. + position, }) => { let (cs, ce) = &theme.code; if lang.is_some() || *fence { @@ -1171,7 +1185,11 @@ impl Node { // Empty body skips the content line so it doesn't gain a blank one. let fence_str = code_fence(value, &info); if value.is_empty() { - format!("{}{}{}\n{}{}", cs, fence_str, info, fence_str, ce) + // Blank lines are not in the value, but the lines of the block tell of them. + let blank = position + .as_ref() + .map_or(0, |p| p.end.line.saturating_sub(p.start.line).saturating_sub(1)); + format!("{}{}{}\n{}{}{}", cs, fence_str, info, "\n".repeat(blank), fence_str, ce) } else { format!("{}{}{}\n{}\n{}{}", cs, fence_str, info, value, fence_str, ce) } @@ -1204,7 +1222,6 @@ impl Node { format!("{}~~{}~~{}", ds, render_values(values, options, theme), de) } Self::Emphasis(Emphasis { values, .. }) => { - let (es, ee) = &theme.emphasis; // A lone nested Emphasis child needs the other delimiter, or adjacent // `*` `*` pairs fuse into `**` and reparse as Strong instead. let delim = if matches!(values.as_slice(), [Self::Emphasis(_)]) { @@ -1212,14 +1229,7 @@ impl Node { } else { "*" }; - format!( - "{}{}{}{}{}", - es, - delim, - render_values(values, options, theme), - delim, - ee - ) + render_emphasis(values, delim, options, theme) } Self::Footnote(Footnote { values, ident, .. }) => { format!("[^{}]: {}", escape_label(ident), render_values(values, options, theme)) @@ -1303,6 +1313,11 @@ impl Node { } Self::Link(Link { url, title, values, .. }) => { let (ls, le) = &theme.link; + if title.is_none() + && let Some(target) = autolink_target(url.as_str(), values) + { + return format!("{ls}<{target}>{le}"); + } format!( "{}[{}]({}{}){}", ls, @@ -1328,7 +1343,9 @@ impl Node { let (ls, le) = &theme.link; let rendered = render_values(values, options, theme); let plain = values_to_value(values); - let mismatched = normalize_reference_identifier(&plain) != ident.as_str(); + // The label of a shortcut reference is its text as written, markup included. + let written = render_values(values, options, &ColorTheme::PLAIN); + let mismatched = normalize_reference_identifier(&written) != ident.as_str(); if mismatched || needs_broad_escaping(&plain) { format!("{}[{}][{}]{}", ls, rendered, ident, le) @@ -1412,15 +1429,19 @@ impl Node { Self::MdxJsEsm(mdxjs_esm) => mdxjs_esm.value.to_string(), Self::Strong(Strong { values, .. }) => { let (ss, se) = &theme.strong; - format!( - "{}**{}**{}", - ss, - values - .iter() - .map(|value| value.render_with_theme(options, theme)) - .collect::(), - se - ) + let last = values.len().saturating_sub(1); + let content = values + .iter() + .enumerate() + .map(|(index, value)| match value { + // Emphasis at an end uses the other delimiter, or `*` and `**` fuse into `***`. + Self::Emphasis(Emphasis { values, .. }) if index == 0 || index == last => { + render_emphasis(values, "_", options, theme) + } + value => value.render_with_theme(options, theme), + }) + .collect::(); + format!("{}**{}**{}", ss, content, se) } Self::Yaml(Yaml { value, .. }) => { let (fs, fe) = &theme.frontmatter; @@ -1431,10 +1452,10 @@ impl Node { format!("{}+++\n{}\n+++{}", fs, value, fe) } Self::Break(_) => "\\\n".to_string(), - Self::HorizontalRule(_) => { - // `***` avoids ambiguity with a setext heading underline (`---`). + Self::HorizontalRule(HorizontalRule { marker, .. }) => { let (hs, he) = &theme.horizontal_rule; - format!("{}***{}", hs, he) + let marker = marker.filter(|c| matches!(c, '-' | '_')).unwrap_or('*'); + format!("{hs}{marker}{marker}{marker}{he}") } Self::Fragment(Fragment { values }) => values .iter() @@ -2290,9 +2311,14 @@ impl Node { }, #[cfg(feature = "callout")] Node::Callout(Callout { - kind, title, values, .. + kind, + fold, + title, + values, + .. }) => match attr { attr_keys::KIND => Some(AttrValue::String(kind.clone())), + attr_keys::FOLD => fold.map(|fold| AttrValue::String(fold.to_string())), attr_keys::TITLE => title.clone().map(AttrValue::String), attr_keys::VALUE => Some(AttrValue::String(values_to_string(values, &RenderOptions::default()))), attr_keys::VALUES | attr_keys::CHILDREN => Some(AttrValue::Array(values.clone())), @@ -2645,6 +2671,7 @@ impl Node { #[cfg(feature = "callout")] Node::Callout(c) => match attr { attr_keys::KIND => c.kind = value_str, + attr_keys::FOLD => c.fold = value_str.chars().next().filter(|c| matches!(c, '+' | '-')), attr_keys::TITLE => c.title = if value_str.is_empty() { None } else { Some(value_str) }, _ => (), }, @@ -2672,70 +2699,112 @@ impl Node { /// Tries to parse a `Blockquote`'s converted nodes as an Obsidian callout. /// /// Returns a `Callout` node when the first text starts with `[!TYPE]`, otherwise - /// falls back to a plain `Blockquote`. + /// falls back to a plain `Blockquote`. `TYPE` is letters, digits, `-` and `_`. A `+` or `-` right after + /// it is the fold marker, and the rest of the line is the title, inline markup included. #[cfg(feature = "callout")] fn try_parse_callout(values: Vec, position: Option) -> Node { // Peek without cloning so plain blockquotes pay no allocation cost. - if !matches!(values.first(), Some(Node::Text(t)) if t.value.starts_with("[!")) { + let valid = matches!(values.first(), Some(Node::Text(t)) if Self::callout_header(&t.value).is_some()); + if !valid { return Node::Blockquote(Blockquote { values, position }); } let mut iter = values.into_iter(); - // SAFETY: the peek above guarantees at least one Text node exists. - let first_text = match iter.next().unwrap() { - Node::Text(t) => t.value, - _ => unreachable!(), - }; - - let Some(rest) = first_text.strip_prefix("[!") else { - unreachable!() - }; - - let Some(bracket_end) = rest.find(']') else { - // Malformed `[!…` without closing `]` — treat as plain blockquote. - let mut values = vec![Node::Text(Text { - value: first_text, - position: None, - })]; - values.extend(iter); - return Node::Blockquote(Blockquote { values, position }); - }; - - let kind = rest[..bracket_end].to_string(); - let after_bracket = &rest[bracket_end + 1..]; - - let (title, remaining_body) = if let Some(nl) = after_bracket.find('\n') { - let t = after_bracket[..nl].trim(); - ( - if t.is_empty() { None } else { Some(t.to_string()) }, - after_bracket[nl + 1..].to_string(), - ) - } else { - let t = after_bracket.trim(); - (if t.is_empty() { None } else { Some(t.to_string()) }, String::new()) + let first_text = match iter.next() { + Some(Node::Text(t)) => t.value, + _ => unreachable!("checked above"), }; + let (kind, fold, after_header) = + Self::callout_header(&first_text).unwrap_or_else(|| unreachable!("checked above")); + + // The title is the rest of the first line, which markup in it can spread over several nodes. + let mut title = String::new(); + let mut rest = iter.collect::>().into_iter().peekable(); + let mut body_start = String::new(); + match after_header.split_once('\n') { + Some((line, remaining)) => { + title.push_str(line); + body_start = remaining.to_string(); + } + None => { + title.push_str(after_header); + // Text right after the first text is the next paragraph, markup in the title is not text. + while rest.next_if(Self::is_title_markup).is_some_and(|node| { + title.push_str(&node.to_string_with(&RenderOptions::default())); + true + }) { + // What follows markup goes on with the title, up to a line ending. + let Some(Node::Text(t)) = rest.peek() else { continue }; + if let Some((line, remaining)) = t.value.split_once('\n') { + title.push_str(line); + body_start = remaining.to_string(); + rest.next(); + break; + } + title.push_str(&t.value); + rest.next(); + } + } + } + let title = title.trim(); - let body = if !remaining_body.trim().is_empty() { - let rest_nodes: Vec = iter.collect(); - let mut b = Vec::with_capacity(rest_nodes.len() + 1); - b.push(Node::Text(Text { - value: remaining_body, + let mut body = Vec::new(); + if !body_start.trim().is_empty() { + body.push(Node::Text(Text { + value: body_start, position: None, })); - b.extend(rest_nodes); - b - } else { - iter.collect() - }; + } + body.extend(rest); Node::Callout(Callout { kind, - title, + fold, + title: (!title.is_empty()).then(|| title.to_string()), values: body, position, }) } + /// Whether `node` is markup that can be part of the title of a callout. + #[cfg(feature = "callout")] + fn is_title_markup(node: &Node) -> bool { + matches!( + node, + Node::Emphasis(_) + | Node::Strong(_) + | Node::Delete(_) + | Node::CodeInline(_) + | Node::MathInline(_) + | Node::Link(_) + | Node::LinkRef(_) + | Node::Image(_) + | Node::ImageRef(_) + | Node::FootnoteRef(_) + ) || { + #[cfg(feature = "wikilink")] + let wiki = matches!(node, Node::WikiLink(_)); + #[cfg(not(feature = "wikilink"))] + let wiki = false; + wiki + } + } + + /// Splits `[!TYPE]` and the fold marker off the start of `text`, returning the type, the marker and + /// what follows. + #[cfg(feature = "callout")] + fn callout_header(text: &str) -> Option<(String, Option, &str)> { + let rest = text.strip_prefix("[!")?; + let end = rest.find(']')?; + let kind = &rest[..end]; + if kind.is_empty() || !kind.chars().all(|c| c.is_alphanumeric() || matches!(c, '-' | '_')) { + return None; + } + let after = &rest[end + 1..]; + let fold = after.chars().next().filter(|c| matches!(c, '+' | '-')); + Some((kind.to_string(), fold, &after[fold.map_or(0, char::len_utf8)..])) + } + /// Returns true if any `Text` descendant contains `[[`, covering both /// embed (`![[`) and wikilink (`[[`) patterns. #[cfg(any(feature = "embed", feature = "wikilink"))] @@ -3088,6 +3157,63 @@ pub(crate) fn values_to_string(values: &[Node], options: &RenderOptions) -> Stri render_values(values, options, &ColorTheme::PLAIN) } +/// Indents every line, the first included, by `width` spaces, except empty ones. +pub(crate) fn indent_lines(content: &str, width: usize) -> String { + let pad = " ".repeat(width); + content + .split('\n') + .map(|line| { + if line.is_empty() { + line.to_string() + } else { + format!("{pad}{line}") + } + }) + .collect::>() + .join("\n") +} + +/// Indents every line after the first by `width` spaces, except empty ones. +fn indent_continuation(content: &str, width: usize) -> String { + let Some((first, rest)) = content.split_once('\n') else { + return content.to_string(); + }; + let pad = " ".repeat(width); + let mut result = String::with_capacity(content.len() + width * 2); + result.push_str(first); + for line in rest.split('\n') { + result.push('\n'); + if !line.is_empty() { + result.push_str(&pad); + result.push_str(line); + } + } + result +} + +fn render_emphasis(values: &[Node], delim: &str, options: &RenderOptions, theme: &ColorTheme<'_>) -> String { + let (es, ee) = &theme.emphasis; + format!("{es}{delim}{}{delim}{ee}", render_values(values, options, theme)) +} + +/// Renders the content of a table cell. A pipe that is not escaped yet, as in code, would end the cell. +pub(crate) fn render_cell_values(values: &[Node], options: &RenderOptions, theme: &ColorTheme<'_>) -> String { + let rendered = render_values(values, options, theme); + if !rendered.contains('|') { + return rendered; + } + let mut result = String::with_capacity(rendered.len() + 4); + let mut escaped = false; + for c in rendered.chars() { + if c == '|' && !escaped { + result.push('\\'); + } + escaped = c == '\\' && !escaped; + result.push(c); + } + result +} + pub(crate) fn render_values(values: &[Node], options: &RenderOptions, theme: &ColorTheme<'_>) -> String { let mut pre_position: Option = None; values @@ -3096,7 +3222,7 @@ pub(crate) fn render_values(values: &[Node], options: &RenderOptions, theme: &Co if let Some(pos) = value.position() { let new_line_count = pre_position .as_ref() - .map(|p: &Position| pos.start.line - p.end.line) + .map(|p: &Position| pos.start.line.saturating_sub(p.end.line)) .unwrap_or_default(); let space = if new_line_count > 0 @@ -3143,27 +3269,54 @@ pub(crate) fn render_values(values: &[Node], options: &RenderOptions, theme: &Co /// so applying column-based spacing causes double-indentation. This variant preserves /// blank lines between values (via newline counting) but ignores the column position. pub(crate) fn render_values_block(values: &[Node], options: &RenderOptions, theme: &ColorTheme<'_>) -> String { + let is_table_part = |node: &Node| matches!(node, Node::TableCell(_) | Node::TableAlign(_)); + let mut result = String::new(); let mut pre_position: Option = None; - values - .iter() - .map(|value| { - if let Some(pos) = value.position() { - let new_line_count = pre_position - .as_ref() - .map(|p: &Position| pos.start.line - p.end.line) - .unwrap_or_default(); - pre_position = Some(pos); - format!( - "{}{}", - "\n".repeat(new_line_count), - value.render_with_theme(options, theme) - ) - } else { - pre_position = None; - value.render_with_theme(options, theme) - } - }) - .collect::() + let mut index = 0; + + while index < values.len() { + // The cells of a table are laid out together, as at the top level. + let (rendered, position, next) = if is_table_part(&values[index]) { + let end = values[index..] + .iter() + .position(|node| !is_table_part(node)) + .map_or(values.len(), |offset| index + offset); + let run = &values[index..end]; + let table = crate::Markdown { + nodes: run.to_vec(), + options: options.clone(), + }; + let position = match (run[0].position(), run[run.len() - 1].position()) { + (Some(first), Some(last)) => Some(Position { + start: first.start, + end: last.end, + }), + _ => None, + }; + ( + table.render_with_theme(theme).trim_end_matches('\n').to_string(), + position, + end, + ) + } else { + let value = &values[index]; + (value.render_with_theme(options, theme), value.position(), index + 1) + }; + index = next; + + if let Some(pos) = position { + let new_line_count = pre_position + .as_ref() + .map(|p: &Position| pos.start.line.saturating_sub(p.end.line)) + .unwrap_or_default(); + pre_position = Some(pos); + result.push_str(&"\n".repeat(new_line_count)); + } else { + pre_position = None; + } + result.push_str(&rendered); + } + result } fn values_to_value(values: &[Node]) -> String { @@ -3256,53 +3409,16 @@ pub(crate) fn reindent_all_lines(content: &str, delta: isize) -> String { .join("\n") } -/// Indents a `Code`/`Math`/`Yaml`/`Toml` first child's own continuation lines, -/// which `render_values` leaves flush-left since it only indents non-first children. -fn reindent_first_leaf_block( - content: String, - first: Option<&Node>, - options: &RenderOptions, - theme: &ColorTheme<'_>, - target_indent: usize, -) -> String { - let Some(first) = first else { return content }; - if !matches!(first, Node::Code(_) | Node::Math(_) | Node::Yaml(_) | Node::Toml(_)) { - return content; - } - let first_rendered = first.render_with_theme(options, theme); - if !first_rendered.contains('\n') { - return content; - } - let reindented_first = reindent_continuation(&first_rendered, target_indent as isize); - let mut lines: Vec<&str> = content.split('\n').collect(); - for (line, replacement) in lines.iter_mut().zip(reindented_first.split('\n')) { - *line = replacement; - } - lines.join("\n") -} - -/// Like [`reindent_all_lines`] but leaves the first line untouched, for shifting a -/// value's own continuation lines without disturbing its already-placed first line. -fn reindent_continuation(content: &str, delta: isize) -> String { - if delta == 0 || !content.contains('\n') { - return content.to_string(); - } - let mut lines = content.splitn(2, '\n'); - let first = lines.next().unwrap_or_default(); - let rest = lines.next().unwrap_or_default(); - format!("{}\n{}", first, reindent_all_lines(rest, delta)) -} - -/// The width of a list item's own marker + trailing space + checkbox (not -/// including ancestor indentation), e.g. 3 for `"1. "`, 4 for `"10. "` or `"- [x] "`. -pub(crate) fn list_own_prefix_width(ordered: bool, index: usize, start: Option, checked: Option) -> usize { +/// The width of a list item's own marker and trailing space (not including ancestor indentation), +/// e.g. 2 for `"- "`, 3 for `"1. "`, 4 for `"10. "`. A checkbox is not part of it, as the content of +/// the item starts at the checkbox, so children are indented to the marker alone. +pub(crate) fn list_own_prefix_width(ordered: bool, index: usize, start: Option) -> usize { let marker_len = if ordered { (start.unwrap_or(1) as usize + index).to_string().chars().count() + 1 } else { 1 }; - let checkbox_len = if checked.is_some() { 4 } else { 0 }; - marker_len + 1 + checkbox_len + marker_len + 1 } /// Longest fence-char run in `value` + 1 (min 3); switches to tilde when `info` @@ -3358,6 +3474,33 @@ fn escape_label(s: &str) -> String { result } +/// The text of an autolink for a link whose only content is its own destination, when writing that +/// text as a link label would need escapes. A URL in a label is read as it is, so `\_` would stay. +fn autolink_target(url: &str, values: &[Node]) -> Option { + let [Node::Text(Text { value, .. })] = values else { + return None; + }; + let target = if value == url { + url + } else if url.strip_prefix("mailto:") == Some(value.as_str()) && value.contains('@') { + value.as_str() + } else { + return None; + }; + let plain = !target + .chars() + .any(|c| c.is_whitespace() || c.is_control() || matches!(c, '<' | '>')); + let scheme = target.split_once(':').is_some_and(|(scheme, _)| { + (2..=32).contains(&scheme.len()) + && scheme.starts_with(|c: char| c.is_ascii_alphabetic()) + && scheme + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '+' | '.' | '-')) + }); + let email = value != url; + (plain && (scheme || email) && escape_text(target.to_string()) != target).then(|| target.to_string()) +} + /// Re-escapes markdown-significant characters so plain text can't be reinterpreted as /// syntax on re-parse. `# - + > =` only unsafe at line start, `. )` only after leading digits; /// a leading `\t` and runs of 2+ `\n` (only possible via decoded entities) use `&#N;` instead. @@ -3518,15 +3661,15 @@ mod tests { #[case::footnote(Node::Footnote(Footnote {ident: "test".to_string(), values: Vec::new(), position: None }), "test".to_string(), Node::Footnote(Footnote{ident: "test".to_string(), values: Vec::new(), position: None }))] - #[case::list(Node::List(List{start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: vec!["test".to_string().into()], position: None }), + #[case::list(Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: vec!["test".to_string().into()], position: None }), "test".to_string(), - Node::List(List{start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: vec!["test".to_string().into()], position: None }))] - #[case::list(Node::List(List{start: None, spread: false, index: 1, level: 1, checked: Some(true), ordered: false, values: vec!["test".to_string().into()], position: None }), + Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: vec!["test".to_string().into()], position: None }))] + #[case::list(Node::List(List{ marker: None,start: None, spread: false, index: 1, level: 1, checked: Some(true), ordered: false, values: vec!["test".to_string().into()], position: None }), "test".to_string(), - Node::List(List{start: None, spread: false, index: 1, level: 1, checked: Some(true), ordered: false, values: vec!["test".to_string().into()], position: None }))] - #[case::list(Node::List(List{start: None, spread: false, index: 2, level: 2, checked: Some(false), ordered: false, values: vec!["test".to_string().into()], position: None }), + Node::List(List{ marker: None,start: None, spread: false, index: 1, level: 1, checked: Some(true), ordered: false, values: vec!["test".to_string().into()], position: None }))] + #[case::list(Node::List(List{ marker: None,start: None, spread: false, index: 2, level: 2, checked: Some(false), ordered: false, values: vec!["test".to_string().into()], position: None }), "test".to_string(), - Node::List(List{start: None, spread: false, index: 2, level: 2, checked: Some(false), ordered: false, values: vec!["test".to_string().into()], position: None }))] + Node::List(List{ marker: None,start: None, spread: false, index: 2, level: 2, checked: Some(false), ordered: false, values: vec!["test".to_string().into()], position: None }))] #[case::code_inline(Node::CodeInline(CodeInline{ value: "t".into(), position: None }), "test".to_string(), Node::CodeInline(CodeInline{ value: "test".into(), position: None }))] @@ -3568,9 +3711,9 @@ mod tests { #[case::break_(Node::Break(Break{ position: None}), "test".to_string(), Node::Break(Break{position: None}))] - #[case::horizontal_rule(Node::HorizontalRule(HorizontalRule{ position: None}), + #[case::horizontal_rule(Node::HorizontalRule(HorizontalRule{ marker: None, position: None}), "test".to_string(), - Node::HorizontalRule(HorizontalRule{position: None}))] + Node::HorizontalRule(HorizontalRule{ marker: None,position: None}))] #[case::mdx_flow_expression(Node::MdxFlowExpression(MdxFlowExpression{value: "test".into(), position: None}), "updated".to_string(), Node::MdxFlowExpression(MdxFlowExpression{value: "updated".into(), position: None}))] @@ -3675,13 +3818,13 @@ mod tests { Node::Text(Text{value: "first".to_string(), position: None}), Node::Text(Text{value: "new".to_string(), position: None}) ], position: None}))] - #[case(Node::List(List{start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: vec![ + #[case(Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: vec![ Node::Text(Text{value: "first".to_string(), position: None}), Node::Text(Text{value: "second".to_string(), position: None}) ], position: None}), "new", 0, - Node::List(List{start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: vec![ + Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: vec![ Node::Text(Text{value: "new".to_string(), position: None}), Node::Text(Text{value: "second".to_string(), position: None}) ], position: None}))] @@ -3703,20 +3846,20 @@ mod tests { "new", 0, Node::Code(Code{value: "code".to_string(), lang: None, fence: true, meta: None, position: None}))] - #[case(Node::List(List{start: None, spread: false, index: 0, level: 1, checked: Some(true), ordered: false, values: vec![ + #[case(Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 1, checked: Some(true), ordered: false, values: vec![ Node::Text(Text{value: "first".to_string(), position: None}) ], position: None}), "new", 0, - Node::List(List{start: None, spread: false, index: 0, level: 1, checked: Some(true), ordered: false, values: vec![ + Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 1, checked: Some(true), ordered: false, values: vec![ Node::Text(Text{value: "new".to_string(), position: None}) ], position: None}))] - #[case(Node::List(List{start: None, spread: false, index: 0, level: 1, checked: None, ordered: false, values: vec![ + #[case(Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 1, checked: None, ordered: false, values: vec![ Node::Text(Text{value: "first".to_string(), position: None}) ], position: None}), "new", 2, - Node::List(List{start: None, spread: false, index: 0, level: 1, checked: None, ordered: false, values: vec![ + Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 1, checked: None, ordered: false, values: vec![ Node::Text(Text{value: "first".to_string(), position: None}) ], position: None}))] #[case::link_ref(Node::LinkRef(LinkRef{ident: "id".to_string(), values: vec![ @@ -3814,7 +3957,7 @@ mod tests { #[rstest] #[case(Node::Text(Text{value: "test".to_string(), position: None }), "test".to_string())] - #[case(Node::List(List{start: None, spread: false, index: 0, level: 2, checked: None, ordered: false, values: vec!["test".to_string().into()], position: None}), + #[case(Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 2, checked: None, ordered: false, values: vec!["test".to_string().into()], position: None}), " - test".to_string())] fn test_display(#[case] node: Node, #[case] expected: String) { assert_eq!(node.to_string_with(&RenderOptions::default()), expected); @@ -3904,7 +4047,7 @@ mod tests { } #[rstest] - #[case(Node::HorizontalRule(HorizontalRule{position: None}), true)] + #[case(Node::HorizontalRule(HorizontalRule{ marker: None,position: None}), true)] #[case(Node::Text(Text{value: "test".to_string(), position: None}), false)] fn test_is_horizontal_rule(#[case] node: Node, #[case] expected: bool) { assert_eq!(node.is_horizontal_rule(), expected); @@ -4135,7 +4278,7 @@ mod tests { &Node::Heading(Heading{depth: 1, values: vec!["test".to_string().into()], position: None})), vec!["test".to_string().into()])] #[case(Node::node_values( - &Node::List(List{values: vec!["test".to_string().into()], ordered: false, level: 1, checked: Some(false), index: 0, start: None, spread: false, position: None})), + &Node::List(List{ marker: None,values: vec!["test".to_string().into()], ordered: false, level: 1, checked: Some(false), index: 0, start: None, spread: false, position: None})), vec!["test".to_string().into()])] fn test_node_value(#[case] actual: Vec, #[case] expected: Vec) { assert_eq!(actual, expected); @@ -4234,12 +4377,12 @@ mod tests { #[rstest] #[case::text(Node::Text(Text{value: "test".to_string(), position: None }), RenderOptions::default(), "test")] - #[case::list(Node::List(List{start: None, spread: false, index: 0, level: 2, checked: None, ordered: false, values: vec!["test".to_string().into()], position: None}), RenderOptions::default(), " - test")] - #[case::list(Node::List(List{start: None, spread: false, index: 0, level: 1, checked: None, ordered: false, values: vec!["test".to_string().into()], position: None}), RenderOptions { list_style: ListStyle::Plus, ..Default::default() }, " + test")] - #[case::list(Node::List(List{start: None, spread: false, index: 0, level: 1, checked: Some(true), ordered: false, values: vec!["test".to_string().into()], position: None}), RenderOptions { list_style: ListStyle::Star, ..Default::default() }, " * [x] test")] - #[case::list(Node::List(List{start: None, spread: false, index: 0, level: 1, checked: Some(false), ordered: false, values: vec!["test".to_string().into()], position: None}), RenderOptions::default(), " - [ ] test")] - #[case::list(Node::List(List{start: None, spread: false, index: 0, level: 1, checked: None, ordered: true, values: vec!["test".to_string().into()], position: None}), RenderOptions::default(), " 1. test")] - #[case::list(Node::List(List{start: None, spread: false, index: 0, level: 1, checked: Some(false), ordered: true, values: vec!["test".to_string().into()], position: None}), RenderOptions::default(), " 1. [ ] test")] + #[case::list(Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 2, checked: None, ordered: false, values: vec!["test".to_string().into()], position: None}), RenderOptions::default(), " - test")] + #[case::list(Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 1, checked: None, ordered: false, values: vec!["test".to_string().into()], position: None}), RenderOptions { list_style: Some(ListStyle::Plus), ..Default::default() }, " + test")] + #[case::list(Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 1, checked: Some(true), ordered: false, values: vec!["test".to_string().into()], position: None}), RenderOptions { list_style: Some(ListStyle::Star), ..Default::default() }, " * [x] test")] + #[case::list(Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 1, checked: Some(false), ordered: false, values: vec!["test".to_string().into()], position: None}), RenderOptions::default(), " - [ ] test")] + #[case::list(Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 1, checked: None, ordered: true, values: vec!["test".to_string().into()], position: None}), RenderOptions::default(), " 1. test")] + #[case::list(Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 1, checked: Some(false), ordered: true, values: vec!["test".to_string().into()], position: None}), RenderOptions::default(), " 1. [ ] test")] #[case::table_row(Node::TableRow(TableRow{values: vec![Node::TableCell(TableCell{column: 0, row: 0, values: vec!["test".to_string().into()], position: None})], position: None}), RenderOptions::default(), "|test|")] #[case::table_row(Node::TableRow(TableRow{values: vec![Node::TableCell(TableCell{column: 0, row: 0, values: vec!["test".to_string().into()], position: None})], position: None}), RenderOptions::default(), "|test|")] #[case::table_cell(Node::TableCell(TableCell{column: 0, row: 0, values: vec!["test".to_string().into()], position: None}), RenderOptions::default(), "test")] @@ -4293,7 +4436,7 @@ mod tests { #[case::yaml(Node::Yaml(Yaml{value: "key: value".to_string(), position: None}), RenderOptions::default(), "---\nkey: value\n---")] #[case::toml(Node::Toml(Toml{value: "key = \"value\"".to_string(), position: None}), RenderOptions::default(), "+++\nkey = \"value\"\n+++")] #[case::break_(Node::Break(Break{position: None}), RenderOptions::default(), "\\\n")] - #[case::horizontal_rule(Node::HorizontalRule(HorizontalRule{position: None}), RenderOptions::default(), "***")] + #[case::horizontal_rule(Node::HorizontalRule(HorizontalRule{ marker: None,position: None}), RenderOptions::default(), "***")] #[case::mdx_jsx_flow_element(Node::MdxJsxFlowElement(MdxJsxFlowElement{ name: Some("div".to_string()), attributes: vec![ @@ -4490,13 +4633,13 @@ mod tests { #[case(Node::Link(Link{url: Url::new("".to_string()), title: None, values: Vec::new(), position: None}), "link")] #[case(Node::LinkRef(LinkRef{ident: "".to_string(), values: Vec::new(), label: None, position: None}), "link_ref")] #[case(Node::Math(Math{value: "".to_string(), position: None}), "math")] - #[case(Node::List(List{start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: Vec::new(), position: None}), "list")] + #[case(Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: Vec::new(), position: None}), "list")] #[case(Node::TableAlign(TableAlign{align: Vec::new(), position: None}), "table_align")] #[case(Node::TableRow(TableRow{values: Vec::new(), position: None}), "table_row")] #[case(Node::TableCell(TableCell{column: 0, row: 0, values: Vec::new(), position: None}), "table_cell")] #[case(Node::Code(Code{value: "".to_string(), lang: None, fence: true, meta: None, position: None}), "code")] #[case(Node::Strong(Strong{values: Vec::new(), position: None}), "strong")] - #[case(Node::HorizontalRule(HorizontalRule{position: None}), "Horizontal_rule")] + #[case(Node::HorizontalRule(HorizontalRule{ marker: None,position: None}), "Horizontal_rule")] #[case(Node::MdxFlowExpression(MdxFlowExpression{value: "".into(), position: None}), "mdx_flow_expression")] #[case(Node::MdxJsxFlowElement(MdxJsxFlowElement{name: None, attributes: Vec::new(), children: Vec::new(), position: None}), "mdx_jsx_flow_element")] #[case(Node::MdxJsxTextElement(MdxJsxTextElement{name: None, attributes: Vec::new(), children: Vec::new(), position: None}), "mdx_jsx_text_element")] @@ -4509,7 +4652,7 @@ mod tests { #[rstest] #[case(Node::Text(Text{value: "test".to_string(), position: None}), "test")] - #[case(Node::List(List{start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: vec![Node::Text(Text{value: "test".to_string(), position: None})], position: None}), "test")] + #[case(Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: vec![Node::Text(Text{value: "test".to_string(), position: None})], position: None}), "test")] #[case(Node::Blockquote(Blockquote{values: vec![Node::Text(Text{value: "test".to_string(), position: None})], position: None}), "test")] #[case(Node::Delete(Delete{values: vec![Node::Text(Text{value: "test".to_string(), position: None})], position: None}), "test")] #[case(Node::Heading(Heading{depth: 1, values: vec![Node::Text(Text{value: "test".to_string(), position: None})], position: None}), "test")] @@ -4531,7 +4674,7 @@ mod tests { #[case(Node::TableCell(TableCell{column: 0, row: 0, values: vec![Node::Text(Text{value: "test".to_string(), position: None})], position: None}), "test")] #[case(Node::TableRow(TableRow{values: vec![Node::TableCell(TableCell{column: 0, row: 0, values: vec![Node::Text(Text{value: "test".to_string(), position: None})], position: None})], position: None}), "test")] #[case(Node::Break(Break{position: None}), "")] - #[case(Node::HorizontalRule(HorizontalRule{position: None}), "")] + #[case(Node::HorizontalRule(HorizontalRule{ marker: None,position: None}), "")] #[case(Node::TableAlign(TableAlign{align: Vec::new(), position: None}), "")] #[case(Node::MdxFlowExpression(MdxFlowExpression{value: "test".into(), position: None}), "test")] #[case(Node::MdxTextExpression(MdxTextExpression{value: "test".into(), position: None}), "test")] @@ -4546,7 +4689,7 @@ mod tests { #[rstest] #[case(Node::Text(Text{value: "test".to_string(), position: None}), None)] #[case(Node::Text(Text{value: "test".to_string(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] - #[case(Node::List(List{start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: Vec::new(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] + #[case(Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: Vec::new(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] #[case(Node::Blockquote(Blockquote{values: Vec::new(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] #[case(Node::Delete(Delete{values: Vec::new(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] #[case(Node::Heading(Heading{depth: 1, values: Vec::new(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] @@ -4569,7 +4712,7 @@ mod tests { #[case(Node::TableRow(TableRow{values: Vec::new(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] #[case(Node::TableAlign(TableAlign{align: Vec::new(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] #[case(Node::Break(Break{position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] - #[case(Node::HorizontalRule(HorizontalRule{position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] + #[case(Node::HorizontalRule(HorizontalRule{ marker: None,position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] #[case(Node::MdxFlowExpression(MdxFlowExpression{value: "test".into(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] #[case(Node::MdxTextExpression(MdxTextExpression{value: "test".into(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] #[case(Node::MdxJsEsm(MdxJsEsm{value: "test".into(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] @@ -4629,7 +4772,7 @@ mod tests { Node::Text(Text{value: "first".to_string(), position: None}), Node::Text(Text{value: "second".to_string(), position: None}) ], position: None}), 0, Some(Node::Text(Text{value: "first".to_string(), position: None})))] - #[case(Node::List(List{start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: vec![ + #[case(Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: vec![ Node::Text(Text{value: "first".to_string(), position: None}), Node::Text(Text{value: "second".to_string(), position: None}) ], position: None}), 1, Some(Node::Text(Text{value: "second".to_string(), position: None})))] @@ -4656,7 +4799,7 @@ mod tests { Node::Fragment(Fragment{values: vec!["test".to_string().into()]}))] #[case(Node::Emphasis(Emphasis{values: vec!["test".to_string().into()], position: None}), Node::Fragment(Fragment{values: vec!["test".to_string().into()]}))] - #[case(Node::List(List{start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: vec!["test".to_string().into()], position: None}), + #[case(Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: vec!["test".to_string().into()], position: None}), Node::Fragment(Fragment{values: vec!["test".to_string().into()]}))] #[case(Node::Strong(Strong{values: vec!["test".to_string().into()], position: None}), Node::Fragment(Fragment{values: vec!["test".to_string().into()]}))] @@ -4699,8 +4842,8 @@ mod tests { #[case::heading_mixed(Node::Heading(Heading{depth: 1, values: vec![Node::Empty, Node::Text(Text{value: "kept".to_string(), position: None})], position: None}), "kept")] #[case::emphasis_all_empty(Node::Emphasis(Emphasis{values: vec![Node::Empty, Node::Empty], position: None}), "")] #[case::emphasis_mixed(Node::Emphasis(Emphasis{values: vec![Node::Empty, Node::Text(Text{value: "kept".to_string(), position: None})], position: None}), "kept")] - #[case::list_all_empty(Node::List(List{start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: vec![Node::Empty, Node::Empty], position: None}), "")] - #[case::list_mixed(Node::List(List{start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: vec![Node::Empty, Node::Text(Text{value: "kept".to_string(), position: None})], position: None}), "kept")] + #[case::list_all_empty(Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: vec![Node::Empty, Node::Empty], position: None}), "")] + #[case::list_mixed(Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: vec![Node::Empty, Node::Text(Text{value: "kept".to_string(), position: None})], position: None}), "kept")] #[case::strong_all_empty(Node::Strong(Strong{values: vec![Node::Empty, Node::Empty], position: None}), "")] #[case::strong_mixed(Node::Strong(Strong{values: vec![Node::Text(Text{value: "kept".to_string(), position: None}), Node::Empty], position: None}), "kept")] #[case::link_all_empty(Node::Link(Link{url: Url(attr_keys::URL.to_string()), title: None, values: vec![Node::Empty, Node::Empty], position: None}), "")] @@ -4761,13 +4904,13 @@ mod tests { ], position: None}) )] #[case( - &mut Node::List(List{start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: vec![ + &mut Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: vec![ Node::Text(Text{value: "old".to_string(), position: None}) ], position: None}), Node::Fragment(Fragment{values: vec![ Node::Text(Text{value: "new".to_string(), position: None}) ]}), - Node::List(List{start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: vec![ + Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: vec![ Node::Text(Text{value: "new".to_string(), position: None}) ], position: None}) )] @@ -4879,7 +5022,7 @@ mod tests { ], position: None}) )] #[case( - &mut Node::List(List{start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: vec![ + &mut Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: vec![ Node::Text(Text{value: "text1".to_string(), position: None}), Node::Text(Text{value: "text2".to_string(), position: None}) ], position: None}), @@ -4887,7 +5030,7 @@ mod tests { Node::Text(Text{value: "new1".to_string(), position: None}), Node::Fragment(Fragment{values: Vec::new()}) ]}), - Node::List(List{start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: vec![ + Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: vec![ Node::Text(Text{value: "new1".to_string(), position: None}), Node::Text(Text{value: "text2".to_string(), position: None}) ], position: None}) @@ -4904,9 +5047,9 @@ mod tests { #[case(Node::Code(Code{value: "code".to_string(), lang: None, fence: true, meta: None, position: None}), Position{start: Point{line: 1, column: 1}, end: Point{line: 3, column: 3}}, Node::Code(Code{value: "code".to_string(), lang: None, fence: true, meta: None, position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 3, column: 3}})}))] - #[case(Node::List(List{start: None, spread: false, index: 0, level: 1, checked: None, ordered: false, values: vec![], position: None}), + #[case(Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 1, checked: None, ordered: false, values: vec![], position: None}), Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}, - Node::List(List{start: None, spread: false, index: 0, level: 1, checked: None, ordered: false, values: vec![], position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}))] + Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 1, checked: None, ordered: false, values: vec![], position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}))] #[case(Node::Definition(Definition{ident: "id".to_string(), url: Url::new(attr_keys::URL.to_string()), title: None, label: None, position: None}), Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 10}}, Node::Definition(Definition{ident: "id".to_string(), url: Url::new(attr_keys::URL.to_string()), title: None, label: None, position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 10}})}))] @@ -5090,6 +5233,7 @@ mod tests { #[test] fn test_clear_text_position_at_selected_index() { let mut node = Node::List(List { + marker: None, start: None, spread: false, index: 0, @@ -5120,8 +5264,8 @@ mod tests { } #[rstest] - #[case(Node::List(List{start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: vec!["test".to_string().into()], position: None}), true)] - #[case(Node::List(List{start: None, spread: false, index: 1, level: 2, checked: Some(true), ordered: false, values: vec!["test".to_string().into()], position: None}), true)] + #[case(Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: vec!["test".to_string().into()], position: None}), true)] + #[case(Node::List(List{ marker: None,start: None, spread: false, index: 1, level: 2, checked: Some(true), ordered: false, values: vec!["test".to_string().into()], position: None}), true)] #[case(Node::Text(Text{value: "test".to_string(), position: None}), false)] fn test_is_list(#[case] node: Node, #[case] expected: bool) { assert_eq!(node.is_list(), expected); @@ -5199,10 +5343,10 @@ mod tests { #[case::definition(Node::Definition(Definition{ident: "id".to_string(), url: Url::new(attr_keys::URL.to_string()), title: Some(Title::new(attr_keys::TITLE.to_string())), label: Some(attr_keys::LABEL.to_string()), position: None}), attr_keys::TITLE, Some(AttrValue::String(attr_keys::TITLE.to_string())))] #[case::definition(Node::Definition(Definition{ident: "id".to_string(), url: Url::new(attr_keys::URL.to_string()), title: Some(Title::new(attr_keys::TITLE.to_string())), label: Some(attr_keys::LABEL.to_string()), position: None}), attr_keys::LABEL, Some(AttrValue::String(attr_keys::LABEL.to_string())))] #[case::heading(Node::Heading(Heading{depth: 3, values: Vec::new(), position: None}), "depth", Some(AttrValue::Integer(3)))] - #[case::list(Node::List(List{start: None, spread: false, index: 2, level: 1, checked: Some(true), ordered: true, values: Vec::new(), position: None}), "index", Some(AttrValue::Integer(2)))] - #[case::list(Node::List(List{start: None, spread: false, index: 2, level: 1, checked: Some(true), ordered: true, values: Vec::new(), position: None}), "level", Some(AttrValue::Integer(1)))] - #[case::list(Node::List(List{start: None, spread: false, index: 2, level: 1, checked: Some(true), ordered: true, values: Vec::new(), position: None}), "ordered", Some(AttrValue::Boolean(true)))] - #[case::list(Node::List(List{start: None, spread: false, index: 2, level: 1, checked: Some(true), ordered: true, values: Vec::new(), position: None}), attr_keys::CHECKED, Some(AttrValue::Boolean(true)))] + #[case::list(Node::List(List{ marker: None,start: None, spread: false, index: 2, level: 1, checked: Some(true), ordered: true, values: Vec::new(), position: None}), "index", Some(AttrValue::Integer(2)))] + #[case::list(Node::List(List{ marker: None,start: None, spread: false, index: 2, level: 1, checked: Some(true), ordered: true, values: Vec::new(), position: None}), "level", Some(AttrValue::Integer(1)))] + #[case::list(Node::List(List{ marker: None,start: None, spread: false, index: 2, level: 1, checked: Some(true), ordered: true, values: Vec::new(), position: None}), "ordered", Some(AttrValue::Boolean(true)))] + #[case::list(Node::List(List{ marker: None,start: None, spread: false, index: 2, level: 1, checked: Some(true), ordered: true, values: Vec::new(), position: None}), attr_keys::CHECKED, Some(AttrValue::Boolean(true)))] #[case::table_cell(Node::TableCell(TableCell{column: 1, row: 2, values: Vec::new(), position: None}), "column", Some(AttrValue::Integer(1)))] #[case::table_cell(Node::TableCell(TableCell{column: 1, row: 2, values: Vec::new(), position: None}), "row", Some(AttrValue::Integer(2)))] #[case::table_align(Node::TableAlign(TableAlign{align: vec![TableAlignKind::Left, TableAlignKind::Right], position: None}), "align", Some(AttrValue::String(":---,---:".to_string())))] @@ -5212,7 +5356,7 @@ mod tests { #[case::mdx_jsx_flow_element(Node::MdxJsxFlowElement(MdxJsxFlowElement{name: Some("div".to_string()), attributes: Vec::new(), children: Vec::new(), position: None}), attr_keys::NAME, Some(AttrValue::String("div".to_string())))] #[case::mdx_jsx_flow_element(Node::MdxJsxTextElement(MdxJsxTextElement{name: Some("span".into()), attributes: Vec::new(), children: Vec::new(), position: None}), attr_keys::NAME, Some(AttrValue::String("span".to_string())))] #[case::break_(Node::Break(Break{position: None}), attr_keys::VALUE, None)] - #[case::horizontal_rule(Node::HorizontalRule(HorizontalRule{position: None}), attr_keys::VALUE, None)] + #[case::horizontal_rule(Node::HorizontalRule(HorizontalRule{ marker: None,position: None}), attr_keys::VALUE, None)] #[case::fragment(Node::Fragment(Fragment{values: Vec::new()}), attr_keys::VALUE, Some(AttrValue::String("".to_string())))] #[case::heading(Node::Heading(Heading{depth: 1, values: vec![Node::Text(Text{value: "heading text".to_string(), position: None})], position: None}), attr_keys::VALUE, Some(AttrValue::String("heading text".to_string())))] #[case::heading(Node::Heading(Heading{depth: 2, values: vec![], position: None}), attr_keys::VALUE, Some(AttrValue::String("".to_string())))] @@ -5221,7 +5365,7 @@ mod tests { Node::Text(Text{value: "second".to_string(), position: None}), ], position: None}), attr_keys::VALUE, Some(AttrValue::String("firstsecond".to_string())))] #[case( - Node::List(List { + Node::List(List { marker: None, index: 0, level: 1, checked: None, @@ -5303,7 +5447,7 @@ mod tests { ])) )] #[case::list( - Node::List(List { + Node::List(List { marker: None, index: 0, level: 1, checked: None, @@ -5469,9 +5613,9 @@ mod tests { Node::Heading(Heading{depth: 1, values: vec![Node::Text(Text{value: "child".to_string(), position: None})], position: None}) )] #[case::list( - Node::List(List{start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: vec![], position: None}), + Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: vec![], position: None}), vec![Node::Text(Text{value: "item".to_string(), position: None})], - Node::List(List{start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: vec![Node::Text(Text{value: "item".to_string(), position: None})], position: None}) + Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: vec![Node::Text(Text{value: "item".to_string(), position: None})], position: None}) )] #[case::blockquote( Node::Blockquote(Blockquote{values: vec![], position: None}), @@ -5596,16 +5740,16 @@ mod tests { Node::Heading(Heading{depth: 3, values: vec![], position: None}) )] #[case( - Node::List(List{start: None, spread: false, index: 1, level: 2, checked: Some(true), ordered: false, values: vec![], position: None}), + Node::List(List{ marker: None,start: None, spread: false, index: 1, level: 2, checked: Some(true), ordered: false, values: vec![], position: None}), attr_keys::CHECKED, "false", - Node::List(List{start: None, spread: false, index: 1, level: 2, checked: Some(false), ordered: false, values: vec![], position: None}) + Node::List(List{ marker: None,start: None, spread: false, index: 1, level: 2, checked: Some(false), ordered: false, values: vec![], position: None}) )] #[case( - Node::List(List{start: None, spread: false, index: 1, level: 2, checked: Some(true), ordered: false, values: vec![], position: None}), + Node::List(List{ marker: None,start: None, spread: false, index: 1, level: 2, checked: Some(true), ordered: false, values: vec![], position: None}), "ordered", "true", - Node::List(List{start: None, spread: false, index: 1, level: 2, checked: Some(true), ordered: true, values: vec![], position: None}) + Node::List(List{ marker: None,start: None, spread: false, index: 1, level: 2, checked: Some(true), ordered: true, values: vec![], position: None}) )] #[case( Node::TableCell(TableCell{column: 1, row: 2, values: vec![], position: None}), @@ -5848,7 +5992,7 @@ mod tests { #[case( vec![Node::Text(Text { value: "[!NOTE]\nbody text".to_string(), position: None })], Node::Callout(Callout { - kind: "NOTE".to_string(), + fold: None, kind: "NOTE".to_string(), title: None, values: vec![Node::Text(Text { value: "body text".to_string(), position: None })], position: None, @@ -5858,7 +6002,7 @@ mod tests { #[case( vec![Node::Text(Text { value: "[!WARNING] My Title\nbody".to_string(), position: None })], Node::Callout(Callout { - kind: "WARNING".to_string(), + fold: None, kind: "WARNING".to_string(), title: Some("My Title".to_string()), values: vec![Node::Text(Text { value: "body".to_string(), position: None })], position: None, @@ -5867,13 +6011,13 @@ mod tests { // [!TIP] with no body #[case( vec![Node::Text(Text { value: "[!TIP]".to_string(), position: None })], - Node::Callout(Callout { kind: "TIP".to_string(), title: None, values: vec![], position: None }) + Node::Callout(Callout { fold: None, kind: "TIP".to_string(), title: None, values: vec![], position: None }) )] // lowercase kind is preserved as-is #[case( vec![Node::Text(Text { value: "[!note]\ncontent".to_string(), position: None })], Node::Callout(Callout { - kind: "note".to_string(), + fold: None, kind: "note".to_string(), title: None, values: vec![Node::Text(Text { value: "content".to_string(), position: None })], position: None, @@ -5910,7 +6054,7 @@ mod tests { Node::Text(Text { value: "second paragraph".to_string(), position: None }), ], Node::Callout(Callout { - kind: "NOTE".to_string(), + fold: None, kind: "NOTE".to_string(), title: None, values: vec![Node::Text(Text { value: "second paragraph".to_string(), position: None })], position: None, @@ -5925,33 +6069,33 @@ mod tests { #[rstest] // basic render #[case( - Node::Callout(Callout { kind: "NOTE".to_string(), title: None, + Node::Callout(Callout { fold: None, kind: "NOTE".to_string(), title: None, values: vec![Node::Text(Text { value: "content".to_string(), position: None })], position: None }), "> [!NOTE]\n> content" )] // with title #[case( - Node::Callout(Callout { kind: "WARNING".to_string(), title: Some("Heads up".to_string()), + Node::Callout(Callout { fold: None, kind: "WARNING".to_string(), title: Some("Heads up".to_string()), values: vec![Node::Text(Text { value: "watch out".to_string(), position: None })], position: None }), "> [!WARNING] Heads up\n> watch out" )] // empty body #[case( - Node::Callout(Callout { kind: "TIP".to_string(), title: None, values: vec![], position: None }), + Node::Callout(Callout { fold: None, kind: "TIP".to_string(), title: None, values: vec![], position: None }), "> [!TIP]" )] // multiline body: single Text with embedded '\n' — each line prefixed with "> " #[case( - Node::Callout(Callout { kind: "INFO".to_string(), title: None, + Node::Callout(Callout { fold: None, kind: "INFO".to_string(), title: None, values: vec![Node::Text(Text { value: "line one\nline two".to_string(), position: None })], position: None }), "> [!INFO]\n> line one\n> line two" )] // two separate position-less Text values are concatenated inline (no implicit newline) #[case( - Node::Callout(Callout { kind: "INFO".to_string(), title: None, + Node::Callout(Callout { fold: None, kind: "INFO".to_string(), title: None, values: vec![ Node::Text(Text { value: "part a".to_string(), position: None }), Node::Text(Text { value: "part b".to_string(), position: None }), @@ -5965,11 +6109,11 @@ mod tests { #[cfg(feature = "callout")] #[rstest] - #[case(Node::Callout(Callout { kind: "NOTE".to_string(), title: None, values: vec![], position: None }), "kind", Some(AttrValue::String("NOTE".to_string())))] - #[case(Node::Callout(Callout { kind: "WARNING".to_string(), title: Some("Title".to_string()), values: vec![], position: None }), "title", Some(AttrValue::String("Title".to_string())))] - #[case(Node::Callout(Callout { kind: "TIP".to_string(), title: None, values: vec![], position: None }), "title", None)] - #[case(Node::Callout(Callout { kind: "NOTE".to_string(), title: None, values: vec![Node::Text(Text { value: "body".to_string(), position: None })], position: None }), "value", Some(AttrValue::String("body".to_string())))] - #[case(Node::Callout(Callout { kind: "NOTE".to_string(), title: None, values: vec![Node::Text(Text { value: "body".to_string(), position: None })], position: None }), "children", Some(AttrValue::Array(vec![Node::Text(Text { value: "body".to_string(), position: None })])))] + #[case(Node::Callout(Callout { fold: None, kind: "NOTE".to_string(), title: None, values: vec![], position: None }), "kind", Some(AttrValue::String("NOTE".to_string())))] + #[case(Node::Callout(Callout { fold: None, kind: "WARNING".to_string(), title: Some("Title".to_string()), values: vec![], position: None }), "title", Some(AttrValue::String("Title".to_string())))] + #[case(Node::Callout(Callout { fold: None, kind: "TIP".to_string(), title: None, values: vec![], position: None }), "title", None)] + #[case(Node::Callout(Callout { fold: None, kind: "NOTE".to_string(), title: None, values: vec![Node::Text(Text { value: "body".to_string(), position: None })], position: None }), "value", Some(AttrValue::String("body".to_string())))] + #[case(Node::Callout(Callout { fold: None, kind: "NOTE".to_string(), title: None, values: vec![Node::Text(Text { value: "body".to_string(), position: None })], position: None }), "children", Some(AttrValue::Array(vec![Node::Text(Text { value: "body".to_string(), position: None })])))] fn test_callout_attr(#[case] node: Node, #[case] attr: &str, #[case] expected: Option) { assert_eq!(node.attr(attr), expected); } @@ -5977,26 +6121,26 @@ mod tests { #[cfg(feature = "callout")] #[rstest] #[case( - Node::Callout(Callout { kind: "NOTE".to_string(), title: None, values: vec![], position: None }), + Node::Callout(Callout { fold: None, kind: "NOTE".to_string(), title: None, values: vec![], position: None }), "kind", "WARNING", - Node::Callout(Callout { kind: "WARNING".to_string(), title: None, values: vec![], position: None }) + Node::Callout(Callout { fold: None, kind: "WARNING".to_string(), title: None, values: vec![], position: None }) )] // set_attr stores kind as-is without case conversion #[case( - Node::Callout(Callout { kind: "NOTE".to_string(), title: None, values: vec![], position: None }), + Node::Callout(Callout { fold: None, kind: "NOTE".to_string(), title: None, values: vec![], position: None }), "kind", "tip", - Node::Callout(Callout { kind: "tip".to_string(), title: None, values: vec![], position: None }) + Node::Callout(Callout { fold: None, kind: "tip".to_string(), title: None, values: vec![], position: None }) )] #[case( - Node::Callout(Callout { kind: "NOTE".to_string(), title: None, values: vec![], position: None }), + Node::Callout(Callout { fold: None, kind: "NOTE".to_string(), title: None, values: vec![], position: None }), "title", "My title", - Node::Callout(Callout { kind: "NOTE".to_string(), title: Some("My title".to_string()), values: vec![], position: None }) + Node::Callout(Callout { fold: None, kind: "NOTE".to_string(), title: Some("My title".to_string()), values: vec![], position: None }) )] // empty string clears title #[case( - Node::Callout(Callout { kind: "NOTE".to_string(), title: Some("old".to_string()), values: vec![], position: None }), + Node::Callout(Callout { fold: None, kind: "NOTE".to_string(), title: Some("old".to_string()), values: vec![], position: None }), "title", "", - Node::Callout(Callout { kind: "NOTE".to_string(), title: None, values: vec![], position: None }) + Node::Callout(Callout { fold: None, kind: "NOTE".to_string(), title: None, values: vec![], position: None }) )] fn test_callout_set_attr(#[case] mut node: Node, #[case] attr: &str, #[case] value: &str, #[case] expected: Node) { node.set_attr(attr, value); @@ -6007,11 +6151,11 @@ mod tests { #[rstest] // with_value changes first body node #[case( - Node::Callout(Callout { kind: "NOTE".to_string(), title: None, + Node::Callout(Callout { fold: None, kind: "NOTE".to_string(), title: None, values: vec![Node::Text(Text { value: "old".to_string(), position: None })], position: None }), "new", - Node::Callout(Callout { kind: "NOTE".to_string(), title: None, + Node::Callout(Callout { fold: None, kind: "NOTE".to_string(), title: None, values: vec![Node::Text(Text { value: "new".to_string(), position: None })], position: None }) )] @@ -6022,7 +6166,7 @@ mod tests { #[cfg(feature = "callout")] #[rstest] - #[case(Node::Callout(Callout { kind: "NOTE".to_string(), title: None, values: vec![], position: None }), "callout")] + #[case(Node::Callout(Callout { fold: None, kind: "NOTE".to_string(), title: None, values: vec![], position: None }), "callout")] fn test_callout_name(#[case] node: Node, #[case] expected: &str) { assert_eq!(node.name(), expected); } @@ -6030,7 +6174,7 @@ mod tests { #[cfg(feature = "callout")] #[rstest] #[case( - Node::Callout(Callout { kind: "NOTE".to_string(), title: None, + Node::Callout(Callout { fold: None, kind: "NOTE".to_string(), title: None, values: vec![Node::Text(Text { value: "body".to_string(), position: None })], position: None }), "body" @@ -6041,7 +6185,7 @@ mod tests { #[cfg(feature = "callout")] #[rstest] - #[case(Node::Callout(Callout { kind: "NOTE".to_string(), title: None, values: vec![], position: None }), true)] + #[case(Node::Callout(Callout { fold: None, kind: "NOTE".to_string(), title: None, values: vec![], position: None }), true)] #[case(Node::Blockquote(Blockquote { values: vec![], position: None }), false)] #[case(Node::Text(Text { value: "test".to_string(), position: None }), false)] fn test_is_callout(#[case] node: Node, #[case] expected: bool) { @@ -6456,14 +6600,12 @@ mod tests { } #[rstest] - #[case::no_newline_unchanged("abc", 5, "abc")] - #[case::zero_delta_unchanged("a\n b", 0, "a\n b")] - #[case::positive_delta_adds_spaces("a\n b", 2, "a\n b")] - #[case::negative_delta_removes_spaces_clamped_at_zero("a\n b", -5, "a\nb")] - #[case::first_line_never_shifted(" a\nb", 3, " a\n b")] + #[case::no_newline_unchanged("abc", 2, "abc")] + #[case::continuation_indented("a\nb", 2, "a\n b")] #[case::empty_lines_untouched("a\n\nb", 2, "a\n\n b")] - fn test_reindent_continuation(#[case] content: &str, #[case] delta: isize, #[case] expected: &str) { - assert_eq!(reindent_continuation(content, delta), expected); + #[case::first_line_never_indented(" a\nb", 3, " a\n b")] + fn test_indent_continuation(#[case] content: &str, #[case] width: usize, #[case] expected: &str) { + assert_eq!(indent_continuation(content, width), expected); } #[rstest] @@ -6476,57 +6618,17 @@ mod tests { } #[rstest] - #[case::code_first_child_indented( - Some(Node::Code(Code { value: "foo".to_string(), lang: None, fence: true, meta: None, position: None })), - "```\nfoo\n```".to_string(), - 3, - "```\n foo\n ```" - )] - #[case::empty_code_first_child_closing_fence_indented( - Some(Node::Code(Code { value: "".to_string(), lang: None, fence: true, meta: None, position: None })), - "```\n```".to_string(), - 3, - "```\n ```" - )] - #[case::non_leaf_first_child_untouched( - Some(Node::Text(Text { value: "foo".to_string(), position: None })), - "foo".to_string(), - 3, - "foo" - )] - #[case::no_first_child_untouched(None, "".to_string(), 3, "")] - fn test_reindent_first_leaf_block( - #[case] first: Option, - #[case] content: String, - #[case] target_indent: usize, - #[case] expected: &str, - ) { - assert_eq!( - reindent_first_leaf_block( - content, - first.as_ref(), - &RenderOptions::default(), - &ColorTheme::PLAIN, - target_indent - ), - expected - ); - } - - #[rstest] - #[case::unordered_bullet(false, 0, None, None, 2)] - #[case::unordered_checked(false, 0, None, Some(true), 6)] - #[case::ordered_single_digit(true, 0, None, None, 3)] - #[case::ordered_double_digit(true, 9, None, None, 4)] - #[case::ordered_with_start(true, 0, Some(10), None, 4)] + #[case::unordered_bullet(false, 0, None, 2)] + #[case::ordered_single_digit(true, 0, None, 3)] + #[case::ordered_double_digit(true, 9, None, 4)] + #[case::ordered_with_start(true, 0, Some(10), 4)] fn test_list_own_prefix_width( #[case] ordered: bool, #[case] index: usize, #[case] start: Option, - #[case] checked: Option, #[case] expected: usize, ) { - assert_eq!(list_own_prefix_width(ordered, index, start, checked), expected); + assert_eq!(list_own_prefix_width(ordered, index, start), expected); } #[rstest] @@ -6584,24 +6686,6 @@ mod tests { let _ = node.to_string_with(&RenderOptions::default()); } - // A list item whose first child is a code block with arbitrary Unicode - // content must render without panicking, regardless of target indent. - #[test] - fn list_with_code_first_child_never_panics_on_unicode_text( - code_value in prop::collection::vec(any::(), 0..30).prop_map(|cs| cs.into_iter().collect::()), - target_indent in 0usize..10, - ) { - let code = Node::Code(Code { value: code_value, lang: None, fence: true, meta: None, position: None }); - let rendered = code.render_with_theme(&RenderOptions::default(), &ColorTheme::PLAIN); - let _ = reindent_first_leaf_block( - rendered, - Some(&code), - &RenderOptions::default(), - &ColorTheme::PLAIN, - target_indent, - ); - } - #[test] fn escape_label_is_losslessly_invertible( s in prop::collection::vec(any::(), 0..40).prop_map(|cs| cs.into_iter().collect::()) @@ -6630,18 +6714,6 @@ mod tests { prop_assert_eq!(link, Some(url)); } - // Shifting continuation lines out by `delta` and back by the same amount must - // restore the original when the lines start with no indentation of their own, - // since the second shift can never be clamped away. - #[test] - fn reindent_continuation_add_then_remove_round_trips( - body in "[a-zA-Z]{0,5}(\n[a-zA-Z]{0,5}){0,4}", - delta in 0isize..20, - ) { - let shifted = reindent_continuation(&body, delta); - let restored = reindent_continuation(&shifted, -delta); - prop_assert_eq!(restored, body); - } } fn naive_unescape(s: &str) -> String { diff --git a/crates/mq-markdown/src/node/attr_value.rs b/crates/mq-markdown/src/node/attr_value.rs index 8df80b775..be45a10b3 100644 --- a/crates/mq-markdown/src/node/attr_value.rs +++ b/crates/mq-markdown/src/node/attr_value.rs @@ -22,6 +22,8 @@ pub mod attr_keys { pub(crate) const COLUMN: &str = "column"; #[cfg(feature = "callout")] pub(crate) const KIND: &str = "kind"; + #[cfg(feature = "callout")] + pub(crate) const FOLD: &str = "fold"; pub(crate) const ROW: &str = "row"; pub(crate) const LINE: &str = "line"; pub(crate) const END_LINE: &str = "end_line"; diff --git a/crates/mq-markdown/src/parser.rs b/crates/mq-markdown/src/parser.rs index ef43bf1dd..f1aa01353 100644 --- a/crates/mq-markdown/src/parser.rs +++ b/crates/mq-markdown/src/parser.rs @@ -1,32 +1,11 @@ -//! Native Markdown parser that builds [`Node`] values directly, without going through `mdast`. +//! Markdown parser that builds [`Node`] values directly. //! //! Parsing runs in phases: [`block`] resolves the block structure line by line into a tree, //! [`resolve`] collects the definitions and turns the tree into nodes, and [`inline`] parses the raw //! text of paragraphs, headings and table cells along the way. //! -//! The output, positions included, is the one that `markdown-rs`, which this parser replaced, gave for -//! CommonMark, GFM, frontmatter, math and MDX (without a JavaScript parser, so expressions only need -//! balanced braces). HTML is rendered by [`render_html`] in the way `markdown-rs` did. Where the two -//! differ, this parser follows the specifications: -//! -//! - a byte order mark at the start is not content -//! - a table has no body rows that end it for starting with `*`, `#` or a backtick, a table of one -//! column needs no pipe, and its last cell takes the whitespace that follows it -//! - an info string loses its trailing whitespace, and a list is not loose for blank lines after it -//! - a tab that a container only partly consumes is not kept in text, the text starts where the tab ends -//! - `![^a](b)` is an image -//! - the end of text that has a tab before its line ending, which `markdown-rs` places beyond the text -//! -//! What `markdown-rs` did and this parser does not do: -//! -//! - lazy continuation lines after definitions, footnotes or thematic breaks in containers, in containers -//! nested more than two deep, or with several container markers on a line, and unclosed fences in -//! containers whose end depends on the line that follows -//! - documents that start with a `---` or `+++` line that never closes -//! - character references in an image destination that contains an email address -//! - MDX text expressions and tags that span lines interrupted by container markers, and tabs in them -//! - an image in the alt text of an image, and the tabs that a container consumes in part inside of -//! code, in HTML +//! It reads `CommonMark`, GFM, frontmatter, math and MDX (without a JavaScript parser, so expressions +//! only need balanced braces). HTML is rendered by [`render_html`]. mod block; mod code; mod definition; @@ -135,7 +114,7 @@ mod tests { ("no_gfm", "| a |\n|-|\n\n~a~ www.a.b [^a]\n\n- [ ] a\n"), ("no_math", "$a$\n\n$$\na\n$$\n"), ("no_frontmatter", "---\na\n---\n"), - ("esm_is_text", "import a from 'b'\n\nexport const c = 1\n"), + ("esm", "import a from 'b'\n\nexport const c = 1\n"), ("definition", "[a]: /u\n\n[a]\n"), ("list_deep_indent", "- a\n"), ("fence", "```rust\ncode\n```\n"), @@ -1107,6 +1086,10 @@ mod tests { const SPEC_URL: &str = "https://raw.githubusercontent.com/github/cmark-gfm/828322d1ee4facdab56f0d3edccb13e9af90dcd2/test/spec.txt"; + const EXTENSIONS_SPEC_URL: &str = "https://raw.githubusercontent.com/github/cmark-gfm/828322d1ee4facdab56f0d3edccb13e9af90dcd2/test/extensions.txt"; + + const COMMONMARK_SPEC_URL: &str = "https://raw.githubusercontent.com/commonmark/commonmark-spec/0.31.2/spec.txt"; + /// The markdown and the HTML of each example of the `CommonMark` and GFM `spec.txt`, with its number. fn spec_examples(text: &str) -> Vec<(usize, String, String)> { let fence = "`".repeat(32); @@ -1129,44 +1112,107 @@ mod tests { examples } - /// Examples of the spec that are not rendered as in the spec, by number. The parser has kept the - /// behavior of `markdown-rs` here: + /// Examples of the GFM spec, which is based on `CommonMark` 0.29, that are not rendered as in the + /// spec, by number: /// - /// - code with a tab that a container consumes in part, which keeps the rest of the tab as spaces in - /// the spec, and an image in the alt text of an image (5, 6, 7, 570) /// - a document that starts with `---` has frontmatter (66, 68) - /// - emphasis next to other runs of `*` and `_` follows the algorithm of `markdown-rs` (388, 416, - /// 424, 425, 426, 463, 464, 465, 467) - /// - links with a protocol other than http, https, irc, ircs, mailto and xmpp have no `href` (496, - /// 594, 595, 597) + /// - nested `strong` of runs of `*` and `_`, which `CommonMark` 0.30 and later render as nested + /// elements where 0.29 merged them (388, 416, 424, 425, 426, 463, 464, 465, 467) + /// - links with a protocol other than http, https, irc, ircs, mailto and xmpp have no `href`, as + /// `markdown-rs` does it by default (496, 594, 595, 597) /// - GFM autolink literals, which the spec does not have (598, 604, 607, 608) + /// - the GFM tag filter, which writes the tags `script`, `style` and `textarea` as text (140, 141, + /// 142, 145, 147) const KNOWN_DIFFERENCES: &[usize] = &[ - 5, 6, 7, 66, 68, 388, 416, 424, 425, 426, 463, 464, 465, 467, 496, 570, 594, 595, 597, 598, 604, 607, 608, + 66, 68, 140, 141, 142, 145, 147, 388, 416, 424, 425, 426, 463, 464, 465, 467, 496, 594, 595, 597, 598, 604, + 607, 608, ]; + /// Examples of the `CommonMark` 0.31.2 spec that are not rendered as in the spec, by number: + /// + /// - a document that starts with `---` has frontmatter (96, 98) + /// - links with a protocol other than http, https, irc, ircs, mailto and xmpp have no `href` (500, + /// 598, 599, 601) + /// - GFM autolink literals (602, 608, 611, 612) + /// - the GFM tag filter, which writes the tags `script`, `style` and `textarea` as text (170, 171, 172, + /// 173, 176, 178) + const COMMONMARK_KNOWN_DIFFERENCES: &[usize] = &[ + 96, 98, 170, 171, 172, 173, 176, 178, 500, 598, 599, 601, 602, 608, 611, 612, + ]; + + /// Examples of the GFM extensions spec that are not rendered as in the spec, by number: + /// + /// - input that must only not crash, whose output the spec leaves out (20) + /// - the HTML of footnotes, which is the one of the current micromark and GitHub (23, 24, 25) + const EXTENSIONS_KNOWN_DIFFERENCES: &[usize] = &[20, 23, 24, 25]; + + /// Fetches `url`, or reads the file in the environment variable `var`. + fn spec_text(var: &str, url: &str) -> String { + match std::env::var(var) { + Ok(path) => std::fs::read_to_string(path).unwrap(), + Err(_) => ureq::get(url).call().unwrap().into_body().read_to_string().unwrap(), + } + } + + /// The HTML in the form of the spec, which gives the language of code as a class and not as `lang`. + fn spec_html(html: &str) -> String { + let mut result = String::with_capacity(html.len()); + let mut rest = html; + while let Some(start) = rest.find("
    ").map(|end| value_start + end);
    +            let Some(end) = end else { break };
    +            result.push_str(before);
    +            result.push_str(&format!("
    ", &tag[value_start..end]));
    +            rest = &tag[end + "\">".len()..];
    +        }
    +        result.push_str(rest);
    +        result
    +    }
    +
    +    /// The numbers of the examples of `text` that are not rendered as in the spec.
    +    fn differing_examples(text: &str, minimum: usize) -> Vec {
    +        let examples = spec_examples(text);
    +        assert!(examples.len() > minimum, "only {} examples were found", examples.len());
    +        examples
    +            .iter()
    +            .filter(|(_, markdown, html)| spec_html(&to_html(markdown)) != *html)
    +            .map(|(number, ..)| *number)
    +            .collect()
    +    }
    +
         /// Renders every example of the spec to HTML, from `SPEC_FILE` or fetched over the network.
         #[test]
         #[ignore = "fetches spec.txt over the network; set SPEC_FILE to use a local copy"]
         fn spec_examples_render_as_in_the_spec() {
    -        let text = match std::env::var("SPEC_FILE") {
    -            Ok(path) => std::fs::read_to_string(path).unwrap(),
    -            Err(_) => ureq::get(SPEC_URL)
    -                .call()
    -                .unwrap()
    -                .into_body()
    -                .read_to_string()
    -                .unwrap(),
    -        };
    -        let examples = spec_examples(&text);
    -        assert!(examples.len() > 600, "only {} examples were found", examples.len());
    +        assert_eq!(
    +            differing_examples(&spec_text("SPEC_FILE", SPEC_URL), 600),
    +            KNOWN_DIFFERENCES,
    +            "examples that are not rendered as in the spec"
    +        );
    +    }
     
    -        let differing = examples
    -            .iter()
    -            .filter(|(_, markdown, html)| to_html(markdown) != *html)
    -            .map(|(number, ..)| *number)
    -            .collect::>();
    +    /// Renders every example of the `CommonMark` 0.31.2 spec to HTML, from `COMMONMARK_SPEC_FILE` or
    +    /// fetched over the network.
    +    #[test]
    +    #[ignore = "fetches spec.txt over the network; set COMMONMARK_SPEC_FILE to use a local copy"]
    +    fn commonmark_spec_examples_render_as_in_the_spec() {
    +        assert_eq!(
    +            differing_examples(&spec_text("COMMONMARK_SPEC_FILE", COMMONMARK_SPEC_URL), 600),
    +            COMMONMARK_KNOWN_DIFFERENCES,
    +            "examples that are not rendered as in the spec"
    +        );
    +    }
    +
    +    /// Renders every example of the GFM extensions spec to HTML, from `EXTENSIONS_SPEC_FILE` or fetched
    +    /// over the network.
    +    #[test]
    +    #[ignore = "fetches extensions.txt over the network; set EXTENSIONS_SPEC_FILE to use a local copy"]
    +    fn extensions_spec_examples_render_as_in_the_spec() {
             assert_eq!(
    -            differing, KNOWN_DIFFERENCES,
    +            differing_examples(&spec_text("EXTENSIONS_SPEC_FILE", EXTENSIONS_SPEC_URL), 20),
    +            EXTENSIONS_KNOWN_DIFFERENCES,
                 "examples that are not rendered as in the spec"
             );
         }
    diff --git a/crates/mq-markdown/src/parser/block.rs b/crates/mq-markdown/src/parser/block.rs
    index a17d52f84..9d80ef14c 100644
    --- a/crates/mq-markdown/src/parser/block.rs
    +++ b/crates/mq-markdown/src/parser/block.rs
    @@ -14,7 +14,7 @@ use super::mdx_flow::{
     };
     use super::table;
     use super::tree::{Block, FootnoteBlock, InlineBlock, InlineKind, InlineSource, Item, ListBlock, QuoteBlock};
    -use crate::node::{HorizontalRule, Html, Node, Point, Position, Toml, Yaml};
    +use crate::node::{HorizontalRule, Html, MdxJsEsm, Node, Point, Position, Toml, Yaml};
     
     /// Lines indented by this many columns or more are code, not other blocks.
     pub(super) const CODE_INDENT: usize = 4;
    @@ -22,27 +22,6 @@ pub(super) const CODE_INDENT: usize = 4;
     /// Containers nested deeper than this are parsed as plain paragraph text, which bounds recursion.
     const MAX_DEPTH: usize = 128;
     
    -/// Which list markers end a paragraph on its second line although they could not interrupt it.
    -#[derive(Clone, Copy, PartialEq)]
    -enum Release {
    -    None,
    -    /// Markers with content.
    -    Markers,
    -    /// Every marker.
    -    AllMarkers,
    -}
    -
    -/// How far a paragraph of dashes right after a setext heading extends, in markdown-rs.
    -#[derive(Clone, Copy, PartialEq)]
    -enum Dashes {
    -    /// Not such a paragraph.
    -    No,
    -    /// Up to the next indented code line, at the top level.
    -    UntilCode,
    -    /// A single line, inside containers.
    -    OneLine,
    -}
    -
     /// What the first line of a container cannot start because the container interrupted a paragraph.
     #[derive(Clone, Copy, Default)]
     struct Interrupt {
    @@ -105,15 +84,12 @@ fn parse_blocks(lines: &[Line<'_>], depth: usize, interrupting: Interrupt) -> Ve
         let mut blocks = Vec::new();
         let mut index = 0;
         let mut after_paragraph = interrupting;
    -    let mut after_setext = false;
         // Set after a container, and whether a blank line or an empty list item preceded the next block.
         let mut after_container = false;
    -    let mut after_list = false;
         let mut blank_between = false;
     
         while index < lines.len() {
             let interrupting = std::mem::take(&mut after_paragraph);
    -        let follows_setext = std::mem::take(&mut after_setext);
             let line = &lines[index];
             if line.is_blank() {
                 blank_between = true;
    @@ -121,20 +97,18 @@ fn parse_blocks(lines: &[Line<'_>], depth: usize, interrupting: Interrupt) -> Ve
                 continue;
             }
             let closed_container = std::mem::take(&mut after_container);
    -        let closed_list = std::mem::take(&mut after_list);
             let separated = std::mem::take(&mut blank_between);
     
             let (columns, indent) = line.indent();
             let rest = &line.text[indent..];
     
             if columns >= line.code_indent() && !interrupting.code {
    -            index = indented_code(lines, index, closed_list || closed_container && !separated, &mut blocks);
    +            index = indented_code(lines, index, &mut blocks);
             } else if columns >= line.code_indent() {
                 // Indented code cannot interrupt a paragraph.
    -            let (next, interrupt, setext) = paragraph(lines, index, Release::None, Dashes::No, &mut blocks);
    +            let (next, interrupt) = paragraph(lines, index, &mut blocks);
                 index = next;
                 after_paragraph = interrupt;
    -            after_setext = setext;
             } else if let Some(fence) = Fence::open(rest, !line.mdx) {
                 // The end of an unclosed fence without content is quirky right after a container.
                 let own_end = closed_container
    @@ -144,8 +118,9 @@ fn parse_blocks(lines: &[Line<'_>], depth: usize, interrupting: Interrupt) -> Ve
             } else if let Some(depth) = atx_depth(rest) {
                 blocks.push(atx_heading(line, indent, depth));
                 index += 1;
    -        } else if is_thematic_break(rest) && !(follows_setext && is_dash_run(rest)) {
    +        } else if is_thematic_break(rest) {
                 blocks.push(Block::Node(Node::HorizontalRule(HorizontalRule {
    +                marker: rest.chars().next(),
                     position: Some(Position {
                         start: line.point(0),
                         end: line.end(),
    @@ -160,7 +135,6 @@ fn parse_blocks(lines: &[Line<'_>], depth: usize, interrupting: Interrupt) -> Ve
             {
                 index = list(lines, index, &marker, depth, interrupting, &mut blocks);
                 after_container = true;
    -            after_list = true;
             } else if let Some(kind) = html_start(line, rest) {
                 index = html_block(lines, index, kind, &mut blocks);
             } else if let Some(marker) = footnote_marker(line).filter(|_| containers) {
    @@ -169,6 +143,28 @@ fn parse_blocks(lines: &[Line<'_>], depth: usize, interrupting: Interrupt) -> Ve
             } else if let Some((items, next)) = table::parse(lines, index, interrupts_paragraph, ends_table) {
                 blocks.push(Block::Table(items));
                 index = next;
    +        } else if line.mdx
    +            && depth == 0
    +            && indent == 0
    +            && (line.text.starts_with("import ") || line.text.starts_with("export "))
    +        {
    +            // ESM goes on to the next blank line. Without a JavaScript parser it is taken as it is.
    +            let end = lines[index..]
    +                .iter()
    +                .position(Line::is_blank)
    +                .map_or(lines.len(), |offset| index + offset);
    +            let parts = lines[index..end]
    +                .iter()
    +                .map(|line| (line.text, line.eol))
    +                .collect::>();
    +            blocks.push(Block::Node(Node::MdxJsEsm(MdxJsEsm {
    +                value: join_lines(&parts).into(),
    +                position: Some(Position {
    +                    start: lines[index].point(0),
    +                    end: lines[end - 1].end(),
    +                }),
    +            })));
    +            index = end;
             } else if line.mdx
                 && matches!(rest.as_bytes().first(), Some(b'<' | b'{'))
                 && let FlowOutcome::Flow(next) = mdx_flow(lines, index, &mut blocks)
    @@ -177,25 +173,9 @@ fn parse_blocks(lines: &[Line<'_>], depth: usize, interrupting: Interrupt) -> Ve
                 // Flow content that interrupted a paragraph keeps the restrictions that came with it.
                 after_paragraph = interrupting;
             } else {
    -            let release = match (closed_container, separated) {
    -                (false, _) => Release::None,
    -                (true, false) => Release::AllMarkers,
    -                (true, true) => Release::Markers,
    -            };
    -            let (next, interrupt, setext) = paragraph(
    -                lines,
    -                index,
    -                release,
    -                match (follows_setext && is_dash_run(rest), depth) {
    -                    (false, _) => Dashes::No,
    -                    (true, 0) => Dashes::UntilCode,
    -                    (true, _) => Dashes::OneLine,
    -                },
    -                &mut blocks,
    -            );
    +            let (next, interrupt) = paragraph(lines, index, &mut blocks);
                 index = next;
                 after_paragraph = interrupt;
    -            after_setext = setext;
             }
         }
     
    @@ -216,8 +196,6 @@ struct LeafState<'a> {
         paragraph: bool,
         /// Number of container markers on the line that opened the paragraph.
         paragraph_containers: usize,
    -    /// Whether the last line was a setext underline. markdown-rs still lets unindented text continue.
    -    after_setext: bool,
         /// Restrictions on the next fed line, the first line of a container that interrupted a paragraph.
         restricted: Interrupt,
     }
    @@ -231,20 +209,18 @@ impl<'a> LeafState<'a> {
                 header: None,
                 paragraph: false,
                 paragraph_containers: 0,
    -            after_setext: false,
                 restricted,
             }
         }
     
         /// Whether `line` without its container prefix continues the paragraph of the collected lines.
         fn continues_with(&self, line: &Line<'_>) -> bool {
    -        (self.paragraph || (self.after_setext && line.indent().0 < line.code_indent())) && is_lazy_continuation(line)
    +        self.paragraph && is_lazy_continuation(line)
         }
     
         fn feed(&mut self, line: &Line<'a>) {
             let mut line = *line;
             let restricted = std::mem::take(&mut self.restricted);
    -        let follows_setext = std::mem::take(&mut self.after_setext);
             let mut containers = 0;
             // Whether a paragraph was open when the line started, for markers deeper on the same line.
             let open = self.paragraph;
    @@ -310,13 +286,6 @@ impl<'a> LeafState<'a> {
                 }
                 if self.paragraph && containers == self.paragraph_containers && setext_depth(&line).is_some() {
                     self.paragraph = false;
    -                self.after_setext = true;
    -                return;
    -            }
    -            // Dashes right after a setext heading start a paragraph, not a thematic break.
    -            if follows_setext && is_dash_run(rest) {
    -                self.paragraph = true;
    -                self.paragraph_containers = containers;
                     return;
                 }
                 if atx_depth(rest).is_some() || is_thematic_break(rest) {
    @@ -548,7 +517,7 @@ fn list(
             let next = (end..lines.len()).find(|&i| !lines[i].is_blank());
             match next.filter(|&i| ListMarker::parse(&lines[i]).is_some_and(|m| m.same_list(first))) {
                 Some(next) => {
    -                spread |= next > end || items.last().is_some_and(ends_in_empty_item);
    +                spread |= next > end;
                     index = next;
                 }
                 None => {
    @@ -561,6 +530,7 @@ fn list(
         blocks.push(Block::List(ListBlock {
             ordered: first.ordered,
             start: first.ordered.then_some(first.start),
    +        marker: first.delimiter as char,
             spread,
             items,
         }));
    @@ -639,18 +609,6 @@ fn extend_quote(quote: &mut QuoteBlock, end: &Point) {
         }
     }
     
    -/// Whether the item ends with a nested list whose last item is empty. markdown-rs counts that
    -/// like a blank line between the item and the next one.
    -fn ends_in_empty_item(item: &Item) -> bool {
    -    match item.children.last() {
    -        Some(Block::List(list)) => list
    -            .items
    -            .last()
    -            .is_some_and(|last| last.children.is_empty() || ends_in_empty_item(last)),
    -        _ => false,
    -    }
    -}
    -
     /// Collects one item starting at `start`, returning it and the index of the first line after it.
     fn list_item(
         lines: &[Line<'_>],
    @@ -659,8 +617,7 @@ fn list_item(
         depth: usize,
         interrupting: Interrupt,
     ) -> (Item, usize) {
    -    let mut first = marker.content(lines[start]);
    -    first.own_chunk = first.indent().0 >= first.code_indent();
    +    let first = marker.content(lines[start]);
         let mut inner = vec![first];
         let mut state = LeafState::new(interrupting);
         state.feed(&first);
    @@ -697,16 +654,7 @@ fn list_item(
             index += 1;
         }
     
    -    // The checkbox is on the first content line, which is the next one when the marker line is empty.
    -    let target = usize::from(first.is_blank());
    -    let checked = inner
    -        .get(target)
    -        .filter(|_| !ends_with_setext_underline(&inner[target + 1..]))
    -        .and_then(task_checkbox)
    -        .map(|(checked, bytes)| {
    -            inner[target] = inner[target].skip(bytes);
    -            checked
    -        });
    +    let checkbox = task_checkbox(&first);
     
         // Trailing blank lines belong to whatever follows the item, but still extend its position.
         let reaches_end = index == lines.len();
    @@ -729,6 +677,8 @@ fn list_item(
                 .map_or_else(|| lines[start].end(), Line::end),
         };
         let mut children = parse_blocks(&inner, depth + 1, interrupting);
    +    // The checkbox is the start of the text of the first paragraph.
    +    let checked = checkbox.filter(|_| remove_task_marker(&mut children));
         if let Some(blank) = blanks.last() {
             extend_last_items(
                 &mut children,
    @@ -745,32 +695,37 @@ fn list_item(
         (item, index)
     }
     
    -/// Whether the paragraph that continues on `lines` is turned into a setext heading.
    -fn ends_with_setext_underline(lines: &[Line<'_>]) -> bool {
    -    for line in lines {
    -        if setext_depth(line).is_some() {
    -            return true;
    -        }
    -        if interrupts_paragraph(line) {
    -            return false;
    -        }
    -    }
    -    false
    -}
    -
     /// Recognises a GFM task marker (`[ ] ` or `[x] `) at the start of an item's content, returning
     /// the checked state and the bytes to skip.
    -fn task_checkbox(line: &Line<'_>) -> Option<(bool, usize)> {
    +fn task_checkbox(line: &Line<'_>) -> Option {
         if line.mdx {
             return None;
         }
    -    let bytes = line.text.as_bytes();
    -    let checked = match bytes.get(..4)? {
    -        [b'[', b' ', b']', b' ' | b'\t'] => false,
    -        [b'[', b'x' | b'X', b']', b' ' | b'\t'] => true,
    -        _ => return None,
    +    match line.text.as_bytes().get(..4)? {
    +        [b'[', b' ', b']', b' ' | b'\t'] => Some(false),
    +        [b'[', b'x' | b'X', b']', b' ' | b'\t'] => Some(true),
    +        _ => None,
    +    }
    +}
    +
    +/// Removes the checkbox from the text of the first paragraph of an item, and the paragraph when nothing
    +/// else is in it. Returns whether the item starts with one.
    +fn remove_task_marker(children: &mut Vec) -> bool {
    +    let Some(Block::Inline(InlineBlock {
    +        source,
    +        kind: InlineKind::Paragraph,
    +    })) = children.first_mut()
    +    else {
    +        return false;
         };
    -    (!line.text[4..].trim_matches([' ', '\t']).is_empty()).then_some((checked, 4))
    +    if !matches!(source.text.as_bytes().get(..3), Some([b'[', b' ' | b'x' | b'X', b']'])) {
    +        return false;
    +    }
    +    source.remove_prefix(3);
    +    if source.text.is_empty() {
    +        children.remove(0);
    +    }
    +    true
     }
     
     /// Returns the heading depth when `rest` (indent already removed) is an ATX heading line.
    @@ -792,14 +747,6 @@ fn atx_heading(line: &Line<'_>, indent: usize, depth: u8) -> Block {
                 content
             }
         };
    -    // markdown-rs drops further `#` sequences (and the whitespace between them) at the start of the text.
    -    let content = {
    -        let mut content = content;
    -        while content.starts_with('#') {
    -            content = content.trim_start_matches('#').trim_start_matches([' ', '\t']);
    -        }
    -        content
    -    };
         let start = line.point(offset_in(line.text, content));
     
         Block::Inline(InlineBlock {
    @@ -831,13 +778,6 @@ fn offset_in(parent: &str, child: &str) -> usize {
         child.as_ptr() as usize - parent.as_ptr() as usize
     }
     
    -/// Whether `rest` is only dashes, like `---`, which markdown-rs does not read as a thematic break
    -/// right after a setext heading.
    -fn is_dash_run(rest: &str) -> bool {
    -    let content = rest.trim_end_matches([' ', '\t']);
    -    content.len() >= 3 && content.bytes().all(|b| b == b'-')
    -}
    -
     fn is_thematic_break(rest: &str) -> bool {
         let Some(marker) = rest.chars().next().filter(|c| matches!(c, '*' | '-' | '_')) else {
             return false;
    @@ -871,28 +811,7 @@ fn setext_depth(line: &Line<'_>) -> Option {
     /// Whether a line without its container prefix can continue an open paragraph of that container.
     /// Any list marker ends the paragraph here, even one that could not interrupt it elsewhere.
     fn is_lazy_continuation(line: &Line<'_>) -> bool {
    -    !interrupts_paragraph(line) && ListMarker::parse(line).is_none() && !starts_construct(line)
    -}
    -
    -/// Whether the line is only the start of a construct that a next line could still complete: one or two
    -/// underscores (a thematic break), one or two backticks or tildes (a fence), a dollar, or an angle
    -/// bracket. markdown-rs does not continue paragraphs lazily with such a line.
    -fn starts_construct(line: &Line<'_>) -> bool {
    -    // Only the end of the document leaves such a start undecided.
    -    if !line.eof || !line.eol.is_empty() {
    -        return false;
    -    }
    -    let text = line.text.trim_matches([' ', '\t']);
    -    match text.as_bytes() {
    -        [b'<'] => !line.mdx,
    -        [b'$'] => true,
    -        [first @ (b'`' | b'~'), rest @ ..] => rest.len() < 2 && rest.iter().all(|b| b == first),
    -        _ => {
    -            // Underscores with spaces between, like `_ _`.
    -            let markers = text.bytes().filter(|&b| b == b'_').count();
    -            (1..3).contains(&markers) && text.bytes().all(|b| matches!(b, b'_' | b' ' | b'\t'))
    -        }
    -    }
    +    !interrupts_paragraph(line) && ListMarker::parse(line).is_none()
     }
     
     /// Whether `line` ends the paragraph that is being collected.
    @@ -925,14 +844,7 @@ fn ends_table(line: &Line<'_>) -> bool {
     
     /// Returns the index after the block and what it restricts on the first line of a following container.
     ///
    -/// A paragraph of `dashes` right after a setext heading is cut short as described by [`Dashes`]. A paragraph with a `release` has its second line ended by list markers that could not interrupt it.
    -fn paragraph(
    -    lines: &[Line<'_>],
    -    start: usize,
    -    release: Release,
    -    dashes: Dashes,
    -    blocks: &mut Vec,
    -) -> (usize, Interrupt, bool) {
    +fn paragraph(lines: &[Line<'_>], start: usize, blocks: &mut Vec) -> (usize, Interrupt) {
         let mut index = start + 1;
         // Lines up to this one are taken by flow content that turned out to be text.
         let mut absorbed = match probe_mdx_flow(lines, start) {
    @@ -940,8 +852,6 @@ fn paragraph(
             _ => start,
         };
         let mut setext = None;
    -    // Whether a marker that could not interrupt the paragraph ended it anyway.
    -    let mut released = false;
     
         while let Some(line) = lines.get(index) {
             if table::starts_at(lines, index, interrupts_paragraph) {
    @@ -958,24 +868,11 @@ fn paragraph(
                 index += 1;
                 continue;
             }
    -        let releases = index == start + 1
    -            && ListMarker::parse(line).is_some_and(|marker| match release {
    -                Release::None => false,
    -                Release::Markers => !marker.empty,
    -                Release::AllMarkers => true,
    -            });
    -        if releases {
    -            released = true;
    -            break;
    -        }
             if let Some(depth) = setext_depth(line) {
                 setext = Some((depth, index));
                 break;
             }
    -        if dashes == Dashes::OneLine
    -            || interrupts_paragraph(line)
    -            || (dashes == Dashes::UntilCode && line.indent().0 >= line.code_indent())
    -        {
    +        if interrupts_paragraph(line) {
                 break;
             }
             index += 1;
    @@ -1003,9 +900,8 @@ fn paragraph(
                         code: false,
                         list: true,
                     },
    -                true,
                 ),
    -            None => (index, Interrupt::default(), false),
    +            None => (index, Interrupt::default()),
             };
         }
         let source = paragraph_source(&lines[first..index]);
    @@ -1022,7 +918,7 @@ fn paragraph(
                         },
                     },
                 }));
    -            (underline + 1, Interrupt::default(), true)
    +            (underline + 1, Interrupt::default())
             }
             None => {
                 blocks.push(Block::Inline(InlineBlock {
    @@ -1030,14 +926,7 @@ fn paragraph(
                     kind: InlineKind::Paragraph,
                 }));
                 // A one-line paragraph right after a container does not restrict what follows it.
    -            (
    -                index,
    -                Interrupt {
    -                    code: dashes == Dashes::No,
    -                    list: !(dashes != Dashes::No || released || release == Release::AllMarkers && index == start + 1),
    -                },
    -                false,
    -            )
    +            (index, Interrupt::default())
             }
         }
     }
    @@ -1127,15 +1016,9 @@ fn footnote(
         state.feed(&first);
         let mut blanks = Vec::new();
         let mut index = start + 1;
    -    // The end of a definition without content: it reaches to the end of the blank lines that follow.
    -    let mut empty_end = None;
     
         while let Some(line) = lines.get(index) {
             if line.is_blank() {
    -            if first.is_blank() && inner.len() == 1 {
    -                empty_end = lines[index..].iter().take_while(|l| l.is_blank()).last().map(Line::end);
    -                break;
    -            }
                 blanks.push(line.skip(line.text.len()));
             } else if line.indent().0 >= CODE_INDENT {
                 for blank in &blanks {
    @@ -1157,12 +1040,10 @@ fn footnote(
     
         // Trailing blank lines belong to whatever follows, but still extend the position.
         index -= blanks.len();
    -    let end = empty_end.unwrap_or_else(|| {
    -        blanks
    -            .last()
    -            .or(inner.last())
    -            .map_or_else(|| lines[start].end(), Line::end)
    -    });
    +    let end = blanks
    +        .last()
    +        .or(inner.last())
    +        .map_or_else(|| lines[start].end(), Line::end);
         let position = Position {
             start: lines[start].point(0),
             end,
    diff --git a/crates/mq-markdown/src/parser/code.rs b/crates/mq-markdown/src/parser/code.rs
    index edf07fcf7..8103cf01a 100644
    --- a/crates/mq-markdown/src/parser/code.rs
    +++ b/crates/mq-markdown/src/parser/code.rs
    @@ -114,13 +114,11 @@ fn reaches_line_end(last: &Line<'_>, empty: bool, own_end: bool) -> bool {
         }
     }
     
    -pub(super) fn indented_code(lines: &[Line<'_>], start: usize, after_container: bool, blocks: &mut Vec) -> usize {
    -    // markdown-rs makes the first line a chunk of its own in these two cases.
    -    let own_chunk = lines[start].own_chunk || after_container;
    +pub(super) fn indented_code(lines: &[Line<'_>], start: usize, blocks: &mut Vec) -> usize {
         let mut last_code = start;
         let mut index = start;
     
    -    while let Some(line) = lines.get(index).filter(|_| !(own_chunk && index > start)) {
    +    while let Some(line) = lines.get(index) {
             // Whitespace-only lines are code when indented enough, otherwise they may just separate chunks.
             if line.indent().0 >= CODE_INDENT {
                 last_code = index;
    @@ -132,13 +130,24 @@ pub(super) fn indented_code(lines: &[Line<'_>], start: usize, after_container: b
     
         let mut parts = lines[start..=last_code]
             .iter()
    -        .map(|line| (line.skip_columns(CODE_INDENT).text, line.eol))
    +        .map(|line| {
    +            let content = line.skip_columns(CODE_INDENT);
    +            (content.pad, content.text, line.eol)
    +        })
             .collect::>();
         // Lines that are empty once the indent is removed only extend the position, not the value.
    -    while parts.len() > 1 && parts.last().is_some_and(|(text, _)| text.is_empty()) {
    +    while parts.len() > 1 && parts.last().is_some_and(|(pad, text, _)| *pad == 0 && text.is_empty()) {
             parts.pop();
         }
    -    let value = join_lines(&parts);
    +    // What is left of a tab that the indent consumes in part is code, as spaces.
    +    let mut value = String::with_capacity(parts.iter().map(|(pad, text, eol)| pad + text.len() + eol.len()).sum());
    +    for (index, (pad, text, eol)) in parts.iter().enumerate() {
    +        value.extend(std::iter::repeat_n(' ', *pad));
    +        value.push_str(text);
    +        if index + 1 < parts.len() {
    +            value.push_str(eol);
    +        }
    +    }
     
         let position = Position {
             start: lines[start].point(0),
    diff --git a/crates/mq-markdown/src/parser/inline.rs b/crates/mq-markdown/src/parser/inline.rs
    index 77692e166..ff9f3ca7b 100644
    --- a/crates/mq-markdown/src/parser/inline.rs
    +++ b/crates/mq-markdown/src/parser/inline.rs
    @@ -40,6 +40,8 @@ pub(super) struct Delim {
         /// Offset of the first remaining character.
         pub(super) start: usize,
         pub(super) count: usize,
    +    /// Length of the whole run, which `count` shrinks from as the run is used.
    +    pub(super) original: usize,
         pub(super) can_open: bool,
         pub(super) can_close: bool,
     }
    @@ -280,6 +282,18 @@ struct Scanner<'a> {
         run: usize,
     }
     
    +/// Bytes where `Scanner::scan` may start a construct. Every other byte is plain text.
    +static SPECIAL: [bool; 256] = {
    +    let mut table = [false; 256];
    +    let special = b"\\&`$*_~[!]<{\n\rhHwW";
    +    let mut index = 0;
    +    while index < special.len() {
    +        table[special[index] as usize] = true;
    +        index += 1;
    +    }
    +    table
    +};
    +
     impl Scanner<'_> {
         fn src(&self) -> &str {
             self.context.src()
    @@ -289,7 +303,25 @@ impl Scanner<'_> {
             let len = self.src().len();
             while self.pos < len {
                 let bytes = self.src().as_bytes();
    -            match bytes[self.pos] {
    +            let skip = bytes[self.pos..].iter().take_while(|&&b| !SPECIAL[b as usize]).count();
    +            let pos = self.pos + skip;
    +            if pos == len {
    +                self.pos = len;
    +                break;
    +            }
    +            let (byte, image) = (bytes[pos], bytes.get(pos + 1) == Some(&b'['));
    +            self.pos = pos;
    +            let link = if matches!(byte, b'[' | b'!') {
    +                self.obsidian_link()
    +            } else {
    +                None
    +            };
    +            match byte {
    +                _ if link.is_some() => {
    +                    if let Some((node, end)) = link {
    +                        self.push_node(node, end);
    +                    }
    +                }
                     b'\\' => self.escape(),
                     b'&' => self.entity(),
                     b'`' => self.span(b'`'),
    @@ -297,7 +329,7 @@ impl Scanner<'_> {
                     b'*' | b'_' => self.delimiter(),
                     b'~' if !self.context.references.mdx => self.delimiter(),
                     b'[' => self.open(false),
    -                b'!' if bytes.get(self.pos + 1) == Some(&b'[') => self.open(true),
    +                b'!' if image => self.open(true),
                     b']' => self.close(),
                     b'<' if !self.context.references.mdx => self.angle(),
                     b'<' => self.jsx(),
    @@ -309,6 +341,67 @@ impl Scanner<'_> {
             }
         }
     
    +    /// The wikilink `[[target|text]]` or the embed `![[target|display]]` at the current position, and the
    +    /// offset where it ends. The target is not empty, and neither part holds a bracket or a line ending.
    +    #[cfg(any(feature = "wikilink", feature = "embed"))]
    +    fn obsidian_link(&self) -> Option<(Node, usize)> {
    +        if self.context.references.mdx {
    +            return None;
    +        }
    +        let src = self.src();
    +        let start = self.pos;
    +        let rest = &src[start..];
    +        let (embed, open) = match rest.as_bytes() {
    +            [b'!', b'[', b'[', ..] if cfg!(feature = "embed") => (true, start + 3),
    +            [b'[', b'[', ..] if cfg!(feature = "wikilink") => (false, start + 2),
    +            _ => return None,
    +        };
    +        let close = open + src[open..].find("]]")?;
    +        let content = &src[open..close];
    +        if content.contains(['[', ']', '\n', '\r']) {
    +            return None;
    +        }
    +        let (target, label) = match content.split_once('|') {
    +            Some((target, label)) => (target.trim(), Some(label.trim().to_string())),
    +            None => (content.trim(), None),
    +        };
    +        if target.is_empty() {
    +            return None;
    +        }
    +        let end = close + 2;
    +        let position = Some(self.context.position(start, end));
    +        let target = target.to_string();
    +        let node = if embed {
    +            #[cfg(feature = "embed")]
    +            {
    +                Node::Embed(crate::node::Embed {
    +                    target,
    +                    display: label,
    +                    position,
    +                })
    +            }
    +            #[cfg(not(feature = "embed"))]
    +            unreachable!("embeds are off")
    +        } else {
    +            #[cfg(feature = "wikilink")]
    +            {
    +                Node::WikiLink(crate::node::WikiLink {
    +                    target,
    +                    text: label,
    +                    position,
    +                })
    +            }
    +            #[cfg(not(feature = "wikilink"))]
    +            unreachable!("wikilinks are off")
    +        };
    +        Some((node, end))
    +    }
    +
    +    #[cfg(not(any(feature = "wikilink", feature = "embed")))]
    +    fn obsidian_link(&self) -> Option<(Node, usize)> {
    +        None
    +    }
    +
         /// Pushes the pending plain text up to the current position as an item.
         fn flush(&mut self) {
             if self.run < self.pos {
    @@ -443,6 +536,13 @@ impl Scanner<'_> {
             let end = close + size;
             // In a table cell, `\|` stands for `|` inside code as well.
             let content = &src[start + size..close];
    +        // The lines of a paragraph lose their leading whitespace before the inline content is read.
    +        let content = if self.context.references.mdx {
    +            Cow::Borrowed(content)
    +        } else {
    +            remove_line_indent(content)
    +        };
    +        let content = content.as_ref();
             let value = if self.context.source.table && content.contains("\\|") {
                 span_value(&content.replace("\\|", "|"))
             } else {
    @@ -473,7 +573,7 @@ impl Scanner<'_> {
     
             let before = src[..start].chars().next_back();
             let after = src[end..].chars().next();
    -        let (can_open, can_close) = emphasis::flanking(ch, before, after, !self.context.references.mdx);
    +        let (can_open, can_close) = emphasis::flanking(ch, before, after);
             if !can_open && !can_close {
                 self.pos = end;
                 return;
    @@ -483,6 +583,7 @@ impl Scanner<'_> {
                 ch,
                 start,
                 count: size,
    +            original: size,
                 can_open,
                 can_close,
             };
    diff --git a/crates/mq-markdown/src/parser/inline/emphasis.rs b/crates/mq-markdown/src/parser/inline/emphasis.rs
    index eee94e5d9..69dad388a 100644
    --- a/crates/mq-markdown/src/parser/inline/emphasis.rs
    +++ b/crates/mq-markdown/src/parser/inline/emphasis.rs
    @@ -1,8 +1,7 @@
     //! Emphasis, strong emphasis and strikethrough, resolved with a stack of delimiter runs.
     //!
    -//! This follows the attention algorithm of `markdown-rs`, which differs from the reference
    -//! implementation of `CommonMark` in how runs next to other runs open and close, and in using the
    -//! remaining length of a run for the rule of three.
    +//! This follows the reference implementation of `CommonMark`. `markdown-rs` differs in how runs next
    +//! to other runs open and close, and in using the remaining length of a run for the rule of three.
     
     use super::punctuation::is_punctuation;
     use super::{Context, Delim, Item, to_nodes};
    @@ -24,18 +23,13 @@ fn kind(char: Option) -> Kind {
         }
     }
     
    -/// Whether a run of `ch` between `before` and `after` can open and can close. `~` is a delimiter too
    -/// when `gfm` is set.
    -pub(super) fn flanking(ch: u8, before: Option, after: Option, gfm: bool) -> (bool, bool) {
    +/// Whether a run of `ch` between `before` and `after` can open and can close, by the rules of
    +/// `CommonMark` for left- and right-flanking runs, which `~` follows too.
    +pub(super) fn flanking(ch: u8, before: Option, after: Option) -> (bool, bool) {
         let (before_kind, after_kind) = (kind(before), kind(after));
    -    let marker = |char: Option| matches!(char, Some('*' | '_')) || (gfm && char == Some('~'));
     
    -    let open = after_kind == Kind::Other
    -        || (after_kind == Kind::Punctuation && before_kind != Kind::Other)
    -        || (ch != b'~' && marker(after));
    -    let close = before_kind == Kind::Other
    -        || (before_kind == Kind::Punctuation && after_kind != Kind::Other)
    -        || (ch != b'~' && marker(before));
    +    let open = after_kind == Kind::Other || (after_kind == Kind::Punctuation && before_kind != Kind::Other);
    +    let close = before_kind == Kind::Other || (before_kind == Kind::Punctuation && after_kind != Kind::Other);
     
         if ch == b'_' {
             (
    @@ -47,14 +41,15 @@ pub(super) fn flanking(ch: u8, before: Option, after: Option, gfm: b
         }
     }
     
    -/// Whether `opener` and `closer` can match, by character and by the rule of three.
    +/// Whether `opener` and `closer` can match, by character and by the rule of three, which goes by the
    +/// length of the whole runs.
     fn matches(opener: &Delim, closer: &Delim) -> bool {
         if !opener.can_open || opener.ch != closer.ch {
             return false;
         }
         if (opener.can_close || closer.can_open)
    -        && !closer.count.is_multiple_of(3)
    -        && (opener.count + closer.count).is_multiple_of(3)
    +        && !closer.original.is_multiple_of(3)
    +        && (opener.original + closer.original).is_multiple_of(3)
         {
             return false;
         }
    diff --git a/crates/mq-markdown/src/parser/inline/link.rs b/crates/mq-markdown/src/parser/inline/link.rs
    index 63a1f031e..444b80a6e 100644
    --- a/crates/mq-markdown/src/parser/inline/link.rs
    +++ b/crates/mq-markdown/src/parser/inline/link.rs
    @@ -2,11 +2,13 @@
     
     use super::entity::{remove_line_indent, unescape};
     use super::tail::{Tail, inline_tail, reference};
    -use super::{Context, Item, Scanner, emphasis, to_nodes};
    +use super::{Context, Item, Scanner, Value, emphasis, to_nodes};
     use crate::node::{FootnoteRef, Image, ImageRef, Link, LinkRef, Node, Position, Title, Url};
    +#[cfg(feature = "wikilink")]
    +use crate::node::{RenderOptions, Text};
     
    -/// Normalizes a reference label the way `markdown-rs` does, including its handling of whitespace:
    -/// the first gap between words is dropped.
    +/// Normalizes a reference label: whitespace runs become one space, the ends are trimmed, and the case
    +/// is folded. `markdown-rs` drops the first gap between words, which this does not.
     pub(in crate::parser) fn normalize(label: &str) -> String {
         let bytes = label.as_bytes();
         let mut result = String::with_capacity(label.len());
    @@ -20,7 +22,7 @@ pub(in crate::parser) fn normalize(label: &str) -> String {
                     in_whitespace = true;
                 }
             } else if in_whitespace {
    -            if start != 0 {
    +            if !result.is_empty() {
                     result.push(' ');
                 }
                 start = index;
    @@ -69,8 +71,15 @@ pub(super) fn close(scanner: &mut Scanner<'_>) -> bool {
         let label = &src[label_start..pos];
         let after = pos + 1;
     
    -    let is_footnote =
    -        !image && footnote_label(label).is_some_and(|name| context.references.footnotes.contains(&normalize(name)));
    +    // After a `!`, a footnote reference is the reference and the `!` is text, unless a destination follows.
    +    let is_footnote = footnote_label(label).is_some_and(|name| context.references.footnotes.contains(&normalize(name)))
    +        && (!image || inline_tail(src, after).is_none());
    +    let bang = image && is_footnote;
    +    let (image, opener_start) = if bang {
    +        (false, opener_start + 1)
    +    } else {
    +        (image, opener_start)
    +    };
         let kind = if is_footnote {
             Some(Kind::Footnote {
                 label: label[1..].to_string(),
    @@ -87,6 +96,13 @@ pub(super) fn close(scanner: &mut Scanner<'_>) -> bool {
         // Everything after the opener becomes the content.
         let mut content = scanner.items.split_off(opener_index + 1);
         scanner.items.pop();
    +    if bang {
    +        scanner.items.push(Item::Text {
    +            start: opener_start - 1,
    +            end: opener_start,
    +            value: Value::Slice(opener_start - 1, opener_start),
    +        });
    +    }
         // Email addresses are not linked inside link text.
         let inner = Context {
             emails: false,
    @@ -165,7 +181,7 @@ fn build(
                     Node::LinkRef(LinkRef {
                         ident: normalize(&label),
                         label: Some(shown),
    -                    values,
    +                    values: unwrap_links(values),
                         position,
                     })
                 };
    @@ -180,19 +196,26 @@ fn footnote_label(label: &str) -> Option<&str> {
         (!name.is_empty() && !name.bytes().any(|b| matches!(b, b' ' | b'\t' | b'\n' | b'\r'))).then_some(name)
     }
     
    -/// Builds a link, unwrapping nested links to the same destination (from autolink literals).
    -pub(super) fn make_link(url: String, title: Option, values: Vec, position: Option) -> Node {
    -    let values = values
    +/// Replaces links in link text by their content: a link cannot contain a link, so these come from
    +/// autolink literals. A wikilink there is text.
    +fn unwrap_links(values: Vec) -> Vec {
    +    values
             .into_iter()
             .flat_map(|child| match child {
    -            Node::Link(Link {
    -                url: Url(ref inner),
    -                values,
    -                ..
    -            }) if *inner == url => values,
    +            Node::Link(Link { values, .. }) => values,
    +            #[cfg(feature = "wikilink")]
    +            Node::WikiLink(link) => vec![Node::Text(Text {
    +                value: Node::WikiLink(link).to_string_with(&RenderOptions::default()),
    +                position: None,
    +            })],
                 other => vec![other],
             })
    -        .collect();
    +        .collect()
    +}
    +
    +/// Builds a link, unwrapping the links in its text.
    +pub(super) fn make_link(url: String, title: Option, values: Vec, position: Option) -> Node {
    +    let values = unwrap_links(values);
     
         Node::Link(Link {
             url: Url(url),
    @@ -216,6 +239,9 @@ fn plain_text(nodes: &[Node]) -> String {
                 Node::Delete(node) => text.push_str(&plain_text(&node.values)),
                 Node::Link(node) => text.push_str(&plain_text(&node.values)),
                 Node::LinkRef(node) => text.push_str(&plain_text(&node.values)),
    +            Node::Break(_) => text.push('\n'),
    +            Node::Image(node) => text.push_str(&node.alt),
    +            Node::ImageRef(node) => text.push_str(&node.alt),
                 _ => {}
             }
         }
    diff --git a/crates/mq-markdown/src/parser/inline/literal.rs b/crates/mq-markdown/src/parser/inline/literal.rs
    index 6be17b217..8a587e3a8 100644
    --- a/crates/mq-markdown/src/parser/inline/literal.rs
    +++ b/crates/mq-markdown/src/parser/inline/literal.rs
    @@ -28,6 +28,13 @@ fn kind_at(src: &str, index: usize) -> Kind {
         }
     }
     
    +/// Whether the character at byte `index` is a symbol of the emoji blocks, which a domain may contain.
    +fn is_emoji(src: &str, index: usize) -> bool {
    +    src.get(index..)
    +        .and_then(|rest| rest.chars().next())
    +        .is_some_and(|char| matches!(char, '\u{2600}'..='\u{27BF}' | '\u{1F300}'..='\u{1FAFF}'))
    +}
    +
     /// Whether the text from `index` is only trailing punctuation up to whitespace, the end or `<`.
     fn trail(src: &str, mut index: usize) -> bool {
         let bytes = src.as_bytes();
    @@ -74,7 +81,7 @@ fn domain(src: &str, mut index: usize) -> Option {
                     index += 1;
                 }
                 Some(b'-' | 0x80..=0xBF) => index += 1,
    -            _ if kind_at(src, index) == Kind::Other => {
    +            _ if kind_at(src, index) == Kind::Other || is_emoji(src, index) => {
                     seen = true;
                     index += 1;
                 }
    @@ -193,7 +200,7 @@ fn local_start(bytes: &[u8], min: usize, at: usize) -> Option {
         {
             index -= 1;
         }
    -    (index != at && !(index > min && bytes[index - 1] == b'/')).then_some(index)
    +    (index != at).then_some(index)
     }
     
     /// The end of the domain of an email address that starts at `start`.
    @@ -237,7 +244,7 @@ fn split_emails(src: &str, start: usize, end: usize, out: &mut Vec) {
                         (from, prefixed, xmpp) = (word, false, name == "xmpp");
                     }
                 }
    -            if let Some(to) = email_domain(&bytes[..end], index + 1, xmpp) {
    +            if let Some(to) = email_domain(&bytes[..end], index + 1, xmpp).filter(|&to| bytes.get(to) != Some(&b'@')) {
                     if emitted < from {
                         out.push(Item::Text {
                             start: emitted,
    diff --git a/crates/mq-markdown/src/parser/line.rs b/crates/mq-markdown/src/parser/line.rs
    index e6c9d8eec..ff27447e2 100644
    --- a/crates/mq-markdown/src/parser/line.rs
    +++ b/crates/mq-markdown/src/parser/line.rs
    @@ -16,11 +16,10 @@ pub(super) struct Line<'a> {
         pub(super) item_end: bool,
         /// Whether the line continues a paragraph without its container prefix.
         pub(super) lazy: bool,
    -    /// Whether indented code that starts on this line is a chunk of its own, as for the first line of
    -    /// an item whose marker is followed by five or more spaces.
    -    pub(super) own_chunk: bool,
         /// The whole line as in the document, to compute columns when it contains tabs.
         pub(super) origin: &'a str,
    +    /// Whether `origin` contains a tab.
    +    pub(super) tabs: bool,
         /// Columns left over from a tab that a container consumed only in part. They count as leading
         /// whitespace of `text`.
         pub(super) pad: usize,
    @@ -36,7 +35,7 @@ impl<'a> Line<'a> {
     
         /// The zero-based visual column where `text` starts in the document.
         fn start_column(&self) -> usize {
    -        if self.origin.contains('\t') {
    +        if self.tabs {
                 visual_column(self.origin[..self.column].chars(), 0)
             } else {
                 self.column
    @@ -65,14 +64,14 @@ impl<'a> Line<'a> {
         }
     
         pub(super) fn is_blank(&self) -> bool {
    -        self.text.trim_matches([' ', '\t']).is_empty()
    +        self.text.bytes().all(|b| matches!(b, b' ' | b'\t'))
         }
     
         /// The position of the byte at `byte` in `text`. Columns are in bytes, except that a tab advances
         /// to the next multiple of four, like `markdown-rs` counts them.
         pub(super) fn point(&self, byte: usize) -> Point {
             let offset = self.column + byte;
    -        let mut column = if self.origin.contains('\t') {
    +        let mut column = if self.tabs {
                 visual_column(self.origin[..offset].chars(), 0) + 1
             } else {
                 offset + 1
    @@ -191,8 +190,8 @@ pub(super) fn split_lines(src: &str, mdx: bool) -> Vec> {
                 eof: index == bytes.len(),
                 item_end: false,
                 lazy: false,
    -            own_chunk: false,
                 origin: &src[start..end],
    +            tabs: src[start..end].contains('\t'),
                 pad: 0,
                 mdx,
             });
    @@ -208,8 +207,8 @@ pub(super) fn split_lines(src: &str, mdx: bool) -> Vec> {
                 eof: true,
                 item_end: false,
                 lazy: false,
    -            own_chunk: false,
                 origin: &src[start..],
    +            tabs: src[start..].contains('\t'),
                 pad: 0,
                 mdx,
             });
    diff --git a/crates/mq-markdown/src/parser/render_html.rs b/crates/mq-markdown/src/parser/render_html.rs
    index 5fe3362da..ba5eb1f77 100644
    --- a/crates/mq-markdown/src/parser/render_html.rs
    +++ b/crates/mq-markdown/src/parser/render_html.rs
    @@ -251,7 +251,7 @@ impl Html {
                 }
                 Node::Html(html) => {
                     self.line_ending_if_needed();
    -                self.out.push_str(&html.value);
    +                push_html(&mut self.out, &html.value);
                 }
                 Node::Code(code) => self.code(code, usize::from(!code.value.is_empty()), true),
                 _ => {}
    @@ -272,13 +272,13 @@ impl Html {
     
         fn code(&mut self, code: &Code, lines: usize, closed: bool) {
             self.line_ending_if_needed();
    -        self.out.push_str("
    ');
    +        self.out.push_str(">");
             self.raw_flow(&code.value, lines, closed);
         }
     
    @@ -324,11 +324,11 @@ impl Html {
     
         fn task_check(&mut self) {
             if let Some(checked) = self.check.take() {
    -            self.out.push_str(" ");
    +            self.out.push_str("disabled=\"\" /> ");
             }
         }
     
    @@ -348,7 +348,8 @@ impl Html {
                 self.check = item.checked;
                 let empty = !item.children.iter().any(is_content);
                 self.blocks(&item.children);
    -            self.check = None;
    +            // An item without content has the checkbox alone.
    +            self.task_check();
     
                 let tight_paragraph =
                     tight
    @@ -453,7 +454,7 @@ impl Html {
                     self.out.push_str("
    "); self.line_ending(); } - Node::Html(html) => self.out.push_str(&html.value), + Node::Html(html) => push_html(&mut self.out, &html.value), Node::Link(link) => { let title = link.title.as_ref().map(|title| title.0.as_str()); self.link(&link.url.0, title, &link.values); @@ -709,3 +710,38 @@ fn sanitize_with_protocols(value: &str, protocols: &[&str]) -> String { } value } + +/// The tags that GFM does not let through as raw HTML: their `<` is written as `<`. +const DISALLOWED_TAGS: [&str; 9] = [ + "title", + "textarea", + "style", + "xmp", + "iframe", + "noembed", + "noframes", + "script", + "plaintext", +]; + +/// Writes raw HTML, with the start and end tags of [`DISALLOWED_TAGS`] made text. +fn push_html(out: &mut String, html: &str) { + let mut rest = html; + while let Some(at) = rest.find('<') { + let (before, tag) = rest.split_at(at); + out.push_str(before); + let name = tag[1..].strip_prefix('/').unwrap_or(&tag[1..]); + let end = name.find(|c: char| !c.is_ascii_alphanumeric()).unwrap_or(name.len()); + let boundary = matches!( + name[end..].chars().next(), + None | Some(' ' | '\t' | '\n' | '\r' | '/' | '>') + ); + if boundary && DISALLOWED_TAGS.iter().any(|tag| name[..end].eq_ignore_ascii_case(tag)) { + out.push_str("<"); + } else { + out.push('<'); + } + rest = &tag[1..]; + } + out.push_str(rest); +} diff --git a/crates/mq-markdown/src/parser/resolve.rs b/crates/mq-markdown/src/parser/resolve.rs index d76eea7dd..56860ea3f 100644 --- a/crates/mq-markdown/src/parser/resolve.rs +++ b/crates/mq-markdown/src/parser/resolve.rs @@ -214,6 +214,7 @@ fn list_nodes(list: ListBlock, level: Level, references: &References, nodes: &mu checked: item.checked, spread: list.spread, start: list.start, + marker: Some(list.marker), position: Some(position), })); diff --git a/crates/mq-markdown/src/parser/snapshots.txt b/crates/mq-markdown/src/parser/snapshots.txt index 584c7b917..9c4716d49 100644 --- a/crates/mq-markdown/src/parser/snapshots.txt +++ b/crates/mq-markdown/src/parser/snapshots.txt @@ -71,20 +71,20 @@ html: "

    a

    \n" nodes: [Heading(Heading { depth: 2, values: [Text(Text { value: "a\nb", @1:1-2:2 })], @1:1-3:4 })] html: "

    a\nb

    \n" ## thematic_star "***\n" -nodes: [HorizontalRule(HorizontalRule { @1:1-1:4 })] +nodes: [HorizontalRule(HorizontalRule { marker: Some('*'), @1:1-1:4 })] html: "
    \n" ## thematic_dash_spaced "- - -\n" -nodes: [HorizontalRule(HorizontalRule { @1:1-1:6 })] +nodes: [HorizontalRule(HorizontalRule { marker: Some('-'), @1:1-1:6 })] html: "
    \n" ## thematic_underscore " ___ \n" -nodes: [HorizontalRule(HorizontalRule { @1:1-1:8 })] +nodes: [HorizontalRule(HorizontalRule { marker: Some('_'), @1:1-1:8 })] html: "
    \n" ## thematic_two "**\n" nodes: [Text(Text { value: "**", @1:1-1:3 })] html: "

    **

    \n" ## fence_lang_meta "```rust title\nlet a;\n```\n" nodes: [Code(Code { value: "let a;", lang: Some("rust"), @1:1-3:4, meta: Some("title"), fence: true })] -html: "
    let a;\n
    \n" +html: "
    let a;\n
    \n" ## fence_indented " ```\n a\n b\n ```\n" nodes: [Code(Code { value: "a\n b", lang: None, @1:3-4:6, meta: None, fence: true })] html: "
    a\n b\n
    \n" @@ -149,124 +149,124 @@ html: "
    \n
    a\n
    \n
    \n

    b

    \n" nodes: [Text(Text { value: "a", @1:1-1:2 }), Blockquote(Blockquote { values: [Text(Text { value: "b", @2:3-2:4 })], @2:1-2:4 })] html: "

    a

    \n
    \n

    b

    \n
    \n" ## quote_thematic_not_lazy "> a\n---\n" -nodes: [Blockquote(Blockquote { values: [Text(Text { value: "a", @1:3-1:4 })], @1:1-1:4 }), HorizontalRule(HorizontalRule { @2:1-2:4 })] +nodes: [Blockquote(Blockquote { values: [Text(Text { value: "a", @1:3-1:4 })], @1:1-1:4 }), HorizontalRule(HorizontalRule { marker: Some('-'), @2:1-2:4 })] html: "
    \n

    a

    \n
    \n
    \n" ## quote_multibyte "> あ\n> い\n" nodes: [Blockquote(Blockquote { values: [Text(Text { value: "あ\nい", @1:3-2:6 })], @1:1-2:6 })] html: "
    \n

    あ\nい

    \n
    \n" ## bullet "- a\n- b\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-1:4 }), List(List { values: [Text(Text { value: "b", @2:3-2:4 })], index: 1, level: 0, ordered: false, checked: None, spread: false, start: None, @2:3-2:4 })] +nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-1:4 }), List(List { values: [Text(Text { value: "b", @2:3-2:4 })], index: 1, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @2:3-2:4 })] html: "
      \n
    • a
    • \n
    • b
    • \n
    \n" ## bullet_loose "- a\n\n- b\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 })], index: 0, level: 0, ordered: false, checked: None, spread: true, start: None, @1:3-1:4 }), List(List { values: [Text(Text { value: "b", @3:3-3:4 })], index: 1, level: 0, ordered: false, checked: None, spread: true, start: None, @3:3-3:4 })] +nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 })], index: 0, level: 0, ordered: false, checked: None, spread: true, start: None, marker: Some('-'), @1:3-1:4 }), List(List { values: [Text(Text { value: "b", @3:3-3:4 })], index: 1, level: 0, ordered: false, checked: None, spread: true, start: None, marker: Some('-'), @3:3-3:4 })] html: "
      \n
    • \n

      a

      \n
    • \n
    • \n

      b

      \n
    • \n
    \n" ## bullet_indent " - a\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:4-1:5 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:4-1:5 })] +nodes: [List(List { values: [Text(Text { value: "a", @1:4-1:5 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:4-1:5 })] html: "
      \n
    • a
    • \n
    \n" ## bullet_mixed_markers "* a\n+ b\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-1:4 }), List(List { values: [Text(Text { value: "b", @2:3-2:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @2:3-2:4 })] +nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('*'), @1:3-1:4 }), List(List { values: [Text(Text { value: "b", @2:3-2:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('+'), @2:3-2:4 })] html: "
      \n
    • a
    • \n
    \n
      \n
    • b
    • \n
    \n" ## ordered "1. a\n2. b\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:4-1:5 })], index: 0, level: 0, ordered: true, checked: None, spread: false, start: Some(1), @1:4-1:5 }), List(List { values: [Text(Text { value: "b", @2:4-2:5 })], index: 1, level: 0, ordered: true, checked: None, spread: false, start: Some(1), @2:4-2:5 })] +nodes: [List(List { values: [Text(Text { value: "a", @1:4-1:5 })], index: 0, level: 0, ordered: true, checked: None, spread: false, start: Some(1), marker: Some('.'), @1:4-1:5 }), List(List { values: [Text(Text { value: "b", @2:4-2:5 })], index: 1, level: 0, ordered: true, checked: None, spread: false, start: Some(1), marker: Some('.'), @2:4-2:5 })] html: "
      \n
    1. a
    2. \n
    3. b
    4. \n
    \n" ## ordered_paren "5) a\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:4-1:5 })], index: 0, level: 0, ordered: true, checked: None, spread: false, start: Some(5), @1:4-1:5 })] +nodes: [List(List { values: [Text(Text { value: "a", @1:4-1:5 })], index: 0, level: 0, ordered: true, checked: None, spread: false, start: Some(5), marker: Some(')'), @1:4-1:5 })] html: "
      \n
    1. a
    2. \n
    \n" ## ordered_zero "0. a\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:4-1:5 })], index: 0, level: 0, ordered: true, checked: None, spread: false, start: Some(0), @1:4-1:5 })] +nodes: [List(List { values: [Text(Text { value: "a", @1:4-1:5 })], index: 0, level: 0, ordered: true, checked: None, spread: false, start: Some(0), marker: Some('.'), @1:4-1:5 })] html: "
      \n
    1. a
    2. \n
    \n" ## ordered_too_long "1234567890. a\n" nodes: [Text(Text { value: "1234567890. a", @1:1-1:14 })] html: "

    1234567890. a

    \n" ## ordered_mixed_delimiters "1. a\n2) b\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:4-1:5 })], index: 0, level: 0, ordered: true, checked: None, spread: false, start: Some(1), @1:4-1:5 }), List(List { values: [Text(Text { value: "b", @2:4-2:5 })], index: 0, level: 0, ordered: true, checked: None, spread: false, start: Some(2), @2:4-2:5 })] +nodes: [List(List { values: [Text(Text { value: "a", @1:4-1:5 })], index: 0, level: 0, ordered: true, checked: None, spread: false, start: Some(1), marker: Some('.'), @1:4-1:5 }), List(List { values: [Text(Text { value: "b", @2:4-2:5 })], index: 0, level: 0, ordered: true, checked: None, spread: false, start: Some(2), marker: Some(')'), @2:4-2:5 })] html: "
      \n
    1. a
    2. \n
    \n
      \n
    1. b
    2. \n
    \n" ## task "- [ ] x\n- [x] y\n- [X] z\n" -nodes: [List(List { values: [Text(Text { value: "x", @1:7-1:8 })], index: 0, level: 0, ordered: false, checked: Some(false), spread: false, start: None, @1:7-1:8 }), List(List { values: [Text(Text { value: "y", @2:7-2:8 })], index: 1, level: 0, ordered: false, checked: Some(true), spread: false, start: None, @2:7-2:8 }), List(List { values: [Text(Text { value: "z", @3:7-3:8 })], index: 2, level: 0, ordered: false, checked: Some(true), spread: false, start: None, @3:7-3:8 })] -html: "
      \n
    • x
    • \n
    • y
    • \n
    • z
    • \n
    \n" +nodes: [List(List { values: [Text(Text { value: "x", @1:7-1:8 })], index: 0, level: 0, ordered: false, checked: Some(false), spread: false, start: None, marker: Some('-'), @1:7-1:8 }), List(List { values: [Text(Text { value: "y", @2:7-2:8 })], index: 1, level: 0, ordered: false, checked: Some(true), spread: false, start: None, marker: Some('-'), @2:7-2:8 }), List(List { values: [Text(Text { value: "z", @3:7-3:8 })], index: 2, level: 0, ordered: false, checked: Some(true), spread: false, start: None, marker: Some('-'), @3:7-3:8 })] +html: "
      \n
    • x
    • \n
    • y
    • \n
    • z
    • \n
    \n" ## task_empty "- [ ]\n" -nodes: [List(List { values: [Text(Text { value: "[ ]", @1:3-1:6 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-1:6 })] +nodes: [List(List { values: [Text(Text { value: "[ ]", @1:3-1:6 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-1:6 })] html: "
      \n
    • [ ]
    • \n
    \n" ## task_no_space "- [ ]x\n" -nodes: [List(List { values: [Text(Text { value: "[ ]x", @1:3-1:7 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-1:7 })] +nodes: [List(List { values: [Text(Text { value: "[ ]x", @1:3-1:7 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-1:7 })] html: "
      \n
    • [ ]x
    • \n
    \n" ## nested "- a\n - b\n c\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-1:4 }), List(List { values: [Text(Text { value: "b\nc", @2:5-3:6 })], index: 0, level: 1, ordered: false, checked: None, spread: false, start: None, @2:5-3:6 })] +nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-1:4 }), List(List { values: [Text(Text { value: "b\nc", @2:5-3:6 })], index: 0, level: 1, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @2:5-3:6 })] html: "
      \n
    • a\n
        \n
      • b\nc
      • \n
      \n
    • \n
    \n" ## nested_ordered "1. a\n 1. b\n 2. c\n2. d\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:4-1:5 })], index: 0, level: 0, ordered: true, checked: None, spread: false, start: Some(1), @1:4-1:5 }), List(List { values: [Text(Text { value: "b", @2:7-2:8 })], index: 0, level: 1, ordered: true, checked: None, spread: false, start: Some(1), @2:7-2:8 }), List(List { values: [Text(Text { value: "c", @3:7-3:8 })], index: 1, level: 1, ordered: true, checked: None, spread: false, start: Some(1), @3:7-3:8 }), List(List { values: [Text(Text { value: "d", @4:4-4:5 })], index: 1, level: 0, ordered: true, checked: None, spread: false, start: Some(1), @4:4-4:5 })] +nodes: [List(List { values: [Text(Text { value: "a", @1:4-1:5 })], index: 0, level: 0, ordered: true, checked: None, spread: false, start: Some(1), marker: Some('.'), @1:4-1:5 }), List(List { values: [Text(Text { value: "b", @2:7-2:8 })], index: 0, level: 1, ordered: true, checked: None, spread: false, start: Some(1), marker: Some('.'), @2:7-2:8 }), List(List { values: [Text(Text { value: "c", @3:7-3:8 })], index: 1, level: 1, ordered: true, checked: None, spread: false, start: Some(1), marker: Some('.'), @3:7-3:8 }), List(List { values: [Text(Text { value: "d", @4:4-4:5 })], index: 1, level: 0, ordered: true, checked: None, spread: false, start: Some(1), marker: Some('.'), @4:4-4:5 })] html: "
      \n
    1. a\n
        \n
      1. b
      2. \n
      3. c
      4. \n
      \n
    2. \n
    3. d
    4. \n
    \n" ## nested_dedent "- a\n - b\n- c\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-1:4 }), List(List { values: [Text(Text { value: "b", @2:5-2:6 })], index: 0, level: 1, ordered: false, checked: None, spread: false, start: None, @2:5-2:6 }), List(List { values: [Text(Text { value: "c", @3:3-3:4 })], index: 1, level: 0, ordered: false, checked: None, spread: false, start: None, @3:3-3:4 })] +nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-1:4 }), List(List { values: [Text(Text { value: "b", @2:5-2:6 })], index: 0, level: 1, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @2:5-2:6 }), List(List { values: [Text(Text { value: "c", @3:3-3:4 })], index: 1, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @3:3-3:4 })] html: "
      \n
    • a\n
        \n
      • b
      • \n
      \n
    • \n
    • c
    • \n
    \n" ## two_paragraphs "- a\n\n b\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 }), Text(Text { value: "b", @3:3-3:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-3:4 })] +nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 }), Text(Text { value: "b", @3:3-3:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-3:4 })] html: "
      \n
    • \n

      a

      \n

      b

      \n
    • \n
    \n" ## two_paragraphs_then_item "- a\n\n b\n- c\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 }), Text(Text { value: "b", @3:3-3:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-3:4 }), List(List { values: [Text(Text { value: "c", @4:3-4:4 })], index: 1, level: 0, ordered: false, checked: None, spread: false, start: None, @4:3-4:4 })] +nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 }), Text(Text { value: "b", @3:3-3:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-3:4 }), List(List { values: [Text(Text { value: "c", @4:3-4:4 })], index: 1, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @4:3-4:4 })] html: "
      \n
    • \n

      a

      \n

      b

      \n
    • \n
    • \n

      c

      \n
    • \n
    \n" ## empty_item "-\n" -nodes: [List(List { values: [], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:1-1:2 })] +nodes: [List(List { values: [], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:1-1:2 })] html: "
      \n
    • \n
    \n" ## empty_item_content "-\n foo\n" -nodes: [List(List { values: [Text(Text { value: "foo", @2:3-2:6 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @2:3-2:6 })] +nodes: [List(List { values: [Text(Text { value: "foo", @2:3-2:6 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @2:3-2:6 })] html: "
      \n
    • foo
    • \n
    \n" ## empty_item_blank_then_text "-\n\n foo\n" -nodes: [List(List { values: [], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:1-1:2 }), Text(Text { value: "foo", @3:3-3:6 })] +nodes: [List(List { values: [], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:1-1:2 }), Text(Text { value: "foo", @3:3-3:6 })] html: "
      \n
    • \n
    \n

    foo

    \n" ## empty_items "-\n-\n" -nodes: [List(List { values: [], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:1-1:2 }), List(List { values: [], index: 1, level: 0, ordered: false, checked: None, spread: false, start: None, @2:1-2:2 })] +nodes: [List(List { values: [], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:1-1:2 }), List(List { values: [], index: 1, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @2:1-2:2 })] html: "
      \n
    • \n
    • \n
    \n" ## wide_marker_gap "- a\n b\n" -nodes: [List(List { values: [Text(Text { value: "a\nb", @1:5-2:6 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:5-2:6 })] +nodes: [List(List { values: [Text(Text { value: "a\nb", @1:5-2:6 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:5-2:6 })] html: "
      \n
    • a\nb
    • \n
    \n" ## code_in_item "- a\n" -nodes: [List(List { values: [Code(Code { value: "a", lang: None, @1:3-1:8, meta: None, fence: false })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-1:8 })] +nodes: [List(List { values: [Code(Code { value: "a", lang: None, @1:3-1:8, meta: None, fence: false })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-1:8 })] html: "
      \n
    • \n
      a\n
      \n
    • \n
    \n" ## lazy "- a\nb\n" -nodes: [List(List { values: [Text(Text { value: "a\nb", @1:3-2:2 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-2:2 })] +nodes: [List(List { values: [Text(Text { value: "a\nb", @1:3-2:2 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-2:2 })] html: "
      \n
    • a\nb
    • \n
    \n" ## lazy_after_blank "- a\n\nb\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-1:4 }), Text(Text { value: "b", @3:1-3:2 })] +nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-1:4 }), Text(Text { value: "b", @3:1-3:2 })] html: "
      \n
    • a
    • \n
    \n

    b

    \n" ## list_interrupts_paragraph "a\n- b\n" -nodes: [Text(Text { value: "a", @1:1-1:2 }), List(List { values: [Text(Text { value: "b", @2:3-2:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @2:3-2:4 })] +nodes: [Text(Text { value: "a", @1:1-1:2 }), List(List { values: [Text(Text { value: "b", @2:3-2:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @2:3-2:4 })] html: "

    a

    \n
      \n
    • b
    • \n
    \n" ## ordered_two_no_interrupt "a\n2. b\n" nodes: [Text(Text { value: "a\n2. b", @1:1-2:5 })] html: "

    a\n2. b

    \n" ## ordered_one_interrupts "a\n1. b\n" -nodes: [Text(Text { value: "a", @1:1-1:2 }), List(List { values: [Text(Text { value: "b", @2:4-2:5 })], index: 0, level: 0, ordered: true, checked: None, spread: false, start: Some(1), @2:4-2:5 })] +nodes: [Text(Text { value: "a", @1:1-1:2 }), List(List { values: [Text(Text { value: "b", @2:4-2:5 })], index: 0, level: 0, ordered: true, checked: None, spread: false, start: Some(1), marker: Some('.'), @2:4-2:5 })] html: "

    a

    \n
      \n
    1. b
    2. \n
    \n" ## empty_item_no_interrupt "a\n-\n" nodes: [Heading(Heading { depth: 2, values: [Text(Text { value: "a", @1:1-1:2 })], @1:1-2:2 })] html: "

    a

    \n" ## thematic_over_list "- - -\n" -nodes: [HorizontalRule(HorizontalRule { @1:1-1:6 })] +nodes: [HorizontalRule(HorizontalRule { marker: Some('-'), @1:1-1:6 })] html: "
    \n" ## list_in_quote "> - a\n> - b\n" -nodes: [Blockquote(Blockquote { values: [List(List { values: [Text(Text { value: "a", @1:5-1:6 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:5-1:6 }), List(List { values: [Text(Text { value: "b", @2:5-2:6 })], index: 1, level: 0, ordered: false, checked: None, spread: false, start: None, @2:5-2:6 })], @1:1-2:6 })] +nodes: [Blockquote(Blockquote { values: [List(List { values: [Text(Text { value: "a", @1:5-1:6 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:5-1:6 }), List(List { values: [Text(Text { value: "b", @2:5-2:6 })], index: 1, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @2:5-2:6 })], @1:1-2:6 })] html: "
    \n
      \n
    • a
    • \n
    • b
    • \n
    \n
    \n" ## quote_in_list "- > a\n > b\n" -nodes: [List(List { values: [Blockquote(Blockquote { values: [Text(Text { value: "a\nb", @1:5-2:6 })], @1:3-2:6 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-2:6 })] +nodes: [List(List { values: [Blockquote(Blockquote { values: [Text(Text { value: "a\nb", @1:5-2:6 })], @1:3-2:6 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-2:6 })] html: "
      \n
    • \n
      \n

      a\nb

      \n
      \n
    • \n
    \n" ## quote_in_list_level "- a\n > - b\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 }), Blockquote(Blockquote { values: [List(List { values: [Text(Text { value: "b", @2:7-2:8 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @2:7-2:8 })], @2:3-2:8 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-2:8 })] +nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 }), Blockquote(Blockquote { values: [List(List { values: [Text(Text { value: "b", @2:7-2:8 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @2:7-2:8 })], @2:3-2:8 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-2:8 })] html: "
      \n
    • a\n
      \n
        \n
      • b
      • \n
      \n
      \n
    • \n
    \n" ## heading_in_item "- # h\n text\n" -nodes: [List(List { values: [Heading(Heading { depth: 1, values: [Text(Text { value: "h", @1:5-1:6 })], @1:3-1:6 }), Text(Text { value: "text", @2:3-2:7 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-2:7 })] +nodes: [List(List { values: [Heading(Heading { depth: 1, values: [Text(Text { value: "h", @1:5-1:6 })], @1:3-1:6 }), Text(Text { value: "text", @2:3-2:7 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-2:7 })] html: "
      \n
    • \n

      h

      \ntext
    • \n
    \n" ## fence_in_item "- ```\n a\n ```\n" -nodes: [List(List { values: [Code(Code { value: "a", lang: None, @1:3-3:6, meta: None, fence: true })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-3:6 })] +nodes: [List(List { values: [Code(Code { value: "a", lang: None, @1:3-3:6, meta: None, fence: true })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-3:6 })] html: "
      \n
    • \n
      a\n
      \n
    • \n
    \n" ## multibyte_item "- あ\n- い\n" -nodes: [List(List { values: [Text(Text { value: "あ", @1:3-1:6 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-1:6 }), List(List { values: [Text(Text { value: "い", @2:3-2:6 })], index: 1, level: 0, ordered: false, checked: None, spread: false, start: None, @2:3-2:6 })] +nodes: [List(List { values: [Text(Text { value: "あ", @1:3-1:6 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-1:6 }), List(List { values: [Text(Text { value: "い", @2:3-2:6 })], index: 1, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @2:3-2:6 })] html: "
      \n
    • あ
    • \n
    • い
    • \n
    \n" ## list_after_paragraph_blank "a\n\n- b\n" -nodes: [Text(Text { value: "a", @1:1-1:2 }), List(List { values: [Text(Text { value: "b", @3:3-3:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @3:3-3:4 })] +nodes: [Text(Text { value: "a", @1:1-1:2 }), List(List { values: [Text(Text { value: "b", @3:3-3:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @3:3-3:4 })] html: "

    a

    \n
      \n
    • b
    • \n
    \n" ## two_lists_split "- a\n\n\n- b\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 })], index: 0, level: 0, ordered: false, checked: None, spread: true, start: None, @1:3-1:4 }), List(List { values: [Text(Text { value: "b", @4:3-4:4 })], index: 1, level: 0, ordered: false, checked: None, spread: true, start: None, @4:3-4:4 })] +nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 })], index: 0, level: 0, ordered: false, checked: None, spread: true, start: None, marker: Some('-'), @1:3-1:4 }), List(List { values: [Text(Text { value: "b", @4:3-4:4 })], index: 1, level: 0, ordered: false, checked: None, spread: true, start: None, marker: Some('-'), @4:3-4:4 })] html: "
      \n
    • \n

      a

      \n
    • \n
    • \n

      b

      \n
    • \n
    \n" ## deep_quote ">>>>>> a\n" nodes: [Blockquote(Blockquote { values: [Blockquote(Blockquote { values: [Blockquote(Blockquote { values: [Blockquote(Blockquote { values: [Blockquote(Blockquote { values: [Blockquote(Blockquote { values: [Text(Text { value: "a", @1:8-1:9 })], @1:6-1:9 })], @1:5-1:9 })], @1:4-1:9 })], @1:3-1:9 })], @1:2-1:9 })], @1:1-1:9 })] @@ -275,10 +275,10 @@ html: "
    \n
    \n
    \n
    \n
    \na\n
    \n" ## fence_unclosed_in_item "- ```\n a\n" -nodes: [List(List { values: [Code(Code { value: "a", lang: None, @1:3-3:1, meta: None, fence: true })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-3:1 })] +nodes: [List(List { values: [Code(Code { value: "a", lang: None, @1:3-3:1, meta: None, fence: true })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-3:1 })] html: "
      \n
    • \n
      a\n
      \n
    • \n
    " ## fence_unclosed_in_item_no_eol "- ```\n a" -nodes: [List(List { values: [Code(Code { value: "a", lang: None, @1:3-2:4, meta: None, fence: true })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-2:4 })] +nodes: [List(List { values: [Code(Code { value: "a", lang: None, @1:3-2:4, meta: None, fence: true })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-2:4 })] html: "
      \n
    • \n
      a\n
      \n
    • \n
    " ## fence_unclosed_in_quote_no_eol "> ```\n> a" nodes: [Blockquote(Blockquote { values: [Code(Code { value: "a", lang: None, @1:3-2:4, meta: None, fence: true })], @1:1-2:4 })] @@ -290,14 +290,14 @@ html: "
    \n
    a\n
    \n
    " nodes: [Code(Code { value: "a\n", lang: None, @1:1-4:1, meta: None, fence: true })] html: "
    a\n\n
    \n" ## fence_unclosed_item_then_para "- ```\n a\nb\n" -nodes: [List(List { values: [Code(Code { value: "a", lang: None, @1:3-2:4, meta: None, fence: true })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-2:4 }), Text(Text { value: "b", @3:1-3:2 })] +nodes: [List(List { values: [Code(Code { value: "a", lang: None, @1:3-2:4, meta: None, fence: true })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-2:4 }), Text(Text { value: "b", @3:1-3:2 })] html: "
      \n
    • \n
      a\n
      \n
    • \n
    \n

    b

    \n" ## item_code_then_unindented_text "- ===\n\n a\n===" -nodes: [List(List { values: [Text(Text { value: "===", @1:3-1:6 }), Code(Code { value: "a", lang: None, @3:3-3:8, meta: None, fence: false })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-3:8 }), Text(Text { value: "===", @4:1-4:4 })] +nodes: [List(List { values: [Text(Text { value: "===", @1:3-1:6 }), Code(Code { value: "a", lang: None, @3:3-3:8, meta: None, fence: false })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-3:8 }), Text(Text { value: "===", @4:1-4:4 })] html: "
      \n
    • \n

      ===

      \n
      a\n
      \n
    • \n
    \n

    ===

    " ## quote_setext_dashes "> a\n> ---\n> ---\n> a" -nodes: [Blockquote(Blockquote { values: [Heading(Heading { depth: 2, values: [Text(Text { value: "a", @1:3-1:4 })], @1:3-2:6 }), Text(Text { value: "---", @3:3-3:6 }), Text(Text { value: "a", @4:3-4:4 })], @1:1-4:4 })] -html: "
    \n

    a

    \n

    ---

    \n

    a

    \n
    " +nodes: [Blockquote(Blockquote { values: [Heading(Heading { depth: 2, values: [Text(Text { value: "a", @1:3-1:4 })], @1:3-2:6 }), HorizontalRule(HorizontalRule { marker: Some('-'), @3:3-3:6 }), Text(Text { value: "a", @4:3-4:4 })], @1:1-4:4 })] +html: "
    \n

    a

    \n
    \n

    a

    \n
    " ## table_basic "| a | b |\n|---|:-:|\n| 1 | 2 |\n" nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:3-1:4 })]:0, row: 0, @1:1-1:5 }), TableCell(TableCell { values: [Text(Text { value: "b", @1:7-1:8 })]:1, row: 0, @1:5-1:10 }), TableAlign(TableAlign { align: [None, Center], @2:1-2:1 }), TableCell(TableCell { values: [Text(Text { value: "1", @3:3-3:4 })]:0, row: 1, @3:1-3:5 }), TableCell(TableCell { values: [Text(Text { value: "2", @3:7-3:8 })]:1, row: 1, @3:5-3:10 })] html: "
  • " +## html_img_self_close "\nz" +nodes: [Html(Html { value: "\nz", @1:1-2:2 })] +html: "\nz" +## html_img_then_text " y" +nodes: [Html(Html { value: "", @1:1-1:16 }), Text(Text { value: " y", @1:16-1:18 })] +html: "

    y

    " +## html_multiline_tag_inline "\nz" +nodes: [Html(Html { value: "", @1:1-2:3 }), Text(Text { value: "\nz", @2:3-3:2 })] +html: "

    \nz

    " +## html_indent4 "
    " +nodes: [Code(Code { value: "
    ", lang: None, @1:1-1:10, meta: None, fence: false })] +html: "
    <div>\n
    " +## html_comment_then_text "y\nz" +nodes: [Html(Html { value: "y", @1:1-1:10 }), Text(Text { value: "z", @2:1-2:2 })] +html: "y\n

    z

    " +## html_then_code_line "
    \n code" +nodes: [Html(Html { value: "
    \n code", @1:1-2:9 })] +html: "
    \n code" +## html_comment_short "\nx" +nodes: [Html(Html { value: "", @1:1-1:6 }), Text(Text { value: "x", @2:1-2:2 })] +html: "\n

    x

    " +## html_instruction_short "\nx" +nodes: [Html(Html { value: "", @1:1-1:4 }), Text(Text { value: "x", @2:1-2:2 })] +html: "\n

    x

    " +## html_lazy_quote ">
    \nx" +nodes: [Blockquote(Blockquote { values: [Html(Html { value: "
    ", @1:3-1:8 })], @1:1-1:8 }), Text(Text { value: "x", @2:1-2:2 })] +html: "
    \n
    \n
    \n

    x

    " +## html_attr_forms "\nx" +nodes: [Html(Html { value: "\nx", @1:1-2:2 })] +html: "\nx" +## html_bad_attr "\nx" +nodes: [Text(Text { value: "\nx", @1:1-2:2 })] +html: "

    <a b=>\nx

    " +## fm_yaml "---\na: b\n---\ntext" +nodes: [Yaml(Yaml { value: "a: b", @1:1-3:4 }), Text(Text { value: "text", @4:1-4:5 })] +html: "

    text

    " +## fm_yaml_only "---\na: b\n---" +nodes: [Yaml(Yaml { value: "a: b", @1:1-3:4 })] +html: "" +## fm_empty "---\n---" +nodes: [Yaml(Yaml { value: "", @1:1-2:4 })] +html: "" +## fm_unclosed "---\na\n" +nodes: [HorizontalRule(HorizontalRule { @1:1-1:4 }), Text(Text { value: "a", @2:1-2:2 })] +html: "
    \n

    a

    \n" +## fm_blank_content "---\n\n---\nx" +nodes: [Yaml(Yaml { value: "", @1:1-3:4 }), Text(Text { value: "x", @4:1-4:2 })] +html: "

    x

    " +## fm_toml "+++\na = 1\n+++\nx" +nodes: [Toml(Toml { value: "a = 1", @1:1-3:4 }), Text(Text { value: "x", @4:1-4:2 })] +html: "

    x

    " +## fm_indented " ---\na\n---" +nodes: [HorizontalRule(HorizontalRule { @1:1-1:5 }), Heading(Heading { depth: 2, values: [Text(Text { value: "a", @2:1-2:2 })], @2:1-3:4 })] +html: "
    \n

    a

    " +## fm_close_trailing_space "---\na\n--- \nx" +nodes: [Yaml(Yaml { value: "a", @1:1-3:5 }), Text(Text { value: "x", @4:1-4:2 })] +html: "

    x

    " +## fm_close_longer "---\na\n----\nx" +nodes: [HorizontalRule(HorizontalRule { @1:1-1:4 }), Heading(Heading { depth: 2, values: [Text(Text { value: "a", @2:1-2:2 })], @2:1-3:5 }), Text(Text { value: "x", @4:1-4:2 })] +html: "
    \n

    a

    \n

    x

    " +## fm_open_trailing_space "--- \na\n---" +nodes: [Yaml(Yaml { value: "a", @1:1-3:4 })] +html: "" +## fm_open_junk "---x\na\n---" +nodes: [Heading(Heading { depth: 2, values: [Text(Text { value: "---x\na", @1:1-2:2 })], @1:1-3:4 })] +html: "

    ---x\na

    " +## fm_not_first "text\n---\na\n---" +nodes: [Heading(Heading { depth: 2, values: [Text(Text { value: "text", @1:1-1:5 })], @1:1-2:4 }), Heading(Heading { depth: 2, values: [Text(Text { value: "a", @3:1-3:2 })], @3:1-4:4 })] +html: "

    text

    \n

    a

    " +## fm_blank_before "\n---\na\n---" +nodes: [HorizontalRule(HorizontalRule { @2:1-2:4 }), Heading(Heading { depth: 2, values: [Text(Text { value: "a", @3:1-3:2 })], @3:1-4:4 })] +html: "
    \n

    a

    " +## fm_multiline_content "---\na: 1\n\nb: 2\n---\n# h" +nodes: [Yaml(Yaml { value: "a: 1\n\nb: 2", @1:1-5:4 }), Heading(Heading { depth: 1, values: [Text(Text { value: "h", @6:3-6:4 })], @6:1-6:4 })] +html: "

    h

    " +## fm_hr_after "---\na\n---\n\n---" +nodes: [Yaml(Yaml { value: "a", @1:1-3:4 }), HorizontalRule(HorizontalRule { @5:1-5:4 })] +html: "
    " +## math_basic "$$\na\n$$" +nodes: [Math(Math { value: "a", @1:1-3:3 })] +html: "
    a\n
    " +## math_then_text "$$\na\n$$\nx" +nodes: [Math(Math { value: "a", @1:1-3:3 }), Text(Text { value: "x", @4:1-4:2 })] +html: "
    a\n
    \n

    x

    " +## math_meta "$$ meta\na\n$$" +nodes: [Math(Math { value: "a", @1:1-3:3 })] +html: "
    a\n
    " +## math_unclosed "$$\na" +nodes: [Math(Math { value: "a", @1:1-2:2 })] +html: "
    a\n
    \n" +## math_longer_open "$$$\na\n$$" +nodes: [Math(Math { value: "a\n$$", @1:1-3:3 })] +html: "
    a\n$$\n
    \n" +## math_longer_close "$$\na\n$$$$" +nodes: [Math(Math { value: "a", @1:1-3:5 })] +html: "
    a\n
    " +## math_indented " $$\n a\n b\n $$" +nodes: [Math(Math { value: "a\n b", @1:3-4:5 })] +html: "
    a\n b\n
    " +## math_empty "$$\n$$" +nodes: [Math(Math { value: "", @1:1-2:3 })] +html: "
    " +## math_inline_not_block "$$a$$" +nodes: [MathInline(MathInline { value: "a", @1:1-1:6 })] +html: "

    a

    " +## math_info_no_close "$$a\n$$" +nodes: [Math(Math { value: "", @1:1-2:3 })] +html: "
    " +## math_in_quote "> $$\n> a\n> $$" +nodes: [Blockquote(Blockquote { values: [Math(Math { value: "a", @1:3-3:5 })], @1:1-3:5 })] +html: "
    \n
    a\n
    \n
    " +## math_in_list "- $$\n a\n $$" +nodes: [List(List { values: [Math(Math { value: "a", @1:3-3:5 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-3:5 })] +html: "
      \n
    • \n
      a\n
      \n
    • \n
    " +## math_close_junk "$$\na\n$$ x" +nodes: [Math(Math { value: "a\n$$ x", @1:1-3:5 })] +html: "
    a\n$$ x\n
    \n" +## math_interrupts_paragraph "text\n$$\na\n$$" +nodes: [Text(Text { value: "text", @1:1-1:5 }), Math(Math { value: "a", @2:1-4:3 })] +html: "

    text

    \n
    a\n
    " +## math_blank_inside "$$\n\na\n$$" +nodes: [Math(Math { value: "\na", @1:1-4:3 })] +html: "
    \na\n
    " +## math_single_dollar_line "$ $\nx" +nodes: [MathInline(MathInline { value: " ", @1:1-1:4 }), Text(Text { value: "\nx", @1:4-2:2 })] +html: "

    \nx

    " +## crlf_frontmatter "---\r\na\r\n---\r\nx" +nodes: [Yaml(Yaml { value: "a", @1:1-3:4 }), Text(Text { value: "x", @4:1-4:2 })] +html: "

    x

    " +## break_end_indented "a \n b" +nodes: [Text(Text { value: "a", @1:1-1:2 }), Break(Break { @1:2-2:1 }), Text(Text { value: "b", @2:3-2:4 })] +html: "

    a
    \nb

    " +## text_end_indented_continuation "*a*\n " +nodes: [Emphasis(Emphasis { values: [Text(Text { value: "a", @1:2-1:3 })], @1:1-1:4 }), Text(Text { value: "\n", @1:4-2:1 }), Html(Html { value: "", @2:3-2:6 })] +html: "

    a\n

    " +## soft_end_before_node "a\n `b`" +nodes: [Text(Text { value: "a\n", @1:1-2:1 }), CodeInline(CodeInline { value: "b", @2:3-2:6 })] +html: "

    a\nb

    " +## break_in_quote "> a \n> b" +nodes: [Blockquote(Blockquote { values: [Text(Text { value: "a", @1:3-1:4 }), Break(Break { @1:4-2:1 }), Text(Text { value: "b", @2:3-2:4 })], @1:1-2:4 })] +html: "
    \n

    a
    \nb

    \n
    " +## text_eol_in_footnote "[^a]: x \n [a]" +nodes: [Footnote(Footnote { ident: "a", values: [Text(Text { value: "x\n[a]", @1:7-2:6 })], @1:1-2:6 })] +html: "" +## math_whitespace_content " $$\n \n $$" +nodes: [Math(Math { value: "", @1:3-3:5 })] +html: "
    \n
    " +## math_whitespace_content2 " $$\n \n $$" +nodes: [Math(Math { value: "", @1:3-3:5 })] +html: "
    \n
    " +## table_indented_no_pipe " a|b\n|-|:-:|" +nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:3-1:4 })]:0, row: 0, @1:1-1:4 }), TableCell(TableCell { values: [Text(Text { value: "b", @1:5-1:6 })]:1, row: 0, @1:4-1:6 }), TableAlign(TableAlign { align: [None, Center], @2:1-2:1 })] +html: "\n\n\n\n\n\n\n
    ab
    " +## tab_nested_list "- a\n\t- b\n\t\t- c\n" +nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-1:4 }), List(List { values: [Text(Text { value: "b", @2:7-2:8 })], index: 0, level: 1, ordered: false, checked: None, spread: false, start: None, @2:7-2:8 }), List(List { values: [Text(Text { value: "c", @3:11-3:12 })], index: 0, level: 2, ordered: false, checked: None, spread: false, start: None, @3:11-3:12 })] +html: "
      \n
    • a\n
        \n
      • b\n
          \n
        • c
        • \n
        \n
      • \n
      \n
    • \n
    \n" +## tab_nested_ordered "1. a\n\t1. b\n\t\t1. c\n" +nodes: [List(List { values: [Text(Text { value: "a", @1:4-1:5 })], index: 0, level: 0, ordered: true, checked: None, spread: false, start: Some(1), @1:4-1:5 }), List(List { values: [Text(Text { value: "b", @2:8-2:9 })], index: 0, level: 1, ordered: true, checked: None, spread: false, start: Some(1), @2:8-2:9 }), List(List { values: [Text(Text { value: "c", @3:12-3:13 })], index: 0, level: 2, ordered: true, checked: None, spread: false, start: Some(1), @3:12-3:13 })] +html: "
      \n
    1. a\n
        \n
      1. b\n
          \n
        1. c
        2. \n
        \n
      2. \n
      \n
    2. \n
    \n" +## tab_list_code "- a\n\n\t\tcode\n" +nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 }), Code(Code { value: "code", lang: None, @3:3-3:13, meta: None, fence: false })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-3:13 })] +html: "
      \n
    • \n

      a

      \n
      code\n
      \n
    • \n
    \n" +## tab_in_text "a\tb\t*c*\n" +nodes: [Text(Text { value: "a\tb\t", @1:1-1:9 }), Emphasis(Emphasis { values: [Text(Text { value: "c", @1:10-1:11 })], @1:9-1:12 })] +html: "

    a\tb\tc

    \n" +## tab_code_block "\tcode\n\t\tmore\n" +nodes: [Code(Code { value: "code\n\tmore", lang: None, @1:1-2:13, meta: None, fence: false })] +html: "
    code\n\tmore\n
    \n" +## tab_after_marker "-\ta\n" +nodes: [List(List { values: [Text(Text { value: "a", @1:5-1:6 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:5-1:6 })] +html: "
      \n
    • a
    • \n
    \n" +## atx_leading_sequences "# # a" +nodes: [Heading(Heading { depth: 1, values: [Text(Text { value: "a", @1:5-1:6 })], @1:1-1:6 })] +html: "

    a

    " +## atx_leading_sequences2 "## # a" +nodes: [Heading(Heading { depth: 2, values: [Text(Text { value: "a", @1:6-1:7 })], @1:1-1:7 })] +html: "

    a

    " +## atx_only_sequences "# # #" +nodes: [Heading(Heading { depth: 1, values: [], @1:1-1:6 })] +html: "

    " +## atx_no_space_sequence "# #a" +nodes: [Heading(Heading { depth: 1, values: [Text(Text { value: "a", @1:4-1:5 })], @1:1-1:5 })] +html: "

    a

    " +## atx_double_sequence "# ## a" +nodes: [Heading(Heading { depth: 1, values: [Text(Text { value: "a", @1:6-1:7 })], @1:1-1:7 })] +html: "

    a

    " +## atx_tab_sequence "#\t# a" +nodes: [Heading(Heading { depth: 1, values: [Text(Text { value: "a", @1:7-1:8 })], @1:1-1:8 })] +html: "

    a

    " +## atx_closing_and_leading "# # a #" +nodes: [Heading(Heading { depth: 1, values: [Text(Text { value: "a", @1:5-1:6 })], @1:1-1:8 })] +html: "

    a

    " +## atx_inner_hash "# a # b" +nodes: [Heading(Heading { depth: 1, values: [Text(Text { value: "a # b", @1:3-1:8 })], @1:1-1:8 })] +html: "

    a # b

    " +## quote_footnote_trailing_space "- a\n > [^a]: x\n " +nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 }), Blockquote(Blockquote { values: [Footnote(Footnote { ident: "a", values: [Text(Text { value: "x", @2:11-2:12 })], @2:5-3:3 })], @2:3-3:3 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-3:3 })] +html: "
      \n
    • a\n
      \n
      \n
    • \n
    " +## indented_code_trailing_blank " code\n\n\n" +nodes: [Code(Code { value: "code", lang: None, @1:1-1:9, meta: None, fence: false })] +html: "
    code\n
    \n" +## bom_heading "\u{feff}# a\n" +nodes: [Heading(Heading { depth: 1, values: [Text(Text { value: "a", @1:6-1:7 })], @1:4-1:7 })] +html: "

    a

    \n" +## bom_frontmatter "\u{feff}---\na: b\n---\n" +nodes: [Yaml(Yaml { value: "a: b", @1:4-3:4 })] +html: "" +## bom_text "\u{feff}a\n" +nodes: [Text(Text { value: "a", @1:4-1:5 })] +html: "

    a

    \n" +## www_multibyte_after_ww "wwß x" +nodes: [Text(Text { value: "wwß x", @1:1-1:7 })] +html: "

    wwß x

    " +## www_combining "ww\u{301}.a.com" +nodes: [Text(Text { value: "ww\u{301}.a.com", @1:1-1:11 })] +html: "

    ww\u{301}.a.com

    " +## www_cjk "www.あ.com/パス x" +nodes: [Link(Link { url: Url("http://www.あ.com/パス"), title: None, values: [Text(Text { value: "www.あ.com/パス", @1:1-1:19 })], @1:1-1:19 }), Text(Text { value: " x", @1:19-1:21 })] +html: "

    www.あ.com/パス x

    " +## http_cjk "http://あ.jp/パス ok" +nodes: [Link(Link { url: Url("http://あ.jp/パス"), title: None, values: [Text(Text { value: "http://あ.jp/パス", @1:1-1:21 })], @1:1-1:21 }), Text(Text { value: " ok", @1:21-1:24 })] +html: "

    http://あ.jp/パス ok

    " +## email_cjk_domain "a@あ.com" +nodes: [Text(Text { value: "a@あ.com", @1:1-1:10 })] +html: "

    a@あ.com

    " +## email_cjk_local "あ@b.com" +nodes: [Text(Text { value: "あ@b.com", @1:1-1:10 })] +html: "

    あ@b.com

    " +## html_inline_continuation "" +nodes: [Html(Html { value: "", @1:1-2:12 })] +html: "

    " +## html_inline_continuation_tab "" +nodes: [Html(Html { value: "", @1:1-2:15 })] +html: "

    " +## table_trailing_space "| a | b | \n|-|-| \n| c | d | \n" +nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:3-1:4 })]:0, row: 0, @1:1-1:5 }), TableCell(TableCell { values: [Text(Text { value: "b", @1:7-1:8 })]:1, row: 0, @1:5-1:12 }), TableAlign(TableAlign { align: [None, None], @2:1-2:1 }), TableCell(TableCell { values: [Text(Text { value: "c", @3:3-3:4 })]:0, row: 1, @3:1-3:5 }), TableCell(TableCell { values: [Text(Text { value: "d", @3:7-3:8 })]:1, row: 1, @3:5-3:12 })] +html: "\n\n\n\n\n\n\n\n\n\n\n\n\n
    ab
    cd
    \n" +## table_no_pipes_trailing_space "a | b\n--|--\nd | e \n" +nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:1-1:2 })]:0, row: 0, @1:1-1:3 }), TableCell(TableCell { values: [Text(Text { value: "b", @1:5-1:6 })]:1, row: 0, @1:3-1:6 }), TableAlign(TableAlign { align: [None, None], @2:1-2:1 }), TableCell(TableCell { values: [Text(Text { value: "d", @3:1-3:2 })]:0, row: 1, @3:1-3:3 }), TableCell(TableCell { values: [Text(Text { value: "e", @3:5-3:6 })]:1, row: 1, @3:3-3:7 })] +html: "\n\n\n\n\n\n\n\n\n\n\n\n\n
    ab
    de
    \n" +## table_trailing_tab "| a | b |\t\n|-|-|\n| c | d | \n" +nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:3-1:4 })]:0, row: 0, @1:1-1:5 }), TableCell(TableCell { values: [Text(Text { value: "b", @1:7-1:8 })]:1, row: 0, @1:5-1:13 }), TableAlign(TableAlign { align: [None, None], @2:1-2:1 }), TableCell(TableCell { values: [Text(Text { value: "c", @3:3-3:4 })]:0, row: 1, @3:1-3:5 }), TableCell(TableCell { values: [Text(Text { value: "d", @3:7-3:8 })]:1, row: 1, @3:5-3:13 })] +html: "\n\n\n\n\n\n\n\n\n\n\n\n\n
    ab
    cd
    \n" +## table_cjk "| あ | 🎉 |\n|:-|-:|\n| é | ß |\n" +nodes: [TableCell(TableCell { values: [Text(Text { value: "あ", @1:3-1:6 })]:0, row: 0, @1:1-1:7 }), TableCell(TableCell { values: [Text(Text { value: "🎉", @1:9-1:13 })]:1, row: 0, @1:7-1:15 }), TableAlign(TableAlign { align: [Left, Right], @2:1-2:1 }), TableCell(TableCell { values: [Text(Text { value: "é", @3:3-3:5 })]:0, row: 1, @3:1-3:6 }), TableCell(TableCell { values: [Text(Text { value: "ß", @3:8-3:10 })]:1, row: 1, @3:6-3:12 })] +html: "\n\n\n\n\n\n\n\n\n\n\n\n\n
    あ🎉
    éß
    \n" +## table_single_column_colon "a\n:-\nb\n" +nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:1-1:2 })]:0, row: 0, @1:1-1:2 }), TableAlign(TableAlign { align: [Left], @2:1-2:1 }), TableCell(TableCell { values: [Text(Text { value: "b", @3:1-3:2 })]:0, row: 1, @3:1-3:2 })] +html: "\n\n\n\n\n\n\n\n\n\n\n
    a
    b
    \n" +## table_single_column_right "a\n---:\nb\nc\n" +nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:1-1:2 })]:0, row: 0, @1:1-1:2 }), TableAlign(TableAlign { align: [Right], @2:1-2:1 }), TableCell(TableCell { values: [Text(Text { value: "b", @3:1-3:2 })]:0, row: 1, @3:1-3:2 }), TableCell(TableCell { values: [Text(Text { value: "c", @4:1-4:2 })]:0, row: 2, @4:1-4:2 })] +html: "\n\n\n\n\n\n\n\n\n\n\n\n\n\n
    a
    b
    c
    \n" +## table_double_colon_not_delimiter "a\n::-\n" +nodes: [Text(Text { value: "a\n::-", @1:1-2:4 })] +html: "

    a\n::-

    \n" +## table_body_empty_marker "a|b\n|-|-|\n*\n" +nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:1-1:2 })]:0, row: 0, @1:1-1:2 }), TableCell(TableCell { values: [Text(Text { value: "b", @1:3-1:4 })]:1, row: 0, @1:2-1:4 }), TableAlign(TableAlign { align: [None, None], @2:1-2:1 }), List(List { values: [], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @3:1-3:2 })] +html: "\n\n\n\n\n\n\n
    ab
    \n
      \n
    • \n
    \n" +## table_body_ordered_marker "a|b\n|-|-|\n2. x\n" +nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:1-1:2 })]:0, row: 0, @1:1-1:2 }), TableCell(TableCell { values: [Text(Text { value: "b", @1:3-1:4 })]:1, row: 0, @1:2-1:4 }), TableAlign(TableAlign { align: [None, None], @2:1-2:1 }), List(List { values: [Text(Text { value: "x", @3:4-3:5 })], index: 0, level: 0, ordered: true, checked: None, spread: false, start: Some(2), @3:4-3:5 })] +html: "\n\n\n\n\n\n\n
    ab
    \n
      \n
    1. x
    2. \n
    \n" +## table_body_empty_ordered "a|b\n|-|-|\n1.\n" +nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:1-1:2 })]:0, row: 0, @1:1-1:2 }), TableCell(TableCell { values: [Text(Text { value: "b", @1:3-1:4 })]:1, row: 0, @1:2-1:4 }), TableAlign(TableAlign { align: [None, None], @2:1-2:1 }), List(List { values: [], index: 0, level: 0, ordered: true, checked: None, spread: false, start: Some(1), @3:1-3:3 })] +html: "\n\n\n\n\n\n\n
    ab
    \n
      \n
    1. \n
    \n" +## paragraph_after_partial_tab "- a\n\n\t[P](h) x\n" +nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 }), Link(Link { url: Url("h"), title: None, values: [Text(Text { value: "P", @3:6-3:7 })], @3:5-3:11 }), Text(Text { value: " x", @3:11-3:13 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-3:13 })] +html: "
      \n
    • \n

      a

      \n

      P x

      \n
    • \n
    \n" +## fence_after_partial_tab "- a\n\n\t```ts\n\t\tx\n\t```\n" +nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 }), Code(Code { value: "\tx", lang: Some("ts"), @3:5-5:8, meta: None, fence: true })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-5:8 })] +html: "
      \n
    • \n

      a

      \n
      \tx\n
      \n
    • \n
    \n" +## math_after_partial_tab "- a\n\n\t$$\n\tm\n\t$$\n" +nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 }), Math(Math { value: "m", @3:5-5:7 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-5:7 })] +html: "
      \n
    • \n

      a

      \n
      m\n
      \n
    • \n
    \n" +## quote_after_partial_tab "- a\n\n\t> q\n" +nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 }), Blockquote(Blockquote { values: [Text(Text { value: "q", @3:7-3:8 })], @3:3-3:8 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-3:8 })] +html: "
      \n
    • \n

      a

      \n
      \n

      q

      \n
      \n
    • \n
    \n" +## heading_after_partial_tab "- a\n\n\t# h\n" +nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 }), Heading(Heading { depth: 1, values: [Text(Text { value: "h", @3:7-3:8 })], @3:3-3:8 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-3:8 })] +html: "
      \n
    • \n

      a

      \n

      h

      \n
    • \n
    \n" +## control_character_references "… Ÿ ���" +nodes: [Text(Text { value: "����\u{c}�\u{a0}���", @1:1-1:61 })] +html: "

    ����\u{c}�\u{a0}���

    " +## footnote_escaped_bracket "[^a\\]: x" +nodes: [Text(Text { value: "[^a]: x", @1:1-1:9 })] +html: "

    [^a]: x

    " +## footnote_escaped_backslash "[^a\\\\]: x" +nodes: [Footnote(Footnote { ident: "a\\\\", values: [Text(Text { value: "x", @1:9-1:10 })], @1:1-1:10 })] +html: "" +## footnote_escaped_open "[^a\\[]: x" +nodes: [Footnote(Footnote { ident: "a\\[", values: [Text(Text { value: "x", @1:9-1:10 })], @1:1-1:10 })] +html: "" +## footnote_unescaped_open "[^a[]: x" +nodes: [Text(Text { value: "[^a[]: x", @1:1-1:9 })] +html: "

    [^a[]: x

    " +## footnote_empty_then_blank "[^a]: \n\n" +nodes: [Footnote(Footnote { ident: "a", values: [], @1:1-2:1 })] +html: "" +## footnote_empty_then_spaces "[^a]: \n \n" +nodes: [Footnote(Footnote { ident: "a", values: [], @1:1-2:5 })] +html: "" +## footnote_empty_then_text "[^a]: \n\nb" +nodes: [Footnote(Footnote { ident: "a", values: [], @1:1-2:1 }), Text(Text { value: "b", @3:1-3:2 })] +html: "

    b

    " +## definition_nul_destination "[a]: \0\n\n[a]\n" +nodes: [Definition(Definition { @1:1-1:7, url: Url("\0"), title: None, ident: "a", label: Some("a") }), LinkRef(LinkRef { ident: "a", label: Some("a"), values: [Text(Text { value: "a", @3:2-3:3 })], @3:1-3:4 })] +html: "

    a

    \n" +## link_nul_destination "[a](x\0y)" +nodes: [Link(Link { url: Url("x\0y"), title: None, values: [Text(Text { value: "a", @1:2-1:3 })], @1:1-1:9 })] +html: "

    a

    " +## definition_then_empty_marker "[a]: b\n-" +nodes: [Definition(Definition { @1:1-1:7, url: Url("b"), title: None, ident: "a", label: Some("a") }), Text(Text { value: "-", @2:1-2:2 })] +html: "

    -

    " +## definition_then_item "[a]: b\n- x" +nodes: [Definition(Definition { @1:1-1:7, url: Url("b"), title: None, ident: "a", label: Some("a") }), List(List { values: [Text(Text { value: "x", @2:3-2:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @2:3-2:4 })] +html: "
      \n
    • x
    • \n
    " +## definition_then_ordered "[a]: b\n1." +nodes: [Definition(Definition { @1:1-1:7, url: Url("b"), title: None, ident: "a", label: Some("a") }), Text(Text { value: "1.", @2:1-2:3 })] +html: "

    1.

    " +## emphasis_cjk "*あ*。*い*" +nodes: [Emphasis(Emphasis { values: [Text(Text { value: "あ", @1:2-1:5 })], @1:1-1:6 }), Text(Text { value: "。", @1:6-1:9 }), Emphasis(Emphasis { values: [Text(Text { value: "い", @1:10-1:13 })], @1:9-1:14 })] +html: "

    あ。い

    " +## strong_cjk "**日本語**です" +nodes: [Strong(Strong { values: [Text(Text { value: "日本語", @1:3-1:12 })], @1:1-1:14 }), Text(Text { value: "です", @1:14-1:20 })] +html: "

    日本語です

    " +## underscore_accent "_é_ _ß_" +nodes: [Emphasis(Emphasis { values: [Text(Text { value: "é", @1:2-1:4 })], @1:1-1:5 }), Text(Text { value: " ", @1:5-1:6 }), Emphasis(Emphasis { values: [Text(Text { value: "ß", @1:7-1:9 })], @1:6-1:10 })] +html: "

    é ß

    " +## fullwidth_markers "*a* _b_" +nodes: [Text(Text { value: "*a* _b_", @1:1-1:16 })] +html: "

    *a* _b_

    " +## emphasis_after_cjk_punctuation "。*a*、**b**" +nodes: [Text(Text { value: "。", @1:1-1:4 }), Emphasis(Emphasis { values: [Text(Text { value: "a", @1:5-1:6 })], @1:4-1:7 }), Text(Text { value: "、", @1:7-1:10 }), Strong(Strong { values: [Text(Text { value: "b", @1:12-1:13 })], @1:10-1:15 })] +html: "

    。a、b

    " +## combining_mark "e\u{301}*a*" +nodes: [Text(Text { value: "e\u{301}", @1:1-1:4 }), Emphasis(Emphasis { values: [Text(Text { value: "a", @1:5-1:6 })], @1:4-1:7 })] +html: "

    e\u{301}a

    " +## emoji_zwj "👨\u{200d}👩\u{200d}👧 *a*" +nodes: [Text(Text { value: "👨\u{200d}👩\u{200d}👧 ", @1:1-1:20 }), Emphasis(Emphasis { values: [Text(Text { value: "a", @1:21-1:22 })], @1:20-1:23 })] +html: "

    👨\u{200d}👩\u{200d}👧 a

    " +## fence_cjk_info "```あ\n日本語\n```\n" +nodes: [Code(Code { value: "日本語", lang: Some("あ"), @1:1-3:4, meta: None, fence: true })] +html: "
    日本語\n
    \n" +## fence_in_quote_emoji "> ```🎉\n> x\n> ```\n" +nodes: [Blockquote(Blockquote { values: [Code(Code { value: "x", lang: Some("🎉"), @1:3-3:6, meta: None, fence: true })], @1:1-3:6 })] +html: "
    \n
    x\n
    \n
    \n" +## fence_in_list_cjk "- ```あ\n 日本語\n ```\n" +nodes: [List(List { values: [Code(Code { value: "日本語", lang: Some("あ"), @1:3-3:6, meta: None, fence: true })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-3:6 })] +html: "
      \n
    • \n
      日本語\n
      \n
    • \n
    \n" +## math_cjk "$$\nあ\n$$\n\n$い$\n" +nodes: [Math(Math { value: "あ", @1:1-3:3 }), MathInline(MathInline { value: "い", @5:1-5:6 })] +html: "
    あ\n
    \n

    い

    \n" +## heading_cjk "# 見出し\n\n見出し2\n===\n" +nodes: [Heading(Heading { depth: 1, values: [Text(Text { value: "見出し", @1:3-1:12 })], @1:1-1:12 }), Heading(Heading { depth: 1, values: [Text(Text { value: "見出し2", @3:1-3:11 })], @3:1-4:4 })] +html: "

    見出し

    \n

    見出し2

    \n" +## link_cjk "[あ](http://い.jp/う \"え\")" +nodes: [Link(Link { url: Url("http://い.jp/う"), title: Some(Title("え")), values: [Text(Text { value: "あ", @1:2-1:5 })], @1:1-1:31 })] +html: "

    あ

    " +## reference_cjk "[あ]\n\n[あ]: /u\n" +nodes: [LinkRef(LinkRef { ident: "あ", label: Some("あ"), values: [Text(Text { value: "あ", @1:2-1:5 })], @1:1-1:6 }), Definition(Definition { @3:1-3:10, url: Url("/u"), title: None, ident: "あ", label: Some("あ") })] +html: "

    あ

    \n" +## footnote_cjk "あ[^い]\n\n[^い]: う\n" +nodes: [Text(Text { value: "あ", @1:1-1:4 }), FootnoteRef(FootnoteRef { ident: "い", label: Some("い"), @1:4-1:10 }), Footnote(Footnote { ident: "い", values: [Text(Text { value: "う", @3:9-3:12 })], @3:1-3:12 })] +html: "

    あ1

    \n

    Footnotes

    \n
      \n
    1. \n

      う ↩

      \n
    2. \n
    \n
    \n" +## ideographic_space "a\u{3000}b\n\u{3000}c" +nodes: [Text(Text { value: "a\u{3000}b\n\u{3000}c", @1:1-2:5 })] +html: "

    a\u{3000}b\n\u{3000}c

    " +## nbsp_around_emphasis "a\u{a0}*b*\u{a0}c" +nodes: [Text(Text { value: "a\u{a0}", @1:1-1:4 }), Emphasis(Emphasis { values: [Text(Text { value: "b", @1:5-1:6 })], @1:4-1:7 }), Text(Text { value: "\u{a0}c", @1:7-1:10 })] +html: "

    a\u{a0}b\u{a0}c

    " +## entity_cjk_context "あ&いあ🎉" +nodes: [Text(Text { value: "あ&いあ🎉", @1:1-1:29 })] +html: "

    あ&いあ🎉

    " +## autolink_cjk "" +nodes: [Link(Link { url: Url("http://あ.jp/パス"), title: None, values: [Text(Text { value: "http://あ.jp/パス", @1:2-1:22 })], @1:1-1:23 })] +html: "

    http://あ.jp/パス

    " +## code_span_cjk "`あ`と``い`う``" +nodes: [CodeInline(CodeInline { value: "あ", @1:1-1:6 }), Text(Text { value: "と", @1:6-1:9 }), CodeInline(CodeInline { value: "い`う", @1:9-1:20 })] +html: "

    あとい`う

    " +## hard_break_cjk "あ \nい\\\nう" +nodes: [Text(Text { value: "あ", @1:1-1:4 }), Break(Break { @1:4-2:1 }), Text(Text { value: "い", @2:1-2:4 }), Break(Break { @2:4-3:1 }), Text(Text { value: "う", @3:1-3:4 })] +html: "

    あ
    \nい
    \nう

    " +## nul_in_text "a\0b" +nodes: [Text(Text { value: "a\0b", @1:1-1:4 })] +html: "

    a�b

    " +## mdx jsx_self "" +nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [], @1:1-1:6, name: Some("a"), attributes: [] })] +## mdx jsx_attrs "" +nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [], @1:1-1:39, name: Some("a"), attributes: [Property(MdxJsxAttribute { name: "b", value: Some(Literal("c&d")) }), Property(MdxJsxAttribute { name: "e", value: Some(Literal("f")) }), Property(MdxJsxAttribute { name: "g", value: Some(Expression("h")) }), Expression("...i"), Property(MdxJsxAttribute { name: "j", value: None })] })] +## mdx jsx_member "" +nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [], @1:1-1:10, name: Some("a.b.c"), attributes: [] })] +## mdx jsx_namespace "" +nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [], @1:1-1:16, name: Some("a:b"), attributes: [Property(MdxJsxAttribute { name: "c:d", value: Some(Literal("e")) })] })] +## mdx jsx_fragment "<>" +nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [], @1:1-1:6, name: None, attributes: [] })] +## mdx jsx_text_pair "x" +nodes: [MdxJsxTextElement(MdxJsxTextElement { children: [Text(Text { value: "x", @1:4-1:5 })], @1:1-1:9, name: Some("a"), attributes: [] })] +## mdx jsx_flow_pair "\n\nx\n\n" +nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [Text(Text { value: "x", @3:1-3:2 })], @1:1-5:5, name: Some("a"), attributes: [] })] +## mdx jsx_flow_indented_child "\n x\n" +nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [Text(Text { value: "x", @2:3-2:4 })], @1:1-3:5, name: Some("a"), attributes: [] })] +## mdx jsx_inline "a c d" +nodes: [Text(Text { value: "a ", @1:1-1:3 }), MdxJsxTextElement(MdxJsxTextElement { children: [Text(Text { value: "c", @1:6-1:7 })], @1:3-1:11, name: Some("b"), attributes: [] }), Text(Text { value: " d", @1:11-1:13 })] +## mdx jsx_inline_self "a d" +nodes: [Text(Text { value: "a ", @1:1-1:3 }), MdxJsxTextElement(MdxJsxTextElement { children: [], @1:3-1:7, name: Some("b"), attributes: [] }), Text(Text { value: " d", @1:7-1:9 })] +## mdx expr_flow "{a}" +nodes: [MdxFlowExpression(MdxFlowExpression { value: "a", @1:1-1:4 })] +## mdx expr_nested "{a {b} c}" +nodes: [MdxFlowExpression(MdxFlowExpression { value: "a {b} c", @1:1-1:10 })] +## mdx expr_multiline "{a\nb}" +nodes: [MdxFlowExpression(MdxFlowExpression { value: "a\nb", @1:1-2:3 })] +## mdx expr_multiline_indented "{a\n b\n c}" +nodes: [MdxFlowExpression(MdxFlowExpression { value: "a\nb\n c", @1:1-3:6 })] +## mdx expr_text "x {a} y" +nodes: [Text(Text { value: "x ", @1:1-1:3 }), MdxTextExpression(MdxTextExpression { value: "a", @1:3-1:6 }), Text(Text { value: " y", @1:6-1:8 })] +## mdx expr_then_tag "{a} " +nodes: [MdxFlowExpression(MdxFlowExpression { value: "a", @1:1-1:4 }), MdxJsxFlowElement(MdxJsxFlowElement { children: [], @1:5-1:9, name: Some("b"), attributes: [] })] +## mdx tag_then_expr " {a}" +nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [], @1:1-1:5, name: Some("b"), attributes: [] }), MdxFlowExpression(MdxFlowExpression { value: "a", @1:6-1:9 })] +## mdx tag_then_text " x" +nodes: [MdxJsxTextElement(MdxJsxTextElement { children: [], @1:1-1:5, name: Some("b"), attributes: [] }), Text(Text { value: " x", @1:5-1:7 })] +## mdx expr_then_text "{a}x" +nodes: [MdxTextExpression(MdxTextExpression { value: "a", @1:1-1:4 }), Text(Text { value: "x", @1:4-1:5 })] +## mdx jsx_in_expr_child "{b}" +nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [MdxFlowExpression(MdxFlowExpression { value: "b", @1:4-1:7 })], @1:1-1:11, name: Some("a"), attributes: [] })] +## mdx jsx_unclosed_text "b" +nodes: error: Expected a closing tag for `` before the end of the content +## mdx jsx_mismatch "" +nodes: error: Unexpected closing tag ``, expected corresponding closing tag for `` (1:1) +## mdx lt_space "a < b" +nodes: [Text(Text { value: "a < b", @1:1-1:6 })] +## mdx lt_digit "a <3" +nodes: error: Unexpected character `3` (U+0033) before name +## mdx attr_no_value "" +nodes: error: Unexpected character `>` (U+003E) before attribute value +## mdx jsx_nested_flow "\n\n\n" +nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [MdxJsxFlowElement(MdxJsxFlowElement { children: [], @2:1-3:5, name: Some("b"), attributes: [] })], @1:1-4:5, name: Some("a"), attributes: [] })] +## mdx jsx_in_quote "> \n> x\n> " +nodes: [Blockquote(Blockquote { values: [MdxJsxFlowElement(MdxJsxFlowElement { children: [Text(Text { value: "x", @2:3-2:4 })], @1:3-3:7, name: Some("a"), attributes: [] })], @1:1-3:7 })] +## mdx jsx_in_list "- \n x\n " +nodes: [List(List { values: [MdxJsxFlowElement(MdxJsxFlowElement { children: [Text(Text { value: "x", @2:3-2:4 })], @1:3-3:7, name: Some("a"), attributes: [] })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-3:7 })] +## mdx attr_quote_inside "" +nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [], @1:1-1:14, name: Some("a"), attributes: [Property(MdxJsxAttribute { name: "b", value: Some(Literal("c\"d")) })] })] +## mdx two_tags "" +nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [], @1:1-1:5, name: Some("a"), attributes: [] }), MdxJsxFlowElement(MdxJsxFlowElement { children: [], @1:5-1:9, name: Some("b"), attributes: [] })] +## mdx two_text_elements "xy" +nodes: [MdxJsxTextElement(MdxJsxTextElement { children: [Text(Text { value: "x", @1:4-1:5 })], @1:1-1:9, name: Some("a"), attributes: [] }), MdxJsxTextElement(MdxJsxTextElement { children: [Text(Text { value: "y", @1:12-1:13 })], @1:9-1:17, name: Some("b"), attributes: [] })] +## mdx interrupt_paragraph_tag "a\n" +nodes: [Text(Text { value: "a", @1:1-1:2 }), MdxJsxFlowElement(MdxJsxFlowElement { children: [], @2:1-2:5, name: Some("b"), attributes: [] })] +## mdx interrupt_paragraph_expr "a\n{b}" +nodes: [Text(Text { value: "a", @1:1-1:2 }), MdxFlowExpression(MdxFlowExpression { value: "b", @2:1-2:4 })] +## mdx two_expressions "{a}\n{b}" +nodes: [MdxFlowExpression(MdxFlowExpression { value: "a", @1:1-1:4 }), MdxFlowExpression(MdxFlowExpression { value: "b", @2:1-2:4 })] +## mdx expr_in_flow_element "\n\n{b}\n\n" +nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [MdxFlowExpression(MdxFlowExpression { value: "b", @3:1-3:4 })], @1:1-5:5, name: Some("a"), attributes: [] })] +## mdx indented_tag " " +nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [], @1:4-1:8, name: Some("a"), attributes: [] })] +## mdx very_indented_tag " " +nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [], @1:5-1:9, name: Some("a"), attributes: [] })] +## mdx heading_jsx "# h b" +nodes: [Heading(Heading { depth: 1, values: [Text(Text { value: "h ", @1:3-1:5 }), MdxJsxTextElement(MdxJsxTextElement { children: [Text(Text { value: "b", @1:8-1:9 })], @1:5-1:13, name: Some("a"), attributes: [] })], @1:1-1:13 })] +## mdx emphasis_crossing "*x*" +nodes: [Emphasis(Emphasis { values: [MdxJsxTextElement(MdxJsxTextElement { children: [Text(Text { value: "x", @1:5-1:6 })], @1:2-1:10, name: Some("a"), attributes: [] })], @1:1-1:11 })] +## mdx emphasis_crossing_bad "*x*" +nodes: error: Expected a closing tag for `` before the end of the content +## mdx link_jsx "[x](y)" +nodes: [Link(Link { url: Url("y"), title: None, values: [MdxJsxTextElement(MdxJsxTextElement { children: [Text(Text { value: "x", @1:5-1:6 })], @1:2-1:10, name: Some("a"), attributes: [] })], @1:1-1:14 })] +## mdx code_not_jsx "`` {b}" +nodes: [CodeInline(CodeInline { value: "", @1:1-1:7 }), Text(Text { value: " ", @1:7-1:8 }), MdxTextExpression(MdxTextExpression { value: "b", @1:8-1:11 })] +## mdx escaped "\\ \\{b}" +nodes: [Text(Text { value: " {b}", @1:2-1:11 })] +## mdx attr_entities "" +nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [], @1:1-1:21, name: Some("a"), attributes: [Property(MdxJsxAttribute { name: "b", value: Some(Literal("" +nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [], @1:1-2:6, name: Some("a"), attributes: [Property(MdxJsxAttribute { name: "b", value: Some(Literal("x\ny")) })] })] +## mdx closing_spaces "x" +nodes: [MdxJsxTextElement(MdxJsxTextElement { children: [Text(Text { value: "x", @1:6-1:7 })], @1:1-1:13, name: Some("a"), attributes: [] })] +## mdx lt_space_name "< a>" +nodes: [Text(Text { value: "< a>", @1:1-1:5 })] +## mdx self_closing_space "" +nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [], @1:1-1:7, name: Some("a"), attributes: [] })] +## mdx attr_on_next_line "" +nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [], @1:1-2:5, name: Some("a"), attributes: [Property(MdxJsxAttribute { name: "b", value: None })] })] +## mdx dashed_name "" +nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [], @1:1-1:8, name: Some("a-b"), attributes: [] })] +## mdx dashed_attr "" +nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [], @1:1-1:14, name: Some("a"), attributes: [Property(MdxJsxAttribute { name: "b-c", value: Some(Literal("d")) })] })] +## mdx bad_name_char "" +nodes: error: Unexpected character `$` (U+0024) after name +## mdx expr_unclosed "{a" +nodes: error: Unexpected end of file in expression, expected a corresponding closing brace for `{` +## mdx expr_empty "{}" +nodes: [MdxFlowExpression(MdxFlowExpression { value: "", @1:1-1:3 })] +## mdx expr_space "{ }" +nodes: [MdxFlowExpression(MdxFlowExpression { value: " ", @1:1-1:4 })] +## mdx empty_flow_element "\n" +nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [], @1:1-2:5, name: Some("a"), attributes: [] })] +## mdx text_element_unclosed_line "x\n" +nodes: error: Expected a closing tag for `` before the end of the content +## mdx stray_close "" +nodes: error: Unexpected closing slash `/` in tag, expected an open tag first +## mdx stray_close_text "x " +nodes: error: Unexpected closing slash `/` in tag, expected an open tag first +## mdx element_with_emphasis "*b*" +nodes: [MdxJsxTextElement(MdxJsxTextElement { children: [Emphasis(Emphasis { values: [Text(Text { value: "b", @1:5-1:6 })], @1:4-1:7 })], @1:1-1:11, name: Some("a"), attributes: [] })] +## mdx flow_element_with_emphasis "\n*b*\n" +nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [Emphasis(Emphasis { values: [Text(Text { value: "b", @2:2-2:3 })], @2:1-2:4 })], @1:1-3:5, name: Some("a"), attributes: [] })] +## mdx flow_element_with_list "\n- b\n" +nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [List(List { values: [Text(Text { value: "b", @2:3-2:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @2:3-2:4 })], @1:1-3:5, name: Some("a"), attributes: [] })] +## mdx list_items_unbalanced "- \n- " +nodes: error: Expected a closing tag for `` (1:3) before the end of its container +## mdx spaced_equals "" +nodes: [Text(Text { value: "", @1:1-1:14 })] +## mdx heading "# h\n\ntext *a* [b](c) `d`\n" +nodes: [Heading(Heading { depth: 1, values: [Text(Text { value: "h", @1:3-1:4 })], @1:1-1:4 }), Text(Text { value: "text ", @3:1-3:6 }), Emphasis(Emphasis { values: [Text(Text { value: "a", @3:7-3:8 })], @3:6-3:9 }), Text(Text { value: " ", @3:9-3:10 }), Link(Link { url: Url("c"), title: None, values: [Text(Text { value: "b", @3:11-3:12 })], @3:10-3:16 }), Text(Text { value: " ", @3:16-3:17 }), CodeInline(CodeInline { value: "d", @3:17-3:20 })] +## mdx indented_is_text " not code\n" +nodes: [Text(Text { value: "not code", @1:5-1:13 })] +## mdx indented_heading " # h\n" +nodes: [Heading(Heading { depth: 1, values: [Text(Text { value: "h", @1:8-1:9 })], @1:1-1:9 })] +## mdx no_autolink "" +nodes: error: Unexpected character `/` (U+002F) before local name +## mdx no_gfm "| a |\n|-|\n\n~a~ www.a.b [^a]\n\n- [ ] a\n" +nodes: [Text(Text { value: "| a |\n|-|", @1:1-2:4 }), Text(Text { value: "~a~ www.a.b [^a]", @4:1-4:17 }), List(List { values: [Text(Text { value: "[ ] a", @6:3-6:8 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @6:3-6:8 })] +## mdx no_math "$a$\n\n$$\na\n$$\n" +nodes: [Text(Text { value: "$a$", @1:1-1:4 }), Text(Text { value: "$$\na\n$$", @3:1-5:3 })] +## mdx no_frontmatter "---\na\n---\n" +nodes: [HorizontalRule(HorizontalRule { @1:1-1:4 }), Heading(Heading { depth: 2, values: [Text(Text { value: "a", @2:1-2:2 })], @2:1-3:4 })] +## mdx esm_is_text "import a from 'b'\n\nexport const c = 1\n" +nodes: [Text(Text { value: "import a from 'b'", @1:1-1:18 }), Text(Text { value: "export const c = 1", @3:1-3:19 })] +## mdx definition "[a]: /u\n\n[a]\n" +nodes: [Definition(Definition { @1:1-1:8, url: Url("/u"), title: None, ident: "a", label: Some("a") }), LinkRef(LinkRef { ident: "a", label: Some("a"), values: [Text(Text { value: "a", @3:2-3:3 })], @3:1-3:4 })] +## mdx list_deep_indent "- a\n" +nodes: [List(List { values: [Text(Text { value: "a", @1:8-1:9 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:8-1:9 })] +## mdx fence "```rust\ncode\n```\n" +nodes: [Code(Code { value: "code", lang: Some("rust"), @1:1-3:4, meta: None, fence: true })] +## mdx lone_cr_after_lt "<\r/-x" +nodes: [Text(Text { value: "<\r/-x", @1:1-2:4 })] +## mdx crlf_after_lt "<\r\n/-x" +nodes: [Text(Text { value: "<\r\n/-x", @1:1-2:4 })] +## mdx expression_lone_cr "{\r }" +nodes: [MdxFlowExpression(MdxFlowExpression { value: "\r", @1:1-2:3 })] +## mdx expression_cr_indent "{a\r b\r\n c\n d}" +nodes: [MdxFlowExpression(MdxFlowExpression { value: "a\r b\r\nc\nd", @1:1-4:5 })] +## mdx tilde_not_delimiter "*~*a" +nodes: [Text(Text { value: "*~*a", @1:1-1:5 })] +## mdx tilde_strong_not_delimiter "**~**あ" +nodes: [Text(Text { value: "**~**あ", @1:1-1:9 })] +## mdx dollar_after_name "" +nodes: error: Unexpected character `$` (U+0024) after name +## mdx dollar_after_space "" +nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [], @1:1-1:8, name: Some("a"), attributes: [Property(MdxJsxAttribute { name: "$b", value: None })] })] +## mdx cjk_tag_name "<あ b=\"c\" />" +nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [], @1:1-1:14, name: Some("あ"), attributes: [Property(MdxJsxAttribute { name: "b", value: Some(Literal("c")) })] })] +## mdx fullwidth_underscore_name "" +nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [], @1:1-1:10, name: Some("a_b"), attributes: [] })] +## mdx jsx_text_cjk "あ い う" +nodes: [Text(Text { value: "あ ", @1:1-1:5 }), MdxJsxTextElement(MdxJsxTextElement { children: [Text(Text { value: "い", @1:8-1:11 })], @1:5-1:15, name: Some("b"), attributes: [] }), Text(Text { value: " う", @1:15-1:19 })] +## mdx expression_cjk "{あ} い" +nodes: [MdxTextExpression(MdxTextExpression { value: "あ", @1:1-1:6 }), Text(Text { value: " い", @1:6-1:10 })] diff --git a/crates/mq-markdown/src/parser/table.rs b/crates/mq-markdown/src/parser/table.rs index 47199a7b8..83f2983a5 100644 --- a/crates/mq-markdown/src/parser/table.rs +++ b/crates/mq-markdown/src/parser/table.rs @@ -60,7 +60,8 @@ fn split_row<'a>(line: &Line<'a>) -> Vec> { let content_offset = from + (raw.len() - raw.trim_start_matches([' ', '\t']).len()); // The first cell starts at the start of the line, later ones at their leading pipe. let start = if index == first { 0 } else { pipes[index - 1] }; - let end = if index + 1 == last { end } else { pipes[index] }; + // The last cell takes the whitespace at the end of the line along. + let end = if index + 1 == last { text.len() } else { pipes[index] }; Cell { start, end, @@ -74,8 +75,8 @@ fn split_row<'a>(line: &Line<'a>) -> Vec> { /// Parses a delimiter cell such as `:-:` into its alignment. fn align(content: &str) -> Option { let left = content.starts_with(':'); - let right = content.ends_with(':') && content.len() > 1; - let dashes = content.trim_start_matches(':').trim_end_matches(':'); + let right = content.len() > 1 && content.ends_with(':'); + let dashes = &content[usize::from(left)..content.len() - usize::from(right)]; if dashes.is_empty() || !dashes.bytes().all(|b| b == b'-') { return None; } @@ -103,7 +104,8 @@ fn header<'a>( return None; } let text = delimiter.text.trim_matches([' ', '\t']); - if !text.contains('|') + // Without a pipe, a colon tells the delimiter row from the underline of a heading. + if !text.contains(['|', ':']) || !text.contains('-') || !text.bytes().all(|b| matches!(b, b'|' | b':' | b'-' | b' ' | b'\t')) { @@ -141,13 +143,15 @@ fn cell_item(line: &Line<'_>, row: usize, column: usize, cell: &Cell<'_>) -> Tab } /// Parses a table starting at `lines[start]`, returning its nodes and the index after it. -/// `ends_row` tells which lines end the table besides blank lines. +/// `starts_block` tells which lines cannot be the delimiter row, `ends_row` which lines end the table +/// besides blank lines. pub(super) fn parse( lines: &[Line<'_>], start: usize, + starts_block: fn(&Line<'_>) -> bool, ends_row: fn(&Line<'_>) -> bool, ) -> Option<(Vec, usize)> { - let (cells, aligns) = header(lines, start, ends_row)?; + let (cells, aligns) = header(lines, start, starts_block)?; let mut items = Vec::new(); for (column, cell) in cells.iter().enumerate() { @@ -192,7 +196,8 @@ pub(super) fn row_cells(line: &Line<'_>) -> usize { /// The number of cells of `line` when it is a delimiter row such as `|:-|-:|`. pub(super) fn delimiter_cells(line: &Line<'_>) -> Option { let text = line.text.trim_matches([' ', '\t']); - if !text.contains('|') + // Without a pipe, a colon tells the delimiter row from the underline of a heading. + if !text.contains(['|', ':']) || !text.contains('-') || !text.bytes().all(|b| matches!(b, b'|' | b':' | b'-' | b' ' | b'\t')) { diff --git a/crates/mq-markdown/src/parser/tree.rs b/crates/mq-markdown/src/parser/tree.rs index 292d4f9b5..430e33313 100644 --- a/crates/mq-markdown/src/parser/tree.rs +++ b/crates/mq-markdown/src/parser/tree.rs @@ -76,6 +76,8 @@ impl InlineSource { pub(super) enum Block { /// A finished leaf such as a code block or a thematic break. Node(Node), + /// A fenced code or math block. + Fenced(FencedBlock), Inline(InlineBlock), Quote(QuoteBlock), List(ListBlock), @@ -87,6 +89,14 @@ pub(super) enum Block { Error(String), } +pub(super) struct FencedBlock { + pub(super) node: Node, + /// Whether it has a closing fence. + pub(super) closed: bool, + /// The number of lines of content, blank ones included. + pub(super) lines: usize, +} + pub(super) struct InlineBlock { pub(super) source: InlineSource, pub(super) kind: InlineKind, From 3f656bd720a4ccab50cf93cb38005240ac4349e8 Mon Sep 17 00:00:00 2001 From: harehare Date: Thu, 8 Oct 2026 13:05:18 +0900 Subject: [PATCH 04/88] =?UTF-8?q?=E2=9C=85=20test(markdown):=20add=20param?= =?UTF-8?q?eterized=20tests=20for=20the=20HTML=20output?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cover every construct, URLs and protocols, footnotes, line endings and the final line ending with the exact output that is expected. --- .../mq-markdown/tests/html_rendering_tests.rs | 376 ++++++++++++++++++ 1 file changed, 376 insertions(+) create mode 100644 crates/mq-markdown/tests/html_rendering_tests.rs diff --git a/crates/mq-markdown/tests/html_rendering_tests.rs b/crates/mq-markdown/tests/html_rendering_tests.rs new file mode 100644 index 000000000..d5a2e2198 --- /dev/null +++ b/crates/mq-markdown/tests/html_rendering_tests.rs @@ -0,0 +1,376 @@ +//! Parameterized tests for the HTML that `to_html` renders: every construct, the line endings, the +//! escaping, and the URLs. Each case lists the input and the exact output that is expected. + +use mq_markdown::{Markdown, to_html}; +use rstest::rstest; + +#[rstest] +#[case::paragraph("a\n", "

    a

    \n")] +#[case::paragraph_without_line_ending("a", "

    a

    ")] +#[case::two_paragraphs("a\n\nb\n", "

    a

    \n

    b

    \n")] +#[case::soft_break("a\nb\n", "

    a\nb

    \n")] +#[case::hard_break_spaces("a \nb\n", "

    a
    \nb

    \n")] +#[case::hard_break_backslash("a\\\nb\n", "

    a
    \nb

    \n")] +#[case::atx_heading("# a\n", "

    a

    \n")] +#[case::atx_closing_sequence("## a ##\n", "

    a

    \n")] +#[case::setext_one("a\n===\n", "

    a

    \n")] +#[case::setext_two("a\n---\n", "

    a

    \n")] +#[case::heading_with_inline("# a *b* `c`\n", "

    a b c

    \n")] +#[case::thematic_break("---\n", "
    \n")] +#[case::thematic_break_after_paragraph("a\n\n***\n", "

    a

    \n
    \n")] +#[case::empty("", "")] +#[case::blank_lines("\n\n \n", "")] +fn blocks(#[case] input: &str, #[case] expected: &str) { + assert_eq!(to_html(input), expected); +} + +#[rstest] +#[case::emphasis("*a*\n", "

    a

    \n")] +#[case::emphasis_underscore("_a_\n", "

    a

    \n")] +#[case::strong("**a**\n", "

    a

    \n")] +#[case::strong_emphasis("***a***\n", "

    a

    \n")] +#[case::strikethrough("~~a~~\n", "

    a

    \n")] +#[case::strikethrough_single_tilde("~a~\n", "

    a

    \n")] +#[case::code_span("`a`\n", "

    a

    \n")] +#[case::code_span_trim("` a `\n", "

    a

    \n")] +#[case::code_span_keeps_inner_spaces("`a b`\n", "

    a b

    \n")] +#[case::code_span_line_ending("`a\nb`\n", "

    a b

    \n")] +#[case::code_span_escapes_html("`
    &`\n", "

    <a>&

    \n")] +#[case::math_inline("$a$\n", "

    a

    \n")] +#[case::escape("\\*a\\*\n", "

    *a*

    \n")] +#[case::entity("© # A\n", "

    © # A

    \n")] +#[case::unknown_entity("&nope;\n", "

    &nope;

    \n")] +#[case::control_character_reference("�  \n", "

    \u{fffd} \u{fffd} \u{fffd}

    \n")] +#[case::special_characters("a < b > c & \"d\"\n", "

    a < b > c & "d"

    \n")] +#[case::nul("a\0b\n", "

    a\u{fffd}b

    \n")] +#[case::raw_html_inline("a c d\n", "

    a c d

    \n")] +#[case::raw_html_inline_multiline("x
    \n", "

    x

    \n")] +fn inline(#[case] input: &str, #[case] expected: &str) { + assert_eq!(to_html(input), expected); +} + +#[rstest] +#[case::link("[a](b)\n", "

    a

    \n")] +#[case::link_title("[a](b \"c\")\n", "

    a

    \n")] +#[case::link_title_escaped("[a](b \"<&>\")\n", "

    a

    \n")] +#[case::link_emphasis("[*a*](b)\n", "

    a

    \n")] +#[case::link_angle_destination("[a]()\n", "

    a

    \n")] +#[case::link_parentheses("[a](b(c))\n", "

    a

    \n")] +#[case::link_non_ascii( + "[a](http://あ.jp/パス)\n", + "

    a

    \n" +)] +#[case::link_keeps_percent_encoding("[a](b%20c)\n", "

    a

    \n")] +#[case::link_encodes_quote("[a](b\"c)\n", "

    a

    \n")] +#[case::link_ampersand("[a](b?c=1&d=2)\n", "

    a

    \n")] +#[case::link_entity("[a](b&c)\n", "

    a

    \n")] +#[case::link_empty("[a]()\n", "

    a

    \n")] +#[case::link_relative("[a](../b#c)\n", "

    a

    \n")] +#[case::link_http("[a](HTTP://b)\n", "

    a

    \n")] +#[case::link_mailto("[a](mailto:b@c)\n", "

    a

    \n")] +#[case::link_javascript_dropped("[a](javascript:b)\n", "

    a

    \n")] +#[case::link_javascript_mixed_case_dropped("[a](JaVaScRiPt:b)\n", "

    a

    \n")] +#[case::link_data_dropped("[a](data:text/html,b)\n", "

    a

    \n")] +#[case::link_colon_after_path_is_relative("[a](b/c:d)\n", "

    a

    \n")] +#[case::link_colon_after_query_is_relative("[a](b?c:d)\n", "

    a

    \n")] +#[case::image("![a](b)\n", "

    \"a\"

    \n")] +#[case::image_title("![a](b \"c\")\n", "

    \"a\"

    \n")] +#[case::image_alt_is_plain_text("![a *b* `c`](d)\n", "

    \"a

    \n")] +#[case::image_alt_escaped("![a\"b](c)\n", "

    \"a"b\"

    \n")] +#[case::image_data_dropped("![a](data:b)\n", "

    \"a\"

    \n")] +#[case::image_mailto_dropped("![a](mailto:b)\n", "

    \"a\"

    \n")] +#[case::reference("[a]\n\n[a]: /u \"t\"\n", "

    a

    \n")] +#[case::reference_full("[a][b]\n\n[b]: /u\n", "

    a

    \n")] +#[case::reference_collapsed("[a][]\n\n[a]: /u\n", "

    a

    \n")] +#[case::reference_case_insensitive("[A]\n\n[a]: /u\n", "

    A

    \n")] +#[case::reference_first_definition_wins("[a]\n\n[a]: /one\n[a]: /two\n", "

    a

    \n")] +#[case::reference_before_definition("[a]: /u\n\n[a]\n", "

    a

    \n")] +#[case::reference_undefined("[a]\n", "

    [a]

    \n")] +#[case::image_reference("![a]\n\n[a]: /u\n", "

    \"a\"

    \n")] +#[case::definition_alone("[a]: /u\n", "")] +#[case::autolink("\n", "

    http://a.b

    \n")] +#[case::autolink_email("\n", "

    a@b.c

    \n")] +#[case::autolink_escapes( + "\n", + "

    http://a.b/?c=1&d=2

    \n" +)] +#[case::autolink_literal_www("www.a.com\n", "

    www.a.com

    \n")] +#[case::autolink_literal_http( + "see http://a.com/b.\n", + "

    see http://a.com/b.

    \n" +)] +#[case::autolink_literal_email("a@b.com\n", "

    a@b.com

    \n")] +#[case::autolink_literal_in_link_is_text( + "[http://a.b](http://c.d)\n", + "

    http://a.b

    \n" +)] +#[case::autolink_literal_non_ascii( + "http://あ.jp/パス\n", + "

    http://あ.jp/パス

    \n" +)] +fn links_and_images(#[case] input: &str, #[case] expected: &str) { + assert_eq!(to_html(input), expected); +} + +#[rstest] +#[case::fenced("```\na\n```\n", "
    a\n
    \n")] +#[case::fenced_language("```rust\na\n```\n", "
    a\n
    \n")] +#[case::fenced_meta_is_dropped("```rust title=x\na\n```\n", "
    a\n
    \n")] +#[case::fenced_language_escaped("```a\"b\nc\n```\n", "
    c\n
    \n")] +#[case::fenced_empty("```\n```\n", "
    \n")] +#[case::fenced_blank_line("```\n\n```\n", "
    \n
    \n")] +#[case::fenced_blank_lines_kept("```\na\n\n\n```\n", "
    a\n\n\n
    \n")] +#[case::fenced_escapes_html("```\n&\n```\n", "
    <a>&\n
    \n")] +#[case::fenced_tilde("~~~\na\n~~~\n", "
    a\n
    \n")] +#[case::fenced_indent_removed(" ```\n a\n b\n ```\n", "
    a\n b\n
    \n")] +#[case::fenced_unclosed("```\na", "
    a\n
    \n")] +#[case::fenced_unclosed_empty("```\n", "
    \n")] +#[case::fenced_without_final_line_ending("```\na\n```", "
    a\n
    ")] +#[case::indented(" a\n", "
    a\n
    \n")] +#[case::indented_blank_line_inside(" a\n\n b\n", "
    a\n\nb\n
    \n")] +#[case::indented_trailing_blank_lines_dropped(" a\n\n\n", "
    a\n
    \n")] +#[case::math("$$\na\n$$\n", "
    a\n
    \n")] +#[case::html_block("
    \na\n
    \n", "
    \na\n
    \n")] +#[case::html_block_then_paragraph("
    \n\na\n", "
    \n

    a

    \n")] +#[case::html_comment("\n", "\n")] +#[case::frontmatter_is_dropped("---\na: b\n---\n# c\n", "

    c

    \n")] +#[case::toml_frontmatter_is_dropped("+++\na = 1\n+++\n# c\n", "

    c

    \n")] +fn code_and_html(#[case] input: &str, #[case] expected: &str) { + assert_eq!(to_html(input), expected); +} + +#[rstest] +#[case::quote("> a\n", "
    \n

    a

    \n
    \n")] +#[case::quote_lazy("> a\nb\n", "
    \n

    a\nb

    \n
    \n")] +#[case::quote_empty(">\n", "
    \n
    \n")] +#[case::quote_nested("> > a\n", "
    \n
    \n

    a

    \n
    \n
    \n")] +#[case::quote_with_heading_and_code( + "> # a\n> ```\n> b\n> ```\n", + "
    \n

    a

    \n
    b\n
    \n
    \n" +)] +#[case::quote_then_paragraph("> a\n\nb\n", "
    \n

    a

    \n
    \n

    b

    \n")] +#[case::quote_with_list("> - a\n", "
    \n
      \n
    • a
    • \n
    \n
    \n")] +#[case::unordered("- a\n- b\n", "
      \n
    • a
    • \n
    • b
    • \n
    \n")] +#[case::ordered("1. a\n2. b\n", "
      \n
    1. a
    2. \n
    3. b
    4. \n
    \n")] +#[case::ordered_start("3. a\n4. b\n", "
      \n
    1. a
    2. \n
    3. b
    4. \n
    \n")] +#[case::ordered_start_zero("0. a\n", "
      \n
    1. a
    2. \n
    \n")] +#[case::ordered_paren("1) a\n", "
      \n
    1. a
    2. \n
    \n")] +#[case::list_loose("- a\n\n- b\n", "
      \n
    • \n

      a

      \n
    • \n
    • \n

      b

      \n
    • \n
    \n")] +#[case::list_loose_item_with_two_blocks( + "- a\n\n b\n- c\n", + "
      \n
    • \n

      a

      \n

      b

      \n
    • \n
    • \n

      c

      \n
    • \n
    \n" +)] +#[case::list_tight_with_nested("- a\n - b\n", "
      \n
    • a\n
        \n
      • b
      • \n
      \n
    • \n
    \n")] +#[case::list_tight_with_code( + "- a\n ```\n b\n ```\n", + "
      \n
    • a\n
      b\n
      \n
    • \n
    \n" +)] +#[case::list_tight_with_heading("- # a\n- b\n", "
      \n
    • \n

      a

      \n
    • \n
    • b
    • \n
    \n")] +#[case::list_empty_item("-\n- a\n", "
      \n
    • \n
    • a
    • \n
    \n")] +#[case::list_changed_marker_is_new_list("- a\n* b\n", "
      \n
    • a
    • \n
    \n
      \n
    • b
    • \n
    \n")] +#[case::list_after_paragraph("a\n- b\n", "

    a

    \n
      \n
    • b
    • \n
    \n")] +#[case::list_blank_lines_after_are_not_loose("- a\n\nb\n", "
      \n
    • a
    • \n
    \n

    b

    \n")] +#[case::list_quote_in_item("- > a\n", "
      \n
    • \n
      \n

      a

      \n
      \n
    • \n
    \n")] +#[case::task_unchecked("- [ ] a\n", "
      \n
    • a
    • \n
    \n")] +#[case::task_checked( + "- [x] a\n", + "
      \n
    • a
    • \n
    \n" +)] +#[case::task_loose( + "- [ ] a\n\n- b\n", + "
      \n
    • \n

      a

      \n
    • \n
    • \n

      b

      \n
    • \n
    \n" +)] +fn containers(#[case] input: &str, #[case] expected: &str) { + assert_eq!(to_html(input), expected); +} + +#[rstest] +#[case::table( + "| a | b |\n|---|---|\n| c | d |\n", + "\n\n\n\n\n\n\n\n\n\n\n\n\n
    ab
    cd
    \n" +)] +#[case::table_alignment( + "| a | b | c |\n|:--|:-:|--:|\n| d | e | f |\n", + "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n
    abc
    def
    \n" +)] +#[case::table_without_body("| a |\n|---|\n", "\n\n\n\n\n\n
    a
    \n")] +#[case::table_short_row_is_filled( + "| a | b |\n|---|---|\n| c |\n", + "\n\n\n\n\n\n\n\n\n\n\n\n\n
    ab
    c
    \n" +)] +#[case::table_long_row_is_cut( + "| a |\n|---|\n| b | c |\n", + "\n\n\n\n\n\n\n\n\n\n\n
    a
    b
    \n" +)] +#[case::table_without_outer_pipes( + "a | b\n-|-\nc | d\n", + "\n\n\n\n\n\n\n\n\n\n\n\n\n
    ab
    cd
    \n" +)] +#[case::table_inline_content( + "| *a* | `b` |\n|---|---|\n| [c](d) | ~~e~~ |\n", + "\n\n\n\n\n\n\n\n\n\n\n\n\n
    ab
    ce
    \n" +)] +#[case::table_escaped_pipe( + "| a \\| b |\n|---|\n", + "\n\n\n\n\n\n
    a | b
    \n" +)] +#[case::table_ended_by_blank_line( + "| a |\n|---|\n| b |\n\nc\n", + "\n\n\n\n\n\n\n\n\n\n\n
    a
    b
    \n

    c

    \n" +)] +#[case::table_multibyte( + "| あ | 🎉 |\n|---|---|\n| é | ß |\n", + "\n\n\n\n\n\n\n\n\n\n\n\n\n
    あ🎉
    éß
    \n" +)] +fn tables(#[case] input: &str, #[case] expected: &str) { + assert_eq!(to_html(input), expected); +} + +const NOTE_OPEN: &str = "

    Footnotes

    \n
      \n"; + +fn call(id: &str, fnref: &str, number: usize) -> String { + format!( + "{number}" + ) +} + +fn back(fnref: &str, sup: Option) -> String { + let sup = sup.map(|n| format!("{n}")).unwrap_or_default(); + format!( + "\u{21a9}{sup}" + ) +} + +#[test] +fn footnote() { + let expected = format!( + "

      a{}

      \n{NOTE_OPEN}
    1. \n

      b {}

      \n
    2. \n
    \n
    \n", + call("1", "1", 1), + back("1", None) + ); + assert_eq!(to_html("a[^1]\n\n[^1]: b\n"), expected); +} + +#[test] +fn footnote_referenced_twice_has_two_backreferences() { + let expected = format!( + "

    a{} b{}

    \n{NOTE_OPEN}
  • \n

    c {} {}

    \n
  • \n\n\n", + call("1", "1", 1), + call("1", "1-2", 1), + back("1", None), + back("1-2", Some(2)) + ); + assert_eq!(to_html("a[^1] b[^1]\n\n[^1]: c\n"), expected); +} + +#[test] +fn footnotes_are_numbered_in_the_order_of_their_references() { + let html = to_html("a[^b] c[^a]\n\n[^a]: x\n[^b]: y\n"); + let first = html.find("
  • ").unwrap(); + let second = html.find("
  • ").unwrap(); + assert!(first < second, "{html}"); + assert!( + html.contains(&call("b", "b", 1)) && html.contains(&call("a", "a", 2)), + "{html}" + ); +} + +#[test] +fn footnote_label_is_case_insensitive_and_its_id_is_lowercase() { + let html = to_html("a[^Ab]\n\n[^ab]: x\n"); + assert!(html.contains("href=\"#user-content-fn-ab\""), "{html}"); +} + +#[test] +fn footnote_id_is_percent_encoded() { + let html = to_html("a[^é]\n\n[^é]: x\n"); + assert!(html.contains("user-content-fn-%C3%A9"), "{html}"); +} + +#[test] +fn footnote_without_definition_is_text() { + assert_eq!(to_html("a[^1]\n"), "

    a[^1]

    \n"); +} + +#[test] +fn footnote_without_reference_is_not_rendered() { + assert_eq!(to_html("[^1]: a\n"), ""); +} + +#[test] +fn footnote_with_a_list_puts_the_backreference_on_its_own_line() { + let expected = format!( + "

    a{}

    \n{NOTE_OPEN}
  • \n
      \n
    • l
    • \n
    \n{}\n
  • \n\n\n", + call("1", "1", 1), + back("1", None) + ); + assert_eq!(to_html("a[^1]\n\n[^1]:\n - l\n"), expected); +} + +#[test] +fn footnote_with_two_paragraphs_has_the_backreference_in_the_last() { + let html = to_html("a[^1]\n\n[^1]: p\n\n q\n"); + assert!( + html.contains(&format!("

    p

    \n

    q {}

    ", back("1", None))), + "{html}" + ); +} + +#[test] +fn footnote_defined_before_its_reference_is_still_rendered() { + let html = to_html("[^1]: a\n\nb[^1]\n"); + assert!(html.starts_with("

    b"), "{html}"); + assert!(html.contains("

  • "), "{html}"); +} + +#[test] +fn footnote_reference_inside_a_footnote_is_numbered_when_it_is_met() { + let html = to_html("[^1]: x [^2]\n[^2]: y\n\nz[^1]\n"); + // `[^2]` is met first, in the definition of `[^1]`. + assert!(html.contains(&call("2", "2", 1)), "{html}"); + assert!(html.contains(&call("1", "1", 2)), "{html}"); +} + +#[rstest] +#[case::lf("a\n\nb\n")] +#[case::list("- a\n- b\n")] +#[case::quote("> a\n> b\n")] +#[case::code("```\na\nb\n```\n")] +#[case::table("| a |\n|---|\n| b |\n")] +#[case::footnote("a[^1]\n\n[^1]: b\n")] +#[case::hard_break("a \nb\n")] +fn crlf_documents_render_with_crlf(#[case] input: &str) { + let crlf = input.replace('\n', "\r\n"); + assert_eq!(to_html(&crlf), to_html(input).replace('\n', "\r\n")); +} + +#[rstest] +#[case::no_final_line_ending("a", false)] +#[case::final_line_ending("a\n", true)] +#[case::heading("# a", false)] +#[case::heading_final_line_ending("# a\n", true)] +#[case::closed_fence("```\na\n```", false)] +#[case::closed_fence_final_line_ending("```\na\n```\n", true)] +#[case::list("- a", false)] +#[case::list_final_line_ending("- a\n", true)] +#[case::table("| a |\n|---|", false)] +#[case::table_final_line_ending("| a |\n|---|\n", true)] +#[case::blank_lines_after("a\n\n\n", true)] +fn trailing_line_ending_follows_the_source(#[case] input: &str, #[case] ends_with_line_ending: bool) { + assert_eq!( + to_html(input).ends_with('\n'), + ends_with_line_ending, + "{:?}", + to_html(input) + ); +} + +#[rstest] +#[case::paragraph("a *b*\n\n- c\n")] +#[case::link("[a](b \"c\")\n")] +#[case::table("| a | b |\n|:-|-:|\n| c | d |\n")] +#[case::footnote("a[^1]\n\n[^1]: b\n")] +fn to_html_of_a_document_equals_to_html_of_its_source(#[case] input: &str) { + let document: Markdown = input.parse().unwrap(); + assert_eq!(document.to_html(), to_html(input)); +} From 699e6ff8fd7e7c20cd10efebabfac3d0d515a514 Mon Sep 17 00:00:00 2001 From: harehare Date: Thu, 8 Oct 2026 13:05:24 +0900 Subject: [PATCH 05/88] =?UTF-8?q?=E2=9C=85=20test(markdown):=20add=20param?= =?UTF-8?q?eterized=20tests=20for=20nodes=20and=20positions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Check kinds, nesting, attributes and positions, including columns that count bytes of multibyte characters, line endings and a byte order mark. --- .../tests/parser_structure_tests.rs | 258 ++++++++++++++++++ 1 file changed, 258 insertions(+) create mode 100644 crates/mq-markdown/tests/parser_structure_tests.rs diff --git a/crates/mq-markdown/tests/parser_structure_tests.rs b/crates/mq-markdown/tests/parser_structure_tests.rs new file mode 100644 index 000000000..703d3160a --- /dev/null +++ b/crates/mq-markdown/tests/parser_structure_tests.rs @@ -0,0 +1,258 @@ +//! Parameterized tests for the nodes that the parser builds: their kinds and nesting, their +//! attributes, and where they are. Positions count lines from 1 and columns in bytes from 1, so the +//! multibyte cases show what a column is. + +use mq_markdown::{AttrValue, Markdown, Node}; +use rstest::rstest; + +fn parse(input: &str) -> Vec { + input.parse::().unwrap().nodes +} + +/// The kinds of the nodes with their nesting, and the value of the leaves. +fn outline(nodes: &[Node]) -> String { + fn one(node: &Node) -> String { + let name = node.name(); + let children = node.children(); + if children.is_empty() { + format!("{name}\"{}\"", node.value().replace('\n', "\\n")) + } else { + format!("{name}({})", children.iter().map(one).collect::>().join(",")) + } + } + nodes.iter().map(one).collect::>().join(" | ") +} + +#[rstest] +#[case::paragraph("a", r#"text"a""#)] +#[case::paragraph_lines("a\nb", r#"text"a\nb""#)] +#[case::two_paragraphs("a\n\nb", r#"text"a" | text"b""#)] +#[case::heading("# a *b*", r#"h1(text"a ",emphasis(text"b"))"#)] +#[case::heading_depths("###### a", r#"h6(text"a")"#)] +#[case::setext("a\n===", r#"h1(text"a")"#)] +#[case::strong_in_emphasis("*a **b***", r#"emphasis(text"a ",strong(text"b"))"#)] +#[case::strikethrough("~~a~~", r#"delete(text"a")"#)] +#[case::code_span("`a`", r#"code_inline"a""#)] +#[case::math_inline("$a$", r#"math_inline"a""#)] +#[case::hard_break("a \nb", r#"text"a" | break"" | text"b""#)] +#[case::inline_html("x", r#"html"" | text"x" | html"""#)] +#[case::html_block("
    \nx\n
    ", r#"html"
    \nx\n
    ""#)] +#[case::link("[a *b*](c)", r#"link(text"a ",emphasis(text"b"))"#)] +#[case::link_reference("[a][b]\n\n[b]: /u", r#"link_ref"b" | definition"/u""#)] +#[case::image("![a](b)", r#"image"b""#)] +#[case::autolink("", r#"link(text"http://a.b")"#)] +#[case::autolink_literal("see www.a.com.", r#"text"see " | link(text"www.a.com") | text".""#)] +#[case::quote("> a\n> b", r#"blockquote(text"a\nb")"#)] +#[case::quote_nested("> > a", r#"blockquote(blockquote(text"a"))"#)] +#[case::quote_with_blocks("> # a\n> b", r#"blockquote(h1(text"a"),text"b")"#)] +#[case::code("```rs\nx\n```", r#"code"x""#)] +#[case::indented_code(" x", r#"code"x""#)] +#[case::math("$$\nx\n$$", r#"math"x""#)] +#[case::horizontal_rule("***", r#"Horizontal_rule"""#)] +#[case::frontmatter("---\na: 1\n---\n# b", r#"yaml"a: 1" | h1(text"b")"#)] +#[case::toml_frontmatter("+++\na = 1\n+++", r#"toml"a = 1""#)] +#[case::footnote("a[^1]\n\n[^1]: b", r#"text"a" | footnoteref"1" | footnote(text"b")"#)] +#[case::footnote_without_definition("a[^1]", r#"text"a[^1]""#)] +#[case::list_is_flat("- a\n - b\n- c", r#"list(text"a") | list(text"b") | list(text"c")"#)] +#[case::list_item_with_blocks("- a\n\n b", r#"list(text"a",text"b")"#)] +#[case::table( + "| a | b |\n|:-|-:|\n| c |", + r#"table_cell(text"a") | table_cell(text"b") | table_align"" | table_cell(text"c")"# +)] +#[case::definition_only("[a]: /u", r#"definition"/u""#)] +#[case::empty("", "")] +#[case::blank_lines("\n\n \n", "")] +fn nodes(#[case] input: &str, #[case] expected: &str) { + assert_eq!(outline(&parse(input)), expected); +} + +fn attr(node: &Node, name: &str) -> Option { + node.attr(name) +} + +/// The attributes of the last item. +#[rstest] +#[case::top("- a", 0, false, None)] +#[case::second_item("- a\n- b", 0, false, None)] +#[case::nested("- a\n - b", 1, false, None)] +#[case::nested_twice("- a\n - b\n - c", 2, false, None)] +#[case::ordered("1. a", 0, true, None)] +#[case::checked("- [x] a", 0, false, Some(true))] +#[case::unchecked("- [ ] a", 0, false, Some(false))] +fn list_items(#[case] input: &str, #[case] level: i64, #[case] ordered: bool, #[case] checked: Option) { + let nodes = parse(input); + let item = nodes.last().unwrap(); + assert_eq!(attr(item, "level"), Some(AttrValue::Integer(level))); + assert_eq!(attr(item, "ordered"), Some(AttrValue::Boolean(ordered))); + assert_eq!(attr(item, "checked"), checked.map(AttrValue::Boolean)); +} + +#[rstest] +#[case::index_first("- a\n- b\n- c", 0, 0)] +#[case::index_second("- a\n- b\n- c", 1, 1)] +#[case::index_third("- a\n- b\n- c", 2, 2)] +fn list_item_index(#[case] input: &str, #[case] item: usize, #[case] index: i64) { + assert_eq!(attr(&parse(input)[item], "index"), Some(AttrValue::Integer(index))); +} + +#[rstest] +#[case::link("[a](b \"c\")", "url", "b")] +#[case::link_title("[a](b \"c\")", "title", "c")] +#[case::link_entity("[a](b&c)", "url", "b&c")] +#[case::link_escape("[a](b\\)c)", "url", "b)c")] +#[case::link_angle("[a]()", "url", "b c")] +#[case::autolink("", "url", "http://a.b")] +#[case::autolink_email("", "url", "mailto:a@b.c")] +#[case::autolink_literal_www("www.a.com", "url", "http://www.a.com")] +#[case::image_url("![a *b*](c)", "url", "c")] +#[case::image_alt("![a *b*](c)", "alt", "a b")] +#[case::image_title("![a](b 'c')", "title", "c")] +#[case::code_lang("```rs x\ny\n```", "lang", "rs")] +#[case::code_meta("```rs x\ny\n```", "meta", "x")] +#[case::definition_url("[a]: \"t\"", "url", "b")] +#[case::definition_title("[a]: b \"t\"", "title", "t")] +#[case::definition_ident("[ A b ]: c", "ident", "a b")] +#[case::footnote_ident("a[^B]\n\n[^b]: c", "ident", "b")] +fn attributes(#[case] input: &str, #[case] name: &str, #[case] expected: &str) { + let nodes = parse(input); + let found = nodes + .iter() + .flat_map(|node| std::iter::once(node.clone()).chain(node.children())) + .find_map(|node| node.attr(name)) + .unwrap_or_else(|| panic!("no `{name}` in {input:?}")); + assert_eq!(found, AttrValue::String(expected.to_string())); +} + +/// `(input, index of the node, start line, start column, end line, end column)`. +#[rstest] +#[case::paragraph("a", 0, (1, 1), (1, 2))] +#[case::paragraph_second_line("a\nbc", 0, (1, 1), (2, 3))] +#[case::second_paragraph("a\n\nb", 1, (3, 1), (3, 2))] +#[case::heading("## a", 0, (1, 1), (1, 5))] +#[case::quote("> a", 0, (1, 1), (1, 4))] +#[case::quote_second_line("> a\n> b", 0, (1, 1), (2, 4))] +#[case::code("```\nx\n```", 0, (1, 1), (3, 4))] +#[case::code_indented_fence(" ```\n x\n ```", 0, (1, 3), (3, 6))] +// The position of an item is the one of its content. +#[case::list_item("- a", 0, (1, 3), (1, 4))] +#[case::emphasis_position("a *b*", 1, (1, 3), (1, 6))] +#[case::crlf("a\r\nb", 0, (1, 1), (2, 2))] +#[case::cr("a\rb", 0, (1, 1), (2, 2))] +#[case::second_line_after_crlf("a\r\n\r\nb", 1, (3, 1), (3, 2))] +#[case::multibyte_paragraph("あ", 0, (1, 1), (1, 4))] +#[case::multibyte_second_node("あ\n\nい", 1, (3, 1), (3, 4))] +#[case::multibyte_emphasis("あ*い*う", 1, (1, 4), (1, 9))] +#[case::multibyte_link("あ[い](u)", 1, (1, 4), (1, 12))] +#[case::emoji("🎉 *a*", 1, (1, 6), (1, 9))] +#[case::combining_mark("e\u{301}*a*", 1, (1, 4), (1, 7))] +#[case::multibyte_heading("# 見出し", 0, (1, 1), (1, 12))] +#[case::multibyte_quote("> あ", 0, (1, 1), (1, 6))] +#[case::multibyte_list("- あ", 0, (1, 3), (1, 6))] +#[case::multibyte_code("```\nあ\n```", 0, (1, 1), (3, 4))] +#[case::multibyte_fence_info("```あ\nx\n```", 0, (1, 1), (3, 4))] +#[case::ideographic_space("あ\u{3000}い", 0, (1, 1), (1, 10))] +#[case::table_cell("| あ | い |\n|---|---|", 0, (1, 1), (1, 7))] +#[case::bom("\u{feff}# a", 0, (1, 4), (1, 7))] +#[case::after_footnote("a[^1]\n\n[^1]: b", 1, (1, 2), (1, 6))] +fn positions(#[case] input: &str, #[case] index: usize, #[case] start: (usize, usize), #[case] end: (usize, usize)) { + let nodes = parse(input); + let position = nodes[index] + .position() + .unwrap_or_else(|| panic!("no position in {nodes:?}")); + assert_eq!( + ( + (position.start.line, position.start.column), + (position.end.line, position.end.column) + ), + (start, end), + "{:?}", + nodes[index] + ); +} + +#[rstest] +#[case::trailing_space("| a | b | \n|-|-| \n| c | d | \n")] +#[case::no_outer_pipes("a | b\n--|--\nd | e \n")] +fn table_rows_include_trailing_whitespace(#[case] input: &str) { + for node in parse(input) { + let position = node.position().unwrap(); + let line = input.lines().nth(position.end.line - 1).unwrap(); + assert!(position.end.column <= line.len() + 1, "{node:?}"); + } + let last_cell = parse(input) + .into_iter() + .rfind(|node| node.name() == "table_cell") + .unwrap(); + let end = last_cell.position().unwrap().end; + assert_eq!( + end.column, + input.lines().nth(end.line - 1).unwrap().len() + 1, + "{last_cell:?}" + ); +} + +#[rstest] +#[case::bom_heading("\u{feff}# a", "h1")] +#[case::bom_frontmatter("\u{feff}---\na: b\n---", "yaml")] +#[case::bom_text("\u{feff}a", "text")] +fn byte_order_mark_is_not_content(#[case] input: &str, #[case] first: &str) { + let nodes = parse(input); + assert_eq!(nodes[0].name(), first); + assert!(!outline(&nodes).contains('\u{feff}')); +} + +#[rstest] +#[case::wwß("wwß x")] +#[case::www_combining("ww\u{301}.a.com")] +#[case::http_multibyte("http://あ.jp/パス ok")] +#[case::email_multibyte("あ@い.com")] +#[case::www_multibyte("www.あ.com/パス")] +#[case::www_emoji("www.🎉.com")] +#[case::emphasis_multibyte("*あ*。**い**です")] +#[case::underscore_accent("_é_ _ß_")] +#[case::fullwidth_markers("*a* _b_")] +#[case::table_multibyte("| あ | 🎉 |\n|-|-|\n| é | ß |")] +#[case::reference_multibyte("[あ]\n\n[あ]: /u")] +#[case::footnote_multibyte("あ[^い]\n\n[^い]: う")] +fn multibyte_input_parses(#[case] input: &str) { + let nodes = parse(input); + assert!(!nodes.is_empty()); + for node in &nodes { + let position = node.position().unwrap(); + assert!(position.start.line >= 1 && position.start.column >= 1, "{node:?}"); + } +} + +#[rstest] +#[case::emphasis_cjk("*あ*。", r#"emphasis(text"あ") | text"。""#)] +#[case::strong_cjk("**日本語**です", r#"strong(text"日本語") | text"です""#)] +#[case::emphasis_between_cjk("あ*い*う", r#"text"あ" | emphasis(text"い") | text"う""#)] +#[case::underscore_intraword_cjk("あ_い_う", r#"text"あ_い_う""#)] +#[case::emphasis_next_to_cjk_punctuation("。*a*、", r#"text"。" | emphasis(text"a") | text"、""#)] +#[case::emphasis_next_to_ideographic_space( + "\u{3000}*a*\u{3000}", + "text\"\u{3000}\" | emphasis(text\"a\") | text\"\u{3000}\"" +)] +#[case::emphasis_next_to_nbsp("a\u{a0}*b*\u{a0}c", "text\"a\u{a0}\" | emphasis(text\"b\") | text\"\u{a0}c\"")] +#[case::code_span_multibyte("`あ`と``い`う``", r#"code_inline"あ" | text"と" | code_inline"い`う""#)] +#[case::hard_break_multibyte("あ \nい", r#"text"あ" | break"" | text"い""#)] +#[case::entity_multibyte("あ&いあ", r#"text"あ&いあ""#)] +#[case::emoji_zwj( + "👨\u{200d}👩\u{200d}👧 *a*", + "text\"👨\u{200d}👩\u{200d}👧 \" | emphasis(text\"a\")" +)] +fn multibyte_inline_syntax(#[case] input: &str, #[case] expected: &str) { + assert_eq!(outline(&parse(input)), expected); +} + +#[rstest] +#[case::paragraph_then_paragraph("a\n\nb", 2)] +#[case::heading_then_paragraph("# a\nb", 2)] +#[case::definition_between("a\n\n[b]: c\n\nd", 2)] +fn blocks_are_separate_nodes(#[case] input: &str, #[case] count: usize) { + let nodes = parse(input) + .into_iter() + .filter(|node| node.name() != "definition") + .count(); + assert_eq!(nodes, count); +} From fc39f416214d92a334b8bf239be2a4a144dd9e4d Mon Sep 17 00:00:00 2001 From: harehare Date: Thu, 8 Oct 2026 13:05:31 +0900 Subject: [PATCH 06/88] =?UTF-8?q?=E2=9C=85=20test(markdown):=20add=20prope?= =?UTF-8?q?rty-based=20tests=20for=20the=20parser=20and=20HTML?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit State what holds for any document: blocks parse as their parts, CRLF and containers change nothing else, other characters in place of letters keep the structure, positions are on character boundaries, and the HTML is well-formed with safe URLs. --- .../tests/parser_property_tests.rs | 565 ++++++++++++++++++ 1 file changed, 565 insertions(+) create mode 100644 crates/mq-markdown/tests/parser_property_tests.rs diff --git a/crates/mq-markdown/tests/parser_property_tests.rs b/crates/mq-markdown/tests/parser_property_tests.rs new file mode 100644 index 000000000..ca605c55b --- /dev/null +++ b/crates/mq-markdown/tests/parser_property_tests.rs @@ -0,0 +1,565 @@ +//! Property-based tests for the parser and the HTML renderer. They state what has to hold whatever the +//! document is, instead of what one output looks like: documents that are put together from blocks +//! parse as their blocks, line endings and containers change nothing else, other characters in place of +//! letters do not change the structure, every position is on a character boundary, and the HTML is +//! well-formed with safe URLs. + +use mq_markdown::{Markdown, Node, to_html}; +use proptest::prelude::*; + +fn parse(input: &str) -> Vec { + input.parse::().unwrap().nodes +} + +/// The kinds of the nodes with their nesting, without any value. +fn shape(nodes: &[Node]) -> String { + fn one(node: &Node) -> String { + let children = node.children(); + if children.is_empty() { + node.name().to_string() + } else { + format!( + "{}({})", + node.name(), + children.iter().map(one).collect::>().join(",") + ) + } + } + nodes.iter().map(one).collect::>().join(" ") +} + +/// Blocks that mean the same wherever they are and never run into the ones around them. +const BLOCKS: &[&str] = &[ + "a", + "a\nb", + "a \nb", + "a\\\nb", + "# h", + "## h ##", + "a\n===", + "a\n---", + "***", + "> q", + "> a\n> b", + "> > n", + "```\ncode\n```", + "```rust x\ncode\n\ncode\n```", + "~~~\n~\n~~~", + " indented", + "$$\nm\n$$", + "
    \nx\n
    ", + "", + "| a | b |\n|:-|-:|\n| c | d |", + "a | b\n-|-\nc | d", + "*e* **s** `c` ~~d~~ $m$ [l](u \"t\") ![i](u) www.a.com a@b.com & \\*", + "[r]\n\n[r]: /u", + "あい\nう", + "# 見出し", + "> 引用\n> つづき", + "```あ\n日本語\n```", + "| あ | 🎉 |\n|-|-|\n| é | ß |", + "*あ*。**い**です", + "[あ](http://い.jp/う)", + "http://あ.jp/パス あ@い.com www.う.com", + "👨\u{200d}👩\u{200d}👧 e\u{301} ß Dž", +]; + +fn block() -> impl Strategy { + prop::sample::select(BLOCKS) +} + +/// Blocks put together with blank lines. +fn document() -> impl Strategy> { + prop::collection::vec(block(), 1..6) +} + +/// Blocks without a footnote or a definition, whose HTML is only the HTML of its blocks. +fn plain_block() -> impl Strategy { + // Indented code that follows indented code is one block. + block().prop_filter("definition or indented code", |block| { + !block.contains("]:") && !block.starts_with(" ") + }) +} + +/// What counts for a position: the columns are bytes, the lines end at `\n`, `\r\n` or `\r`. +fn byte_offset(input: &str, line: usize, column: usize) -> Option { + let bytes = input.as_bytes(); + let mut start = 0; + let mut current = 1; + let mut index = 0; + while current < line { + match bytes.get(index)? { + b'\r' if bytes.get(index + 1) == Some(&b'\n') => index += 2, + b'\n' | b'\r' => index += 1, + _ => { + index += 1; + continue; + } + } + current += 1; + start = index; + } + Some(start + column - 1) +} + +fn walk(nodes: Vec) -> Vec { + let mut all = Vec::new(); + let mut stack = nodes; + while let Some(node) = stack.pop() { + stack.extend(node.children()); + all.push(node); + } + all +} + +proptest! { + #[test] + fn blocks_separated_by_blank_lines_parse_as_their_blocks(blocks in prop::collection::vec(plain_block(), 1..6)) { + let joined = parse(&blocks.join("\n\n")); + let separate = blocks.iter().flat_map(|block| parse(block)).collect::>(); + prop_assert_eq!(shape(&joined), shape(&separate)); + prop_assert_eq!( + joined.iter().map(Node::value).collect::>(), + separate.iter().map(Node::value).collect::>() + ); + } + + #[test] + fn html_of_blocks_separated_by_blank_lines_is_the_html_of_the_blocks( + blocks in prop::collection::vec(plain_block(), 1..6) + ) { + let joined = to_html(&format!("{}\n", blocks.join("\n\n"))); + let separate = blocks.iter().map(|block| to_html(&format!("{block}\n"))).collect::(); + prop_assert_eq!(joined, separate); + } + + #[test] + fn lines_of_a_block_are_after_the_lines_of_the_blocks_before_it(blocks in document()) { + let source = blocks.join("\n\n"); + let mut last = 0; + for node in parse(&source) { + let line = node.position().map_or(last, |position| position.start.line); + prop_assert!(line >= last, "{source:?}"); + last = line; + } + } + + #[test] + fn crlf_documents_have_the_nodes_of_lf_documents(blocks in document()) { + let lf = blocks.join("\n\n") + "\n"; + let crlf = lf.replace('\n', "\r\n"); + prop_assert_eq!(shape(&parse(&crlf)), shape(&parse(&lf))); + let (lf_nodes, crlf_nodes) = (parse(&lf), parse(&crlf)); + for (a, b) in lf_nodes.iter().zip(&crlf_nodes) { + prop_assert_eq!(a.position().map(|p| (p.start.line, p.end.line)), b.position().map(|p| (p.start.line, p.end.line))); + } + } + + #[test] + fn crlf_documents_have_the_html_of_lf_documents_with_crlf(blocks in document()) { + let lf = blocks.join("\n\n") + "\n"; + let crlf = lf.replace('\n', "\r\n"); + prop_assert_eq!(to_html(&crlf), to_html(&lf).replace('\n', "\r\n")); + } + + #[test] + fn cr_documents_have_the_nodes_of_lf_documents(blocks in document()) { + let lf = blocks.join("\n\n") + "\n"; + let cr = lf.replace('\n', "\r"); + prop_assert_eq!(shape(&parse(&cr)), shape(&parse(&lf))); + } + + #[test] + fn a_block_quote_wraps_the_html_of_its_content(blocks in prop::collection::vec(plain_block(), 1..5)) { + let content = blocks.join("\n\n") + "\n"; + let quoted = content + .lines() + .map(|line| if line.is_empty() { ">".to_string() } else { format!("> {line}") }) + .collect::>() + .join("\n") + + "\n"; + let html = to_html(&content); + prop_assert_eq!(to_html("ed), format!("
    \n{html}
    \n"), "{:?}", quoted); + } + + #[test] + fn a_final_line_ending_changes_nothing_but_the_end_of_the_html(blocks in document()) { + let without = blocks.join("\n\n"); + let with = format!("{without}\n"); + prop_assert_eq!(shape(&parse(&without)), shape(&parse(&with))); + let (a, b) = (to_html(&without), to_html(&with)); + prop_assert_eq!(a.trim_end(), b.trim_end()); + } + + #[test] + fn positions_are_on_character_boundaries(blocks in document()) { + let source = blocks.join("\n\n") + "\n"; + for node in walk(parse(&source)) { + let Some(position) = node.position() else { continue }; + let start = byte_offset(&source, position.start.line, position.start.column); + let end = byte_offset(&source, position.end.line, position.end.column); + prop_assert!(start.is_some() && end.is_some(), "{node:?}"); + let (start, end) = (start.unwrap(), end.unwrap()); + prop_assert!(start <= end && end <= source.len(), "{node:?} in {source:?}"); + prop_assert!( + source.is_char_boundary(start) && source.is_char_boundary(end), + "{node:?} is not on character boundaries in {source:?}" + ); + } + } + + #[test] + fn every_prefix_of_a_document_parses(blocks in document()) { + let source = blocks.join("\n\n") + "\n"; + for end in (0..=source.len()).filter(|end| source.is_char_boundary(*end)) { + let prefix = &source[..end]; + parse(prefix); + to_html(prefix); + } + } + + #[test] + fn every_suffix_of_a_document_parses(blocks in document()) { + let source = blocks.join("\n\n") + "\n"; + for start in (0..=source.len()).filter(|start| source.is_char_boundary(*start)) { + let suffix = &source[start..]; + parse(suffix); + to_html(suffix); + } + } + + #[test] + fn rendering_a_document_and_parsing_it_again_keeps_its_shape(blocks in document()) { + let source = blocks.join("\n\n") + "\n"; + let once = source.parse::().unwrap().to_string(); + let twice = once.parse::().unwrap().to_string(); + prop_assert_eq!(once, twice, "{:?}", source); + } + + #[test] + fn html_of_a_rendered_document_is_the_html_of_the_document(blocks in prop::collection::vec(plain_block(), 1..5)) { + let source = blocks.join("\n\n") + "\n"; + let rendered = source.parse::().unwrap().to_string(); + prop_assert_eq!(to_html(&rendered), to_html(&source), "{:?}", rendered); + } +} + +/// Words of letters, and what to put in their place. +fn words() -> impl Strategy> { + prop::collection::vec("[a-f]{1,4}", 1..5) +} + +fn replace_letters(text: &str, replacements: &[char; 6]) -> String { + text.chars() + .map(|char| match char { + 'a'..='f' => replacements[(char as u8 - b'a') as usize], + _ => char, + }) + .collect() +} + +/// Markup around words: the same markup with other characters for the letters has the same structure. +fn markup() -> impl Strategy { + let wrap = prop::sample::select(vec![ + "{}", + "*{}*", + "**{}**", + "_{}_", + "~~{}~~", + "`{}`", + "[{}](u)", + "![{}](u)", + "# {}", + "> {}", + "- {}", + "1. {}", + "{} \n{}", + "| {} |\n|-|\n| {} |", + "{}[^n]\n\n[^n]: {}", + "{}", + "`` {} ``", + "$ {} $", + "[{}][r]\n\n[r]: /u", + ]); + (words(), wrap).prop_map(|(words, wrap)| { + let mut words = words.into_iter().cycle(); + let mut result = String::new(); + let mut rest = wrap; + while let Some(index) = rest.find("{}") { + result.push_str(&rest[..index]); + result.push_str(&words.next().unwrap()); + rest = &rest[index + 2..]; + } + result.push_str(rest); + result + }) +} + +proptest! { + #[test] + fn other_characters_in_place_of_letters_keep_the_structure( + source in markup(), + replacements in prop::sample::select(vec![ + ['あ', 'い', 'う', 'え', 'お', 'か'], + ['é', 'ß', 'ñ', 'ø', 'ü', 'ç'], + ['🎉', '🚀', '🔥', '✨', '🌟', '💡'], + ['日', '本', '語', '文', '字', '列'], + ['a', 'b', 'c', 'd', 'e', 'f'], + ['א', 'ب', 'क', 'ก', '한', 'ა'], + ]) + ) { + let replaced = replace_letters(&source, &replacements); + prop_assert_eq!(shape(&parse(&replaced)), shape(&parse(&source)), "{:?} -> {:?}", source, replaced); + // The tags of the HTML are the same too. + prop_assert_eq!(tags(&to_html(&replaced)), tags(&to_html(&source)), "{:?} -> {:?}", source, replaced); + } + + #[test] + fn columns_count_bytes_of_the_characters_before( + source in markup(), + replacement in prop::sample::select(vec!['あ', 'é', '🎉', 'a']) + ) { + let replaced = replace_letters(&source, &[replacement; 6]); + let lines = replaced.matches('\n').count() + 1; + for node in walk(parse(&replaced)) { + let Some(position) = node.position() else { continue }; + prop_assert!(position.end.line <= lines, "{node:?}"); + let line = replaced.split('\n').nth(position.end.line - 1).unwrap_or_default(); + // A column can be one past the end of its line, and no further. + prop_assert!(position.end.column <= line.len() + 1, "{node:?} in {replaced:?}"); + } + } +} + +/// One HTML tag that the output of the renderer can have, after its name. +fn tags(html: &str) -> Vec<(bool, String)> { + let mut result = Vec::new(); + let mut rest = html; + while let Some(index) = rest.find('<') { + rest = &rest[index + 1..]; + let end = rest.find('>').expect("a tag that is not closed"); + let tag = &rest[..end]; + // Comments and the like are no tags. + if tag.starts_with(['!', '?']) { + rest = &rest[end + 1..]; + continue; + } + let closing = tag.starts_with('/'); + let name = tag + .trim_start_matches('/') + .split([' ', '/']) + .next() + .unwrap() + .to_string(); + let void = tag.ends_with('/') || matches!(name.as_str(), "br" | "hr" | "img" | "input"); + if !void { + result.push((closing, name)); + } + rest = &rest[end + 1..]; + } + result +} + +fn assert_well_formed(html: &str) -> Result<(), TestCaseError> { + let mut open: Vec = Vec::new(); + for (closing, name) in tags(html) { + if closing { + prop_assert_eq!(open.pop(), Some(name.clone()), "{} in {}", name, html); + } else { + match name.as_str() { + // These cannot be inside of themselves. + "a" | "p" => prop_assert!(!open.contains(&name), "nested <{}> in {}", name, html), + "li" => prop_assert!(matches!(open.last().map(String::as_str), Some("ul" | "ol")), "{}", html), + "td" | "th" => prop_assert_eq!(open.last().map(String::as_str), Some("tr"), "{}", html), + _ => {} + } + open.push(name); + } + } + prop_assert!(open.is_empty(), "unclosed {open:?} in {html}"); + Ok(()) +} + +proptest! { + #[test] + fn html_of_a_document_is_well_formed(blocks in document()) { + let source = blocks.join("\n\n") + "\n"; + assert_well_formed(&to_html(&source))?; + } + + #[test] + fn html_of_nested_blocks_is_well_formed( + blocks in prop::collection::vec(plain_block(), 1..4), + // The prefix of the first line, and the one of the lines that follow. + (first, rest) in prop::sample::select(vec![ + ("> ", "> "), + ("- ", " "), + ("1. ", " "), + ("> - ", "> "), + ("- > ", " > "), + ("- - ", " "), + ]) + ) { + let source = blocks + .join("\n\n") + .lines() + .enumerate() + .map(|(index, line)| match (index, line.is_empty()) { + (0, _) => format!("{first}{line}"), + (_, true) => rest.trim_end().to_string(), + _ => format!("{rest}{line}"), + }) + .collect::>() + .join("\n") + + "\n"; + assert_well_formed(&to_html(&source))?; + } + + #[test] + // Raw HTML is passed through as it is, so it is left out. + fn html_of_arbitrary_text_is_well_formed(source in "[^<\\p{C}]{0,60}") { + assert_well_formed(&to_html(&source))?; + } +} + +fn escape(text: &str) -> String { + text.replace('&', "&") + .replace('<', "<") + .replace('>', ">") + .replace('"', """) +} + +proptest! { + /// Text without markup is a paragraph with the text escaped. + #[test] + fn plain_text_is_an_escaped_paragraph( + words in prop::collection::vec("[a-zA-Z0-9あ-んé🎉,;'\"]{1,8}", 1..6) + ) { + let text = words.join(" "); + prop_assert_eq!(to_html(&text), format!("

    {}

    ", escape(&text))); + } + + #[test] + fn special_characters_in_text_are_escaped( + words in prop::collection::vec(("[a-z]{1,4}", prop::sample::select(vec!["<", ">", "&", "\"", "&&", "<>"])), 1..5) + ) { + // A space after each keeps `<` from starting a tag and `&` from starting a reference. + let text = words.iter().map(|(word, special)| format!("{word}{special} ")).collect::(); + let html = to_html(&text); + let inner = html.strip_prefix("

    ").and_then(|html| html.strip_suffix("

    ")).expect("a paragraph"); + prop_assert!(!inner.contains(['<', '>', '"']), "{}", html); + let after_ampersand = inner.split('&').skip(1).all(|rest| { + ["amp;", "lt;", "gt;", "quot;"].iter().any(|entity| rest.starts_with(entity)) + }); + prop_assert!(after_ampersand, "{}", html); + // What the HTML says is the text that was written. + let decoded = inner.replace("<", "<").replace(">", ">").replace(""", "\"").replace("&", "&"); + prop_assert_eq!(decoded, text.trim_end()); + } + + /// The `href` of a link has the characters of a URL and nothing that could end the attribute, and + /// has no protocol that is not safe. + #[test] + fn href_is_safe( + scheme in prop::sample::select(vec!["", "http:", "https:", "mailto:", "javascript:", "JAVASCRIPT:", "data:", "vbscript:", "file:", "ftp:", "x-y:"]), + path in "[a-zA-Z0-9/._~%?#=&:;,+@!$*'あé🎉 ]{0,12}" + ) { + let url = format!("{scheme}{path}"); + let html = to_html(&format!("[a](<{url}>)")); + let href = html + .strip_prefix("

    ' | '\\' | '`' | '{' | '}' | '|' | '^' | '[' | ']')), + "{href:?} from {url:?}" + ); + let lowered = href.to_ascii_lowercase(); + let protocol = lowered.split([':', '/', '?', '#']).next().filter(|_| { + lowered.find(':').is_some_and(|colon| lowered.find(['/', '?', '#']).is_none_or(|end| colon < end)) + }); + if let Some(protocol) = protocol { + prop_assert!(["http", "https", "irc", "ircs", "mailto", "xmpp"].contains(&protocol), "{href:?}"); + } + } + + #[test] + fn image_source_only_has_http_protocols( + scheme in prop::sample::select(vec!["", "http:", "https:", "mailto:", "javascript:", "data:", "irc:"]), + path in "[a-z/.]{0,8}" + ) { + let html = to_html(&format!("![a]({scheme}{path})")); + let src = html + .strip_prefix("

    &\"']{0,10}", 1..5), + fence in prop::sample::select(vec!["```", "~~~", "````"]) + ) { + prop_assume!(lines.iter().all(|line| !line.trim_start().starts_with(['`', '~']))); + let source = format!("{fence}\n{}\n{fence}\n", lines.join("\n")); + let html = to_html(&source); + let expected = format!("

    {}\n
    \n", escape(&lines.join("\n"))); + prop_assert_eq!(html, expected); + } + + #[test] + fn code_span_text_is_kept_and_escaped(text in "[a-zA-Z0-9あé🎉<>&\"']{1,10}") { + prop_assert_eq!(to_html(&format!("`{text}`")), format!("

    {}

    ", escape(&text))); + } + + #[test] + fn heading_depth_is_the_number_of_hashes(depth in 1usize..=6, text in "[a-zあé🎉]{1,8}") { + prop_assert_eq!( + to_html(&format!("{} {text}\n", "#".repeat(depth))), + format!("{text}\n") + ); + } + + #[test] + fn list_has_an_item_for_each_line(count in 1usize..8, ordered in any::(), start in 0u32..20) { + let source = (0..count) + .map(|index| if ordered { format!("{}. item", start + index as u32) } else { "- item".to_string() }) + .collect::>() + .join("\n"); + let html = to_html(&source); + prop_assert_eq!(html.matches("
  • ").count(), count); + if ordered && start != 1 { + let expected_start = format!("
      "); + prop_assert!(html.starts_with(&expected_start), "{}", html); + } + } + + #[test] + fn table_has_a_cell_for_each_column_of_each_row(columns in 1usize..5, rows in 0usize..4) { + let row = |text: &str| format!("| {} |", vec![text; columns].join(" | ")); + let mut lines = vec![row("h"), format!("|{}", "-|".repeat(columns))]; + lines.extend((0..rows).map(|_| row("c"))); + let html = to_html(&(lines.join("\n") + "\n")); + prop_assert_eq!(html.matches("
  • ").count(), columns); + prop_assert_eq!(html.matches("").count(), columns * rows); + prop_assert_eq!(html.contains("
    \n\n\n\n\n\n\n\n\n\n\n\n\n
    ab
    12
    \n" @@ -341,7 +341,7 @@ html: "\n\n\n\n\n\n
    a
    \n

    h

    nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:2-1:3 })]:0, row: 0, @1:1-1:4 }), TableAlign(TableAlign { align: [None], @2:1-2:1 }), Code(Code { value: "x", lang: None, @3:1-5:4, meta: None, fence: true })] html: "\n\n\n\n\n\n
    a
    \n
    x\n
    \n" ## table_then_list "|a|\n|-|\n- x\n" -nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:2-1:3 })]:0, row: 0, @1:1-1:4 }), TableAlign(TableAlign { align: [None], @2:1-2:1 }), List(List { values: [Text(Text { value: "x", @3:3-3:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @3:3-3:4 })] +nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:2-1:3 })]:0, row: 0, @1:1-1:4 }), TableAlign(TableAlign { align: [None], @2:1-2:1 }), List(List { values: [Text(Text { value: "x", @3:3-3:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @3:3-3:4 })] html: "\n\n\n\n\n\n
    a
    \n
      \n
    • x
    • \n
    \n" ## table_two "|a|\n|-|\n\n|b|\n|-|\n" nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:2-1:3 })]:0, row: 0, @1:1-1:4 }), TableAlign(TableAlign { align: [None], @2:1-2:1 }), TableCell(TableCell { values: [Text(Text { value: "b", @4:2-4:3 })]:0, row: 0, @4:1-4:4 }), TableAlign(TableAlign { align: [None], @5:1-5:1 })] @@ -431,8 +431,8 @@ html: "

    a d a [c](d

    " ## inline_image_alt "![a *b* `c`](d) ![e ![f](g)](h)" -nodes: [Image(Image { alt: "a b c", url: "d", title: None, @1:1-1:16 }), Text(Text { value: " ", @1:16-1:17 }), Image(Image { alt: "e ", url: "h", title: None, @1:17-1:32 })] -html: "

    \"a \"e

    " +nodes: [Image(Image { alt: "a b c", url: "d", title: None, @1:1-1:16 }), Text(Text { value: " ", @1:16-1:17 }), Image(Image { alt: "e f", url: "h", title: None, @1:17-1:32 })] +html: "

    \"a \"e

    " ## inline_link_in_link "[a [b](c) d](e)" nodes: [Text(Text { value: "[a ", @1:1-1:4 }), Link(Link { url: Url("c"), title: None, values: [Text(Text { value: "b", @1:5-1:6 })], @1:4-1:10 }), Text(Text { value: " d](e)", @1:10-1:16 })] html: "

    [a b d](e)

    " @@ -464,7 +464,7 @@ html: "

    a b c

    \n" nodes: [Heading(Heading { depth: 1, values: [Text(Text { value: "a ", @1:1-1:3 }), Emphasis(Emphasis { values: [Text(Text { value: "b", @1:4-1:5 })], @1:3-1:6 }), Text(Text { value: "\nc", @1:6-2:2 })], @1:1-3:4 })] html: "

    a b\nc

    \n" ## inline_list_item "- a *b*\n c **d**\n" -nodes: [List(List { values: [Text(Text { value: "a ", @1:3-1:5 }), Emphasis(Emphasis { values: [Text(Text { value: "b", @1:6-1:7 })], @1:5-1:8 }), Text(Text { value: "\nc ", @1:8-2:5 }), Strong(Strong { values: [Text(Text { value: "d", @2:7-2:8 })], @2:5-2:10 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-2:10 })] +nodes: [List(List { values: [Text(Text { value: "a ", @1:3-1:5 }), Emphasis(Emphasis { values: [Text(Text { value: "b", @1:6-1:7 })], @1:5-1:8 }), Text(Text { value: "\nc ", @1:8-2:5 }), Strong(Strong { values: [Text(Text { value: "d", @2:7-2:8 })], @2:5-2:10 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-2:10 })] html: "
      \n
    • a b\nc d
    • \n
    \n" ## inline_table_cell "| a *b* | `c\\|d` |\n|-|-|\n" nodes: [TableCell(TableCell { values: [Text(Text { value: "a ", @1:3-1:5 }), Emphasis(Emphasis { values: [Text(Text { value: "b", @1:6-1:7 })], @1:5-1:8 })]:0, row: 0, @1:1-1:9 }), TableCell(TableCell { values: [CodeInline(CodeInline { value: "c|d", @1:11-1:17 })]:1, row: 0, @1:9-1:19 }), TableAlign(TableAlign { align: [None, None], @2:1-2:1 })] @@ -512,8 +512,8 @@ html: "

    text\n[a]: b

    " nodes: [Definition(Definition { @1:1-1:7, url: Url("b"), title: None, ident: "a", label: Some("a") }), Text(Text { value: "===", @2:1-2:4 })] html: "

    ===

    " ## def_then_setext_dashes "[a]: b\n---" -nodes: [Definition(Definition { @1:1-1:7, url: Url("b"), title: None, ident: "a", label: Some("a") }), Text(Text { value: "---", @2:1-2:4 })] -html: "

    ---

    " +nodes: [Definition(Definition { @1:1-1:7, url: Url("b"), title: None, ident: "a", label: Some("a") }), HorizontalRule(HorizontalRule { marker: Some('-'), @2:1-2:4 })] +html: "
    " ## def_indent1 " [a]: b" nodes: [Definition(Definition { @1:1-1:8, url: Url("b"), title: None, ident: "a", label: Some("a") })] html: "" @@ -527,7 +527,7 @@ html: "
    [a]: b\n
    " nodes: [Blockquote(Blockquote { values: [Definition(Definition { @1:3-1:9, url: Url("b"), title: None, ident: "a", label: Some("a") })], @1:1-1:9 })] html: "
    \n
    " ## def_in_list "- [a]: b" -nodes: [List(List { values: [Definition(Definition { @1:3-1:9, url: Url("b"), title: None, ident: "a", label: Some("a") })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-1:9 })] +nodes: [List(List { values: [Definition(Definition { @1:3-1:9, url: Url("b"), title: None, ident: "a", label: Some("a") })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-1:9 })] html: "
      \n
    • \n
    " ## def_no_dest "[a]:" nodes: [Text(Text { value: "[a]:", @1:1-1:5 })] @@ -554,7 +554,7 @@ html: "" nodes: [Text(Text { value: "[a]: b c", @1:1-1:9 })] html: "

    [a]: b c

    " ## def_duplicate_labels "[A b]: c\n[a b]: d" -nodes: [Definition(Definition { @1:1-1:10, url: Url("c"), title: None, ident: "ab", label: Some("A b") }), Definition(Definition { @2:1-2:9, url: Url("d"), title: None, ident: "ab", label: Some("a b") })] +nodes: [Definition(Definition { @1:1-1:10, url: Url("c"), title: None, ident: "a b", label: Some("A b") }), Definition(Definition { @2:1-2:9, url: Url("d"), title: None, ident: "a b", label: Some("a b") })] html: "" ## def_empty_angle "[a]: <>" nodes: [Definition(Definition { @1:1-1:8, url: Url(""), title: None, ident: "a", label: Some("a") })] @@ -566,7 +566,7 @@ html: "" nodes: [Definition(Definition { @1:1-1:8, url: Url("/u"), title: None, ident: "a", label: Some("a") }), LinkRef(LinkRef { ident: "a", label: Some("a"), values: [Text(Text { value: "a", @3:2-3:3 })], @3:1-3:4 }), Text(Text { value: " ", @3:4-3:5 }), LinkRef(LinkRef { ident: "a", label: Some("a"), values: [Text(Text { value: "a", @3:6-3:7 })], @3:5-3:10 }), Text(Text { value: " ", @3:10-3:11 }), LinkRef(LinkRef { ident: "a", label: Some("a"), values: [Text(Text { value: "b", @3:12-3:13 })], @3:11-3:17 }), Text(Text { value: " [a][b] [A b]", @3:17-3:30 })] html: "

    a a b [a][b] [A b]

    " ## def_use_normalized "[ab cd]: /u\n\n[AB CD] [ab\ncd]" -nodes: [Definition(Definition { @1:1-1:12, url: Url("/u"), title: None, ident: "abcd", label: Some("ab cd") }), LinkRef(LinkRef { ident: "abcd", label: Some("AB CD"), values: [Text(Text { value: "AB CD", @3:2-3:9 })], @3:1-3:10 }), Text(Text { value: " ", @3:10-3:11 }), LinkRef(LinkRef { ident: "abcd", label: Some("ab\ncd"), values: [Text(Text { value: "ab\ncd", @3:12-4:3 })], @3:11-4:4 })] +nodes: [Definition(Definition { @1:1-1:12, url: Url("/u"), title: None, ident: "ab cd", label: Some("ab cd") }), LinkRef(LinkRef { ident: "ab cd", label: Some("AB CD"), values: [Text(Text { value: "AB CD", @3:2-3:9 })], @3:1-3:10 }), Text(Text { value: " ", @3:10-3:11 }), LinkRef(LinkRef { ident: "ab cd", label: Some("ab\ncd"), values: [Text(Text { value: "ab\ncd", @3:12-4:3 })], @3:11-4:4 })] html: "

    AB CD ab\ncd

    " ## def_use_before "[a] and [b][a]\n\n[a]: /u \"T\"" nodes: [LinkRef(LinkRef { ident: "a", label: Some("a"), values: [Text(Text { value: "a", @1:2-1:3 })], @1:1-1:4 }), Text(Text { value: " and ", @1:4-1:9 }), LinkRef(LinkRef { ident: "a", label: Some("a"), values: [Text(Text { value: "b", @1:10-1:11 })], @1:9-1:15 }), Definition(Definition { @3:1-3:12, url: Url("/u"), title: Some(Title("T")), ident: "a", label: Some("a") })] @@ -614,10 +614,10 @@ html: "" nodes: [Footnote(Footnote { ident: "a", values: [Text(Text { value: "b", @1:7-1:8 })], @1:1-2:1 }), Footnote(Footnote { ident: "c", values: [Text(Text { value: "d", @3:7-3:8 })], @3:1-3:8 })] html: "" ## fn_label_space "[^ a]: b" -nodes: [Definition(Definition { @1:1-1:9, url: Url("b"), title: None, ident: "^a", label: Some("^ a") })] +nodes: [Definition(Definition { @1:1-1:9, url: Url("b"), title: None, ident: "^ a", label: Some("^ a") })] html: "" ## fn_label_inner_space "[^a b]: b" -nodes: [Definition(Definition { @1:1-1:10, url: Url("b"), title: None, ident: "^ab", label: Some("^a b") })] +nodes: [Definition(Definition { @1:1-1:10, url: Url("b"), title: None, ident: "^a b", label: Some("^a b") })] html: "" ## fn_label_empty "[^]: b" nodes: [Definition(Definition { @1:1-1:7, url: Url("b"), title: None, ident: "^", label: Some("^") })] @@ -632,19 +632,19 @@ html: "
    [^a]: b\n
    " nodes: [Blockquote(Blockquote { values: [Footnote(Footnote { ident: "a", values: [Text(Text { value: "b", @1:9-1:10 })], @1:3-1:10 })], @1:1-1:10 })] html: "
    \n
    " ## fn_in_list "- [^a]: b" -nodes: [List(List { values: [Footnote(Footnote { ident: "a", values: [Text(Text { value: "b", @1:9-1:10 })], @1:3-1:10 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-1:10 })] +nodes: [List(List { values: [Footnote(Footnote { ident: "a", values: [Text(Text { value: "b", @1:9-1:10 })], @1:3-1:10 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-1:10 })] html: "
      \n
    • \n
    " ## fn_then_quote "[^a]: b\n> q" nodes: [Footnote(Footnote { ident: "a", values: [Text(Text { value: "b", @1:7-1:8 })], @1:1-1:8 }), Blockquote(Blockquote { values: [Text(Text { value: "q", @2:3-2:4 })], @2:1-2:4 })] html: "
    \n

    q

    \n
    " ## fn_then_list "[^a]: b\n- x" -nodes: [Footnote(Footnote { ident: "a", values: [Text(Text { value: "b", @1:7-1:8 })], @1:1-1:8 }), List(List { values: [Text(Text { value: "x", @2:3-2:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @2:3-2:4 })] +nodes: [Footnote(Footnote { ident: "a", values: [Text(Text { value: "b", @1:7-1:8 })], @1:1-1:8 }), List(List { values: [Text(Text { value: "x", @2:3-2:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @2:3-2:4 })] html: "
      \n
    • x
    • \n
    " ## fn_heading "[^a]: # h" nodes: [Footnote(Footnote { ident: "a", values: [Heading(Heading { depth: 1, values: [Text(Text { value: "h", @1:9-1:10 })], @1:7-1:10 })], @1:1-1:10 })] html: "" ## fn_list "[^a]: - x\n - y" -nodes: [Footnote(Footnote { ident: "a", values: [List(List { values: [Text(Text { value: "x", @1:9-1:10 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:9-1:10 }), List(List { values: [Text(Text { value: "y", @2:7-2:8 })], index: 1, level: 0, ordered: false, checked: None, spread: false, start: None, @2:7-2:8 })], @1:1-2:8 })] +nodes: [Footnote(Footnote { ident: "a", values: [List(List { values: [Text(Text { value: "x", @1:9-1:10 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:9-1:10 }), List(List { values: [Text(Text { value: "y", @2:7-2:8 })], index: 1, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @2:7-2:8 })], @1:1-2:8 })] html: "" ## fn_fence "[^a]: b\n ```\n x\n ```" nodes: [Footnote(Footnote { ident: "a", values: [Text(Text { value: "b", @1:7-1:8 }), Code(Code { value: "x", lang: None, @2:5-4:8, meta: None, fence: true })], @1:1-4:8 })] @@ -659,7 +659,7 @@ html: "" nodes: [Text(Text { value: "text", @1:1-1:5 }), Footnote(Footnote { ident: "a", values: [Text(Text { value: "b", @2:7-2:8 })], @2:1-2:8 })] html: "

    text

    \n" ## fn_then_dashes "[^a]: b\n---" -nodes: [Footnote(Footnote { ident: "a", values: [Text(Text { value: "b", @1:7-1:8 })], @1:1-1:8 }), HorizontalRule(HorizontalRule { @2:1-2:4 })] +nodes: [Footnote(Footnote { ident: "a", values: [Text(Text { value: "b", @1:7-1:8 })], @1:1-1:8 }), HorizontalRule(HorizontalRule { marker: Some('-'), @2:1-2:4 })] html: "
    " ## fn_then_equals "[^a]: b\n===" nodes: [Footnote(Footnote { ident: "a", values: [Text(Text { value: "b\n===", @1:7-2:4 })], @1:1-2:4 })] @@ -687,13 +687,13 @@ nodes: [Html(Html { value: "
    x
    ", @1:1-1:13 })] html: "
    x
    " ## html_script "\nz" nodes: [Html(Html { value: "", @1:1-5:10 }), Text(Text { value: "z", @6:1-6:2 })] -html: "\n

    z

    " +html: "<script>\nx\n\ny\n</script>\n

    z

    " ## html_pre "
    \na\n\nb
    \nc" nodes: [Html(Html { value: "
    \na\n\nb
    ", @1:1-4:8 }), Text(Text { value: "c", @5:1-5:2 })] html: "
    \na\n\nb
    \n

    c

    " ## html_style_inline_end " y\nz" nodes: [Html(Html { value: " y", @1:1-1:19 }), Text(Text { value: "z", @2:1-2:2 })] -html: " y\n

    z

    " +html: "<style>x</style> y\n

    z

    " ## html_comment "\nx" nodes: [Html(Html { value: "", @1:1-3:7 }), Text(Text { value: "x", @4:1-4:2 })] html: "\n

    x

    " @@ -731,7 +731,7 @@ html: "\nz" nodes: [Blockquote(Blockquote { values: [Html(Html { value: "
    \nx", @1:3-2:4 })], @1:1-2:4 })] html: "
    \n
    \nx\n
    " ## html_in_list "-
    \n x" -nodes: [List(List { values: [Html(Html { value: "
    \nx", @1:3-2:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-2:4 })] +nodes: [List(List { values: [Html(Html { value: "
    \nx", @1:3-2:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-2:4 })] html: "
      \n
    • \n
      \nx\n
    • \n
    " ## html_uppercase "
    \nx" nodes: [Html(Html { value: "
    \nx", @1:1-2:2 })] @@ -747,7 +747,7 @@ nodes: [Html(Html { value: "
    \nx", @1:1-2:2 })] html: "
    \nx" ## html_textarea "" nodes: [Html(Html { value: "", @1:1-4:13 })] -html: "" +html: "<textarea>\nx\n\ny</textarea>" ## html_table_two_blocks "\n\n\n
    " nodes: [Html(Html { value: "\n", @1:1-2:5 }), Html(Html { value: "
    ", @4:1-4:5 })] html: "\n\n
    " @@ -794,7 +794,7 @@ html: "" nodes: [Yaml(Yaml { value: "", @1:1-2:4 })] html: "" ## fm_unclosed "---\na\n" -nodes: [HorizontalRule(HorizontalRule { @1:1-1:4 }), Text(Text { value: "a", @2:1-2:2 })] +nodes: [HorizontalRule(HorizontalRule { marker: Some('-'), @1:1-1:4 }), Text(Text { value: "a", @2:1-2:2 })] html: "
    \n

    a

    \n" ## fm_blank_content "---\n\n---\nx" nodes: [Yaml(Yaml { value: "", @1:1-3:4 }), Text(Text { value: "x", @4:1-4:2 })] @@ -803,13 +803,13 @@ html: "

    x

    " nodes: [Toml(Toml { value: "a = 1", @1:1-3:4 }), Text(Text { value: "x", @4:1-4:2 })] html: "

    x

    " ## fm_indented " ---\na\n---" -nodes: [HorizontalRule(HorizontalRule { @1:1-1:5 }), Heading(Heading { depth: 2, values: [Text(Text { value: "a", @2:1-2:2 })], @2:1-3:4 })] +nodes: [HorizontalRule(HorizontalRule { marker: Some('-'), @1:1-1:5 }), Heading(Heading { depth: 2, values: [Text(Text { value: "a", @2:1-2:2 })], @2:1-3:4 })] html: "
    \n

    a

    " ## fm_close_trailing_space "---\na\n--- \nx" nodes: [Yaml(Yaml { value: "a", @1:1-3:5 }), Text(Text { value: "x", @4:1-4:2 })] html: "

    x

    " ## fm_close_longer "---\na\n----\nx" -nodes: [HorizontalRule(HorizontalRule { @1:1-1:4 }), Heading(Heading { depth: 2, values: [Text(Text { value: "a", @2:1-2:2 })], @2:1-3:5 }), Text(Text { value: "x", @4:1-4:2 })] +nodes: [HorizontalRule(HorizontalRule { marker: Some('-'), @1:1-1:4 }), Heading(Heading { depth: 2, values: [Text(Text { value: "a", @2:1-2:2 })], @2:1-3:5 }), Text(Text { value: "x", @4:1-4:2 })] html: "
    \n

    a

    \n

    x

    " ## fm_open_trailing_space "--- \na\n---" nodes: [Yaml(Yaml { value: "a", @1:1-3:4 })] @@ -821,13 +821,13 @@ html: "

    ---x\na

    " nodes: [Heading(Heading { depth: 2, values: [Text(Text { value: "text", @1:1-1:5 })], @1:1-2:4 }), Heading(Heading { depth: 2, values: [Text(Text { value: "a", @3:1-3:2 })], @3:1-4:4 })] html: "

    text

    \n

    a

    " ## fm_blank_before "\n---\na\n---" -nodes: [HorizontalRule(HorizontalRule { @2:1-2:4 }), Heading(Heading { depth: 2, values: [Text(Text { value: "a", @3:1-3:2 })], @3:1-4:4 })] +nodes: [HorizontalRule(HorizontalRule { marker: Some('-'), @2:1-2:4 }), Heading(Heading { depth: 2, values: [Text(Text { value: "a", @3:1-3:2 })], @3:1-4:4 })] html: "
    \n

    a

    " ## fm_multiline_content "---\na: 1\n\nb: 2\n---\n# h" nodes: [Yaml(Yaml { value: "a: 1\n\nb: 2", @1:1-5:4 }), Heading(Heading { depth: 1, values: [Text(Text { value: "h", @6:3-6:4 })], @6:1-6:4 })] html: "

    h

    " ## fm_hr_after "---\na\n---\n\n---" -nodes: [Yaml(Yaml { value: "a", @1:1-3:4 }), HorizontalRule(HorizontalRule { @5:1-5:4 })] +nodes: [Yaml(Yaml { value: "a", @1:1-3:4 }), HorizontalRule(HorizontalRule { marker: Some('-'), @5:1-5:4 })] html: "
    " ## math_basic "$$\na\n$$" nodes: [Math(Math { value: "a", @1:1-3:3 })] @@ -863,7 +863,7 @@ html: "
    " nodes: [Blockquote(Blockquote { values: [Math(Math { value: "a", @1:3-3:5 })], @1:1-3:5 })] html: "
    \n
    a\n
    \n
    " ## math_in_list "- $$\n a\n $$" -nodes: [List(List { values: [Math(Math { value: "a", @1:3-3:5 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-3:5 })] +nodes: [List(List { values: [Math(Math { value: "a", @1:3-3:5 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-3:5 })] html: "
      \n
    • \n
      a\n
      \n
    • \n
    " ## math_close_junk "$$\na\n$$ x" nodes: [Math(Math { value: "a\n$$ x", @1:1-3:5 })] @@ -905,14 +905,14 @@ html: "
    \n
    " nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:3-1:4 })]:0, row: 0, @1:1-1:4 }), TableCell(TableCell { values: [Text(Text { value: "b", @1:5-1:6 })]:1, row: 0, @1:4-1:6 }), TableAlign(TableAlign { align: [None, Center], @2:1-2:1 })] html: "\n\n\n\n\n\n\n
    ab
    " ## tab_nested_list "- a\n\t- b\n\t\t- c\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-1:4 }), List(List { values: [Text(Text { value: "b", @2:7-2:8 })], index: 0, level: 1, ordered: false, checked: None, spread: false, start: None, @2:7-2:8 }), List(List { values: [Text(Text { value: "c", @3:11-3:12 })], index: 0, level: 2, ordered: false, checked: None, spread: false, start: None, @3:11-3:12 })] +nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-1:4 }), List(List { values: [Text(Text { value: "b", @2:7-2:8 })], index: 0, level: 1, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @2:7-2:8 }), List(List { values: [Text(Text { value: "c", @3:11-3:12 })], index: 0, level: 2, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @3:11-3:12 })] html: "
      \n
    • a\n
        \n
      • b\n
          \n
        • c
        • \n
        \n
      • \n
      \n
    • \n
    \n" ## tab_nested_ordered "1. a\n\t1. b\n\t\t1. c\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:4-1:5 })], index: 0, level: 0, ordered: true, checked: None, spread: false, start: Some(1), @1:4-1:5 }), List(List { values: [Text(Text { value: "b", @2:8-2:9 })], index: 0, level: 1, ordered: true, checked: None, spread: false, start: Some(1), @2:8-2:9 }), List(List { values: [Text(Text { value: "c", @3:12-3:13 })], index: 0, level: 2, ordered: true, checked: None, spread: false, start: Some(1), @3:12-3:13 })] +nodes: [List(List { values: [Text(Text { value: "a", @1:4-1:5 })], index: 0, level: 0, ordered: true, checked: None, spread: false, start: Some(1), marker: Some('.'), @1:4-1:5 }), List(List { values: [Text(Text { value: "b", @2:8-2:9 })], index: 0, level: 1, ordered: true, checked: None, spread: false, start: Some(1), marker: Some('.'), @2:8-2:9 }), List(List { values: [Text(Text { value: "c", @3:12-3:13 })], index: 0, level: 2, ordered: true, checked: None, spread: false, start: Some(1), marker: Some('.'), @3:12-3:13 })] html: "
      \n
    1. a\n
        \n
      1. b\n
          \n
        1. c
        2. \n
        \n
      2. \n
      \n
    2. \n
    \n" ## tab_list_code "- a\n\n\t\tcode\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 }), Code(Code { value: "code", lang: None, @3:3-3:13, meta: None, fence: false })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-3:13 })] -html: "
      \n
    • \n

      a

      \n
      code\n
      \n
    • \n
    \n" +nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 }), Code(Code { value: " code", lang: None, @3:3-3:13, meta: None, fence: false })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-3:13 })] +html: "
      \n
    • \n

      a

      \n
        code\n
      \n
    • \n
    \n" ## tab_in_text "a\tb\t*c*\n" nodes: [Text(Text { value: "a\tb\t", @1:1-1:9 }), Emphasis(Emphasis { values: [Text(Text { value: "c", @1:10-1:11 })], @1:9-1:12 })] html: "

    a\tb\tc

    \n" @@ -920,34 +920,34 @@ html: "

    a\tb\tc

    \n" nodes: [Code(Code { value: "code\n\tmore", lang: None, @1:1-2:13, meta: None, fence: false })] html: "
    code\n\tmore\n
    \n" ## tab_after_marker "-\ta\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:5-1:6 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:5-1:6 })] +nodes: [List(List { values: [Text(Text { value: "a", @1:5-1:6 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:5-1:6 })] html: "
      \n
    • a
    • \n
    \n" ## atx_leading_sequences "# # a" -nodes: [Heading(Heading { depth: 1, values: [Text(Text { value: "a", @1:5-1:6 })], @1:1-1:6 })] -html: "

    a

    " +nodes: [Heading(Heading { depth: 1, values: [Text(Text { value: "# a", @1:3-1:6 })], @1:1-1:6 })] +html: "

    # a

    " ## atx_leading_sequences2 "## # a" -nodes: [Heading(Heading { depth: 2, values: [Text(Text { value: "a", @1:6-1:7 })], @1:1-1:7 })] -html: "

    a

    " +nodes: [Heading(Heading { depth: 2, values: [Text(Text { value: "# a", @1:4-1:7 })], @1:1-1:7 })] +html: "

    # a

    " ## atx_only_sequences "# # #" -nodes: [Heading(Heading { depth: 1, values: [], @1:1-1:6 })] -html: "

    " +nodes: [Heading(Heading { depth: 1, values: [Text(Text { value: "#", @1:3-1:4 })], @1:1-1:6 })] +html: "

    #

    " ## atx_no_space_sequence "# #a" -nodes: [Heading(Heading { depth: 1, values: [Text(Text { value: "a", @1:4-1:5 })], @1:1-1:5 })] -html: "

    a

    " +nodes: [Heading(Heading { depth: 1, values: [Text(Text { value: "#a", @1:3-1:5 })], @1:1-1:5 })] +html: "

    #a

    " ## atx_double_sequence "# ## a" -nodes: [Heading(Heading { depth: 1, values: [Text(Text { value: "a", @1:6-1:7 })], @1:1-1:7 })] -html: "

    a

    " +nodes: [Heading(Heading { depth: 1, values: [Text(Text { value: "## a", @1:3-1:7 })], @1:1-1:7 })] +html: "

    ## a

    " ## atx_tab_sequence "#\t# a" -nodes: [Heading(Heading { depth: 1, values: [Text(Text { value: "a", @1:7-1:8 })], @1:1-1:8 })] -html: "

    a

    " +nodes: [Heading(Heading { depth: 1, values: [Text(Text { value: "# a", @1:5-1:8 })], @1:1-1:8 })] +html: "

    # a

    " ## atx_closing_and_leading "# # a #" -nodes: [Heading(Heading { depth: 1, values: [Text(Text { value: "a", @1:5-1:6 })], @1:1-1:8 })] -html: "

    a

    " +nodes: [Heading(Heading { depth: 1, values: [Text(Text { value: "# a", @1:3-1:6 })], @1:1-1:8 })] +html: "

    # a

    " ## atx_inner_hash "# a # b" nodes: [Heading(Heading { depth: 1, values: [Text(Text { value: "a # b", @1:3-1:8 })], @1:1-1:8 })] html: "

    a # b

    " ## quote_footnote_trailing_space "- a\n > [^a]: x\n " -nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 }), Blockquote(Blockquote { values: [Footnote(Footnote { ident: "a", values: [Text(Text { value: "x", @2:11-2:12 })], @2:5-3:3 })], @2:3-3:3 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-3:3 })] +nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 }), Blockquote(Blockquote { values: [Footnote(Footnote { ident: "a", values: [Text(Text { value: "x", @2:11-2:12 })], @2:5-3:3 })], @2:3-3:3 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-3:3 })] html: "
      \n
    • a\n
      \n
      \n
    • \n
    " ## indented_code_trailing_blank " code\n\n\n" nodes: [Code(Code { value: "code", lang: None, @1:1-1:9, meta: None, fence: false })] @@ -1007,28 +1007,28 @@ html: "\n\n\n\n\n\na\n::-

    \n" ## table_body_empty_marker "a|b\n|-|-|\n*\n" -nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:1-1:2 })]:0, row: 0, @1:1-1:2 }), TableCell(TableCell { values: [Text(Text { value: "b", @1:3-1:4 })]:1, row: 0, @1:2-1:4 }), TableAlign(TableAlign { align: [None, None], @2:1-2:1 }), List(List { values: [], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @3:1-3:2 })] +nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:1-1:2 })]:0, row: 0, @1:1-1:2 }), TableCell(TableCell { values: [Text(Text { value: "b", @1:3-1:4 })]:1, row: 0, @1:2-1:4 }), TableAlign(TableAlign { align: [None, None], @2:1-2:1 }), List(List { values: [], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('*'), @3:1-3:2 })] html: "
    a
    \n\n\n\n\n\n\n
    ab
    \n
      \n
    • \n
    \n" ## table_body_ordered_marker "a|b\n|-|-|\n2. x\n" -nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:1-1:2 })]:0, row: 0, @1:1-1:2 }), TableCell(TableCell { values: [Text(Text { value: "b", @1:3-1:4 })]:1, row: 0, @1:2-1:4 }), TableAlign(TableAlign { align: [None, None], @2:1-2:1 }), List(List { values: [Text(Text { value: "x", @3:4-3:5 })], index: 0, level: 0, ordered: true, checked: None, spread: false, start: Some(2), @3:4-3:5 })] +nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:1-1:2 })]:0, row: 0, @1:1-1:2 }), TableCell(TableCell { values: [Text(Text { value: "b", @1:3-1:4 })]:1, row: 0, @1:2-1:4 }), TableAlign(TableAlign { align: [None, None], @2:1-2:1 }), List(List { values: [Text(Text { value: "x", @3:4-3:5 })], index: 0, level: 0, ordered: true, checked: None, spread: false, start: Some(2), marker: Some('.'), @3:4-3:5 })] html: "\n\n\n\n\n\n\n
    ab
    \n
      \n
    1. x
    2. \n
    \n" ## table_body_empty_ordered "a|b\n|-|-|\n1.\n" -nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:1-1:2 })]:0, row: 0, @1:1-1:2 }), TableCell(TableCell { values: [Text(Text { value: "b", @1:3-1:4 })]:1, row: 0, @1:2-1:4 }), TableAlign(TableAlign { align: [None, None], @2:1-2:1 }), List(List { values: [], index: 0, level: 0, ordered: true, checked: None, spread: false, start: Some(1), @3:1-3:3 })] +nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:1-1:2 })]:0, row: 0, @1:1-1:2 }), TableCell(TableCell { values: [Text(Text { value: "b", @1:3-1:4 })]:1, row: 0, @1:2-1:4 }), TableAlign(TableAlign { align: [None, None], @2:1-2:1 }), List(List { values: [], index: 0, level: 0, ordered: true, checked: None, spread: false, start: Some(1), marker: Some('.'), @3:1-3:3 })] html: "\n\n\n\n\n\n\n
    ab
    \n
      \n
    1. \n
    \n" ## paragraph_after_partial_tab "- a\n\n\t[P](h) x\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 }), Link(Link { url: Url("h"), title: None, values: [Text(Text { value: "P", @3:6-3:7 })], @3:5-3:11 }), Text(Text { value: " x", @3:11-3:13 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-3:13 })] +nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 }), Link(Link { url: Url("h"), title: None, values: [Text(Text { value: "P", @3:6-3:7 })], @3:5-3:11 }), Text(Text { value: " x", @3:11-3:13 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-3:13 })] html: "
      \n
    • \n

      a

      \n

      P x

      \n
    • \n
    \n" ## fence_after_partial_tab "- a\n\n\t```ts\n\t\tx\n\t```\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 }), Code(Code { value: "\tx", lang: Some("ts"), @3:5-5:8, meta: None, fence: true })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-5:8 })] -html: "
      \n
    • \n

      a

      \n
      \tx\n
      \n
    • \n
    \n" +nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 }), Code(Code { value: "\tx", lang: Some("ts"), @3:5-5:8, meta: None, fence: true })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-5:8 })] +html: "
      \n
    • \n

      a

      \n
      \tx\n
      \n
    • \n
    \n" ## math_after_partial_tab "- a\n\n\t$$\n\tm\n\t$$\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 }), Math(Math { value: "m", @3:5-5:7 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-5:7 })] +nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 }), Math(Math { value: "m", @3:5-5:7 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-5:7 })] html: "
      \n
    • \n

      a

      \n
      m\n
      \n
    • \n
    \n" ## quote_after_partial_tab "- a\n\n\t> q\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 }), Blockquote(Blockquote { values: [Text(Text { value: "q", @3:7-3:8 })], @3:3-3:8 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-3:8 })] +nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 }), Blockquote(Blockquote { values: [Text(Text { value: "q", @3:7-3:8 })], @3:3-3:8 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-3:8 })] html: "
      \n
    • \n

      a

      \n
      \n

      q

      \n
      \n
    • \n
    \n" ## heading_after_partial_tab "- a\n\n\t# h\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 }), Heading(Heading { depth: 1, values: [Text(Text { value: "h", @3:7-3:8 })], @3:3-3:8 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-3:8 })] +nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 }), Heading(Heading { depth: 1, values: [Text(Text { value: "h", @3:7-3:8 })], @3:3-3:8 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-3:8 })] html: "
      \n
    • \n

      a

      \n

      h

      \n
    • \n
    \n" ## control_character_references "… Ÿ ���" nodes: [Text(Text { value: "����\u{c}�\u{a0}���", @1:1-1:61 })] @@ -1064,7 +1064,7 @@ html: "

    a

    " nodes: [Definition(Definition { @1:1-1:7, url: Url("b"), title: None, ident: "a", label: Some("a") }), Text(Text { value: "-", @2:1-2:2 })] html: "

    -

    " ## definition_then_item "[a]: b\n- x" -nodes: [Definition(Definition { @1:1-1:7, url: Url("b"), title: None, ident: "a", label: Some("a") }), List(List { values: [Text(Text { value: "x", @2:3-2:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @2:3-2:4 })] +nodes: [Definition(Definition { @1:1-1:7, url: Url("b"), title: None, ident: "a", label: Some("a") }), List(List { values: [Text(Text { value: "x", @2:3-2:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @2:3-2:4 })] html: "
      \n
    • x
    • \n
    " ## definition_then_ordered "[a]: b\n1." nodes: [Definition(Definition { @1:1-1:7, url: Url("b"), title: None, ident: "a", label: Some("a") }), Text(Text { value: "1.", @2:1-2:3 })] @@ -1092,13 +1092,13 @@ nodes: [Text(Text { value: "👨\u{200d}👩\u{200d}👧 ", @1:1-1:20 }), Emphas html: "

    👨\u{200d}👩\u{200d}👧 a

    " ## fence_cjk_info "```あ\n日本語\n```\n" nodes: [Code(Code { value: "日本語", lang: Some("あ"), @1:1-3:4, meta: None, fence: true })] -html: "
    日本語\n
    \n" +html: "
    日本語\n
    \n" ## fence_in_quote_emoji "> ```🎉\n> x\n> ```\n" nodes: [Blockquote(Blockquote { values: [Code(Code { value: "x", lang: Some("🎉"), @1:3-3:6, meta: None, fence: true })], @1:1-3:6 })] -html: "
    \n
    x\n
    \n
    \n" +html: "
    \n
    x\n
    \n
    \n" ## fence_in_list_cjk "- ```あ\n 日本語\n ```\n" -nodes: [List(List { values: [Code(Code { value: "日本語", lang: Some("あ"), @1:3-3:6, meta: None, fence: true })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-3:6 })] -html: "
      \n
    • \n
      日本語\n
      \n
    • \n
    \n" +nodes: [List(List { values: [Code(Code { value: "日本語", lang: Some("あ"), @1:3-3:6, meta: None, fence: true })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-3:6 })] +html: "
      \n
    • \n
      日本語\n
      \n
    • \n
    \n" ## math_cjk "$$\nあ\n$$\n\n$い$\n" nodes: [Math(Math { value: "あ", @1:1-3:3 }), MathInline(MathInline { value: "い", @5:1-5:6 })] html: "
    あ\n
    \n

    い

    \n" @@ -1190,7 +1190,7 @@ nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [MdxJsxFlowElement(MdxJs ## mdx jsx_in_quote "> \n> x\n> " nodes: [Blockquote(Blockquote { values: [MdxJsxFlowElement(MdxJsxFlowElement { children: [Text(Text { value: "x", @2:3-2:4 })], @1:3-3:7, name: Some("a"), attributes: [] })], @1:1-3:7 })] ## mdx jsx_in_list "- \n x\n " -nodes: [List(List { values: [MdxJsxFlowElement(MdxJsxFlowElement { children: [Text(Text { value: "x", @2:3-2:4 })], @1:3-3:7, name: Some("a"), attributes: [] })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:3-3:7 })] +nodes: [List(List { values: [MdxJsxFlowElement(MdxJsxFlowElement { children: [Text(Text { value: "x", @2:3-2:4 })], @1:3-3:7, name: Some("a"), attributes: [] })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-3:7 })] ## mdx attr_quote_inside "" nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [], @1:1-1:14, name: Some("a"), attributes: [Property(MdxJsxAttribute { name: "b", value: Some(Literal("c\"d")) })] })] ## mdx two_tags "" @@ -1258,7 +1258,7 @@ nodes: [MdxJsxTextElement(MdxJsxTextElement { children: [Emphasis(Emphasis { val ## mdx flow_element_with_emphasis "\n*b*\n" nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [Emphasis(Emphasis { values: [Text(Text { value: "b", @2:2-2:3 })], @2:1-2:4 })], @1:1-3:5, name: Some("a"), attributes: [] })] ## mdx flow_element_with_list "\n- b\n" -nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [List(List { values: [Text(Text { value: "b", @2:3-2:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @2:3-2:4 })], @1:1-3:5, name: Some("a"), attributes: [] })] +nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [List(List { values: [Text(Text { value: "b", @2:3-2:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @2:3-2:4 })], @1:1-3:5, name: Some("a"), attributes: [] })] ## mdx list_items_unbalanced "- \n- " nodes: error: Expected a closing tag for `` (1:3) before the end of its container ## mdx spaced_equals "" @@ -1272,17 +1272,17 @@ nodes: [Heading(Heading { depth: 1, values: [Text(Text { value: "h", @1:8-1:9 }) ## mdx no_autolink "" nodes: error: Unexpected character `/` (U+002F) before local name ## mdx no_gfm "| a |\n|-|\n\n~a~ www.a.b [^a]\n\n- [ ] a\n" -nodes: [Text(Text { value: "| a |\n|-|", @1:1-2:4 }), Text(Text { value: "~a~ www.a.b [^a]", @4:1-4:17 }), List(List { values: [Text(Text { value: "[ ] a", @6:3-6:8 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @6:3-6:8 })] +nodes: [Text(Text { value: "| a |\n|-|", @1:1-2:4 }), Text(Text { value: "~a~ www.a.b [^a]", @4:1-4:17 }), List(List { values: [Text(Text { value: "[ ] a", @6:3-6:8 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @6:3-6:8 })] ## mdx no_math "$a$\n\n$$\na\n$$\n" nodes: [Text(Text { value: "$a$", @1:1-1:4 }), Text(Text { value: "$$\na\n$$", @3:1-5:3 })] ## mdx no_frontmatter "---\na\n---\n" -nodes: [HorizontalRule(HorizontalRule { @1:1-1:4 }), Heading(Heading { depth: 2, values: [Text(Text { value: "a", @2:1-2:2 })], @2:1-3:4 })] -## mdx esm_is_text "import a from 'b'\n\nexport const c = 1\n" -nodes: [Text(Text { value: "import a from 'b'", @1:1-1:18 }), Text(Text { value: "export const c = 1", @3:1-3:19 })] +nodes: [HorizontalRule(HorizontalRule { marker: Some('-'), @1:1-1:4 }), Heading(Heading { depth: 2, values: [Text(Text { value: "a", @2:1-2:2 })], @2:1-3:4 })] +## mdx esm "import a from 'b'\n\nexport const c = 1\n" +nodes: [MdxJsEsm(MdxJsEsm { value: "import a from 'b'", @1:1-1:18 }), MdxJsEsm(MdxJsEsm { value: "export const c = 1", @3:1-3:19 })] ## mdx definition "[a]: /u\n\n[a]\n" nodes: [Definition(Definition { @1:1-1:8, url: Url("/u"), title: None, ident: "a", label: Some("a") }), LinkRef(LinkRef { ident: "a", label: Some("a"), values: [Text(Text { value: "a", @3:2-3:3 })], @3:1-3:4 })] ## mdx list_deep_indent "- a\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:8-1:9 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, @1:8-1:9 })] +nodes: [List(List { values: [Text(Text { value: "a", @1:8-1:9 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:8-1:9 })] ## mdx fence "```rust\ncode\n```\n" nodes: [Code(Code { value: "code", lang: Some("rust"), @1:1-3:4, meta: None, fence: true })] ## mdx lone_cr_after_lt "<\r/-x" diff --git a/crates/mq-markdown/src/parser/tree.rs b/crates/mq-markdown/src/parser/tree.rs index 430e33313..bf553cc1f 100644 --- a/crates/mq-markdown/src/parser/tree.rs +++ b/crates/mq-markdown/src/parser/tree.rs @@ -14,6 +14,8 @@ pub(super) struct InlineSource { pub(super) lines: Vec<(usize, Point)>, /// Whether this is the content of a table cell, where `\|` in code stands for `|`. pub(super) table: bool, + /// Whether `text` contains a tab, which makes columns differ from byte offsets. + tabs: bool, } impl InlineSource { @@ -30,12 +32,35 @@ impl InlineSource { } } Self { + tabs: text.contains('\t'), text, lines: starts, table: false, } } + /// Removes the first `marker` bytes and the whitespace that follows them, line endings included. + pub(super) fn remove_prefix(&mut self, marker: usize) { + let rest = &self.text[marker..]; + let end = self.text.len() - rest.trim_start_matches([' ', '\t', '\n', '\r']).len(); + self.text.replace_range(..end, ""); + self.tabs = self.text.contains('\t'); + + let index = self.lines.partition_point(|(start, _)| *start <= end).saturating_sub(1); + let (start, point) = &self.lines[index]; + let first = ( + 0, + Point { + line: point.line, + column: point.column + (end - start), + }, + ); + let rest = self.lines[index + 1..] + .iter() + .map(|(start, point)| (start - end, point.clone())); + self.lines = std::iter::once(first).chain(rest).collect(); + } + /// The position where a node ends at `offset`. After a line ending that is the start of the next /// line in the document, not the start of its content inside containers. pub(super) fn end_point(&self, offset: usize) -> Point { @@ -61,7 +86,7 @@ impl InlineSource { .saturating_sub(1); let (start, point) = &self.lines[index]; let passed = &self.text[*start..offset]; - let column = if passed.contains('\t') { + let column = if self.tabs && passed.contains('\t') { visual_column(passed.chars(), point.column - 1) + 1 } else { point.column + passed.len() @@ -122,6 +147,7 @@ pub(super) struct FootnoteBlock { pub(super) struct ListBlock { pub(super) ordered: bool, pub(super) start: Option, + pub(super) marker: char, pub(super) spread: bool, pub(super) items: Vec, } diff --git a/crates/mq-markdown/tests/html_rendering_tests.rs b/crates/mq-markdown/tests/html_rendering_tests.rs index d5a2e2198..704f6b14b 100644 --- a/crates/mq-markdown/tests/html_rendering_tests.rs +++ b/crates/mq-markdown/tests/html_rendering_tests.rs @@ -20,6 +20,74 @@ use rstest::rstest; #[case::thematic_break_after_paragraph("a\n\n***\n", "

    a

    \n
    \n")] #[case::empty("", "")] #[case::blank_lines("\n\n \n", "")] +#[case::code_keeps_rest_of_tab_in_list( + "- foo\n\n\t\tbar\n", + "
      \n
    • \n

      foo

      \n
        bar\n
      \n
    • \n
    \n" +)] +#[case::code_keeps_rest_of_tab_in_quote(">\t\tfoo\n", "
    \n
      foo\n
    \n
    \n")] +#[case::heading_keeps_a_leading_hash("# #[allow(dead_code)]\n", "

    #[allow(dead_code)]

    \n")] +#[case::heading_keeps_a_hash_and_a_space("# # a\n", "

    # a

    \n")] +#[case::code_span_over_lines_loses_indent("a `b\n c` d\n", "

    a b c d

    \n")] +#[case::lazy_line_of_backticks_after_quote("> q\n``", "
    \n

    q\n``

    \n
    ")] +#[case::quote_can_start_with_code_after_paragraph( + "text\n> code\n", + "

    text

    \n
    \n
    code\n
    \n
    \n" +)] +#[case::quote_can_start_with_ordered_item_after_paragraph( + "text\n> 2. a\n", + "

    text

    \n
    \n
      \n
    1. a
    2. \n
    \n
    \n" +)] +#[case::ordered_item_does_not_interrupt_after_quote( + "> q\n\nfoo\n2. a\n", + "
    \n

    q

    \n
    \n

    foo\n2. a

    \n" +)] +#[case::dashes_after_setext_heading_are_a_rule("a\n---\n---\nb\n", "

    a

    \n
    \n

    b

    \n")] +#[case::text_after_setext_heading_in_item_is_a_paragraph( + "- a\n ---\nb\n", + "
      \n
    • \n

      a

      \n
    • \n
    \n

    b

    \n" +)] +#[case::code_of_an_item_is_one_block( + "a\n- b\n c\n", + "

    a

    \n
      \n
    • \n
        b\n  c\n
      \n
    • \n
    \n" +)] +#[case::task_without_text("- [ ] \n", "
      \n
    • \n
    \n")] +#[case::task_without_text_keeps_the_list( + "- [x] \n- b\n", + "
      \n
    • \n
    • b
    • \n
    \n" +)] +#[case::task_text_is_not_a_list( + "- [x] - a\n", + "
      \n
    • - a
    • \n
    \n" +)] +#[case::task_text_is_not_a_quote( + "- [ ] > a\n", + "
      \n
    • > a
    • \n
    \n" +)] +#[case::task_on_the_next_line( + "- [ ] \n b\n", + "
      \n
    • b
    • \n
    \n" +)] +#[case::footnote_with_blank_line_after_marker( + "a[^1]\n\n[^1]:\n\n text\n", + "

    a1

    \n

    Footnotes

    \n
      \n
    1. \n

      text ↩

      \n
    2. \n
    \n
    \n" +)] +#[case::tag_filter_inline("a b <b>c</b>\n", "<p>a &lt;xmp> b <b>c</b></p>\n")] +#[case::tag_filter_end_tag_and_case("a </SCRIPT> b\n", "<p>a &lt;/SCRIPT> b</p>\n")] +#[case::tag_filter_block("<script>\nx\n</script>\n", "&lt;script>\nx\n&lt;/script>\n")] +#[case::tag_filter_keeps_other_tags("<scripts>\n", "<scripts>\n")] +#[case::autolink_email_after_a_slash( + "x@y.com/z@w.org\n", + "<p><a href=\"mailto:x@y.com\">x@y.com</a>/<a href=\"mailto:z@w.org\">z@w.org</a></p>\n" +)] +#[case::autolink_domain_with_emoji( + "http://x\u{1F344}.ga/\n", + "<p><a href=\"http://x%F0%9F%8D%84.ga/\">http://x\u{1F344}.ga/</a></p>\n" +)] +#[case::autolink_email_before_an_at_sign("x@y.com@\n", "<p>x@y.com@</p>\n")] +#[case::emphasis_after_a_word_before_punctuation("r*_q* x\n", "<p>r*_q* x</p>\n")] +#[case::emphasis_next_to_other_runs("]_**é日*\n", "<p>]_*<em>é日</em></p>\n")] +#[case::hard_break_in_image_alt("![a\\\nb](/u)\n", "<p><img src=\"/u\" alt=\"a\nb\" /></p>\n")] +#[case::image_in_image_alt("![foo ![bar](/url)](/url2)\n", "<p><img src=\"/url2\" alt=\"foo bar\" /></p>\n")] fn blocks(#[case] input: &str, #[case] expected: &str) { assert_eq!(to_html(input), expected); } @@ -114,9 +182,9 @@ fn links_and_images(#[case] input: &str, #[case] expected: &str) { #[rstest] #[case::fenced("```\na\n```\n", "<pre><code>a\n</code></pre>\n")] -#[case::fenced_language("```rust\na\n```\n", "<pre><code class=\"language-rust\">a\n</code></pre>\n")] -#[case::fenced_meta_is_dropped("```rust title=x\na\n```\n", "<pre><code class=\"language-rust\">a\n</code></pre>\n")] -#[case::fenced_language_escaped("```a\"b\nc\n```\n", "<pre><code class=\"language-a&quot;b\">c\n</code></pre>\n")] +#[case::fenced_language("```rust\na\n```\n", "<pre lang=\"rust\"><code>a\n</code></pre>\n")] +#[case::fenced_meta_is_dropped("```rust title=x\na\n```\n", "<pre lang=\"rust\"><code>a\n</code></pre>\n")] +#[case::fenced_language_escaped("```a\"b\nc\n```\n", "<pre lang=\"a&quot;b\"><code>c\n</code></pre>\n")] #[case::fenced_empty("```\n```\n", "<pre><code></code></pre>\n")] #[case::fenced_blank_line("```\n\n```\n", "<pre><code>\n</code></pre>\n")] #[case::fenced_blank_lines_kept("```\na\n\n\n```\n", "<pre><code>a\n\n\n</code></pre>\n")] @@ -174,7 +242,7 @@ fn code_and_html(#[case] input: &str, #[case] expected: &str) { #[case::task_unchecked("- [ ] a\n", "<ul>\n<li><input type=\"checkbox\" disabled=\"\" /> a</li>\n</ul>\n")] #[case::task_checked( "- [x] a\n", - "<ul>\n<li><input type=\"checkbox\" disabled=\"\" checked=\"\" /> a</li>\n</ul>\n" + "<ul>\n<li><input type=\"checkbox\" checked=\"\" disabled=\"\" /> a</li>\n</ul>\n" )] #[case::task_loose( "- [ ] a\n\n- b\n", diff --git a/crates/mq-markdown/tests/obsidian_tests.rs b/crates/mq-markdown/tests/obsidian_tests.rs new file mode 100644 index 000000000..1b7804b88 --- /dev/null +++ b/crates/mq-markdown/tests/obsidian_tests.rs @@ -0,0 +1,129 @@ +//! Wikilinks, embeds and callouts, the Obsidian syntax: what each construct becomes, and that it +//! survives being written out and read back. +#![cfg(feature = "obsidian")] + +use mq_markdown::{Markdown, Node}; +use rstest::rstest; + +fn describe(node: &Node) -> String { + let own = match node { + Node::WikiLink(w) => format!("wikilink({}|{})", w.target, w.text.as_deref().unwrap_or("")), + Node::Embed(e) => format!("embed({}|{})", e.target, e.display.as_deref().unwrap_or("")), + Node::Callout(c) => format!( + "callout({}|{}|{})", + c.kind, + c.fold.map(String::from).unwrap_or_default(), + c.title.as_deref().unwrap_or("") + ), + Node::Text(t) => format!("text({})", t.value.replace('\n', "\\n")), + Node::Blockquote(_) => "quote".to_string(), + Node::CodeInline(c) => format!("code({})", c.value), + Node::Code(_) => "codeblock".to_string(), + Node::Emphasis(_) => "em".to_string(), + Node::Strong(_) => "strong".to_string(), + Node::Link(_) => "link".to_string(), + Node::List(_) => "item".to_string(), + Node::Heading(_) => "heading".to_string(), + other => format!("{other:?}") + .split(['(', ' ']) + .next() + .unwrap_or_default() + .to_string(), + }; + let children = node.children().iter().map(describe).collect::<Vec<_>>().join(", "); + if children.is_empty() { + own + } else { + format!("{own}[{children}]") + } +} + +fn parse(input: &str) -> String { + Markdown::from_markdown_str(input) + .unwrap() + .nodes + .iter() + .map(describe) + .collect::<Vec<_>>() + .join(" ") +} + +#[rstest] +#[case::plain("[[Page]]", "wikilink(Page|)")] +#[case::alias("[[Page|Alias]]", "wikilink(Page|Alias)")] +#[case::heading("[[Page#Heading]]", "wikilink(Page#Heading|)")] +#[case::block("[[Page#^blk]]", "wikilink(Page#^blk|)")] +#[case::same_note_heading("[[#Heading]]", "wikilink(#Heading|)")] +#[case::nested_headings("[[Page#H#Sub]]", "wikilink(Page#H#Sub|)")] +#[case::more_than_one_pipe("[[a|b|c]]", "wikilink(a|b|c)")] +#[case::spaces_are_trimmed("[[ a | b ]]", "wikilink(a|b)")] +#[case::unicode("[[日本語|別名]]", "wikilink(日本語|別名)")] +#[case::around_text("x [[a]] y", "text(x ) wikilink(a|) text( y)")] +#[case::next_to_each_other("[[a]][[b]]", "wikilink(a|) wikilink(b|)")] +#[case::underscores_around("[[_a]] and [[b_]]", "wikilink(_a|) text( and ) wikilink(b_|)")] +#[case::asterisks_inside("[[a*b]] [[c*d]]", "wikilink(a*b|) text( ) wikilink(c*d|)")] +#[case::inside_emphasis("*[[a]]*", "em[wikilink(a|)]")] +#[case::inside_strong("**[[a]]**", "strong[wikilink(a|)]")] +#[case::in_a_list("- [[a]]", "item[wikilink(a|)]")] +#[case::in_a_heading("# [[a]]", "heading[wikilink(a|)]")] +#[case::empty_target_is_text("[[]]", "text([[]])")] +#[case::blank_target_is_text("[[ ]]", "text([[ ]])")] +#[case::unclosed_is_text("[[a]", "text([[a])")] +#[case::line_ending_is_text("[[a\nb]]", "text([[a\\nb]])")] +#[case::escaped_is_text("\\[[a]]", "text([[a]])")] +#[case::in_code_span("`[[a]]` [[b]]", "code([[a]]) text( ) wikilink(b|)")] +#[case::in_code_block("```\n[[a]]\n```", "codeblock")] +#[case::triple_bracket("[[[a]]]", "text([) wikilink(a|) text(])")] +fn wikilinks(#[case] input: &str, #[case] expected: &str) { + assert_eq!(parse(input), expected); +} + +#[rstest] +#[case::plain("![[Page]]", "embed(Page|)")] +#[case::image_size("![[img.png|100]]", "embed(img.png|100)")] +#[case::image_width_height("![[img.png|100x145]]", "embed(img.png|100x145)")] +#[case::alt_and_size("![[img.png|alt|200]]", "embed(img.png|alt|200)")] +#[case::heading("![[Page#Heading]]", "embed(Page#Heading|)")] +#[case::pdf_page("![[a.pdf#page=3]]", "embed(a.pdf#page=3|)")] +#[case::after_text("see ![[a.png]]", "text(see ) embed(a.png|)")] +#[case::empty_is_text("![[]]", "text(![[]])")] +#[case::with_wikilink("![[a.png]] [[b]]", "embed(a.png|) text( ) wikilink(b|)")] +fn embeds(#[case] input: &str, #[case] expected: &str) { + assert_eq!(parse(input), expected); +} + +#[rstest] +#[case::bare("> [!note]", "callout(note||)")] +#[case::title("> [!note] Title\n> body", "callout(note||Title)[text(body)]")] +#[case::upper_case("> [!NOTE]\n> a\n> b", "callout(NOTE||)[text(a\\nb)]")] +#[case::folded("> [!faq]- Folded\n> body", "callout(faq|-|Folded)[text(body)]")] +#[case::open("> [!tip]+ Open\n> body", "callout(tip|+|Open)[text(body)]")] +#[case::fold_without_title("> [!faq]-\n> body", "callout(faq|-|)[text(body)]")] +#[case::dash_title_after_a_space("> [!faq] - Title", "callout(faq||- Title)")] +#[case::custom_type("> [!my-custom_1] T", "callout(my-custom_1||T)")] +#[case::no_space_after_the_quote(">[!note] x", "callout(note||x)")] +#[case::title_with_emphasis("> [!note] *Title* more\n> b", "callout(note||*Title* more)[text(b)]")] +#[case::title_with_code("> [!note] a `c` b", "callout(note||a `c` b)")] +#[case::list_body("> [!note]\n> - a", "callout(note||)[item[text(a)]]")] +#[case::code_body("> [!note]\n> ```\n> x\n> ```", "callout(note||)[codeblock]")] +#[case::nested("> [!note]\n> > [!tip]\n> > inner", "callout(note||)[callout(tip||)[text(inner)]]")] +#[case::in_a_list("- > [!note]\n > body", "item[callout(note||)[text(body)]]")] +#[case::empty_type_is_a_quote("> [!] x", "quote[text([!] x)]")] +#[case::space_in_type_is_a_quote("> [! note] x", "quote[text([! note] x)]")] +#[case::space_after_type_is_a_quote("> [!note x] y", "quote[text([!note x] y)]")] +#[case::not_at_the_start("> text [!note]", "quote[text(text [!note])]")] +fn callouts(#[case] input: &str, #[case] expected: &str) { + assert_eq!(parse(input), expected); +} + +#[rstest] +#[case("[[a|b]] ![[c.png|100]] [[p#h]]\n")] +#[case("> [!faq]- Folded\n> body\n")] +#[case("> [!note] *Title*\n> body\n")] +#[case("- [[a_]] and [[_b]]\n")] +#[case("> [!note]\n> > [!tip] T\n> > inner [[x]]\n")] +fn writing_and_reading_back_keeps_the_nodes(#[case] input: &str) { + let written = Markdown::from_markdown_str(input).unwrap().to_string(); + assert_eq!(parse(&written), parse(input), "{written:?}"); + assert_eq!(written, input); +} diff --git a/crates/mq-markdown/tests/parser_structure_tests.rs b/crates/mq-markdown/tests/parser_structure_tests.rs index 703d3160a..72800ce49 100644 --- a/crates/mq-markdown/tests/parser_structure_tests.rs +++ b/crates/mq-markdown/tests/parser_structure_tests.rs @@ -256,3 +256,88 @@ fn blocks_are_separate_nodes(#[case] input: &str, #[case] count: usize) { .count(); assert_eq!(nodes, count); } + +/// Nodes whose positions overlap, as the items of a list inside a quote, must not break rendering. +#[rstest] +#[case::nested_list_in_quote("> - - {x}\n> }\n/>\n")] +fn rendering_overlapping_positions_does_not_panic(#[case] input: &str) { + let markdown = Markdown::from_markdown_str(input).unwrap(); + assert!(!markdown.to_string().is_empty()); +} + +/// A thematic rule of dashes at the start would turn into frontmatter with the next one. +#[rstest] +#[case::dashes_at_start("-----\n\ntext\n\n-----\n\nmore\n")] +fn rendering_dashes_at_start_keeps_the_rules(#[case] input: &str) { + let rendered = Markdown::from_markdown_str(input).unwrap().to_string(); + let nodes = Markdown::from_markdown_str(&rendered).unwrap().nodes; + let rules = |nodes: &[Node]| nodes.iter().filter(|n| matches!(n, Node::HorizontalRule(_))).count(); + assert_eq!( + rules(&nodes), + rules(&Markdown::from_markdown_str(input).unwrap().nodes), + "{rendered:?}" + ); +} + +/// What `to_string` writes must read back as the same document, in HTML. +#[rstest] +#[case::url_with_underscores("see https://a.b/c_d_e/ end\n")] +#[case::nested_list_in_task_item("- [ ] one\n - sub\n- two\n")] +#[case::paragraph_in_task_item("- [x] one\n\n more\n- two\n")] +#[case::reference_label_with_spaces("see [Code of Conduct].\n\n[Code of Conduct]: https://a.b/c\n")] +#[case::reference_with_code_text("type [`Options`][options].\n\n[options]: #options\n")] +#[case::table_code_with_pipe("| a | b |\n|---|---|\n| `x\\|y` | z\\|w |\n")] +#[case::link_text_is_a_url("[www.W3.org](http://www.w3.org) site\n")] +#[case::strong_with_emphasis("**_NOTE:_** text\n")] +#[case::second_paragraph_after_escape("- \\[a] x\n\n b\n")] +#[case::table_in_quote("> | a | b |\n> |---|---|\n> | 1 | 2 |\n")] +#[case::table_in_list_item("- **P:**\n | Name | Type |\n |------|------|\n | a | b |\n- next\n")] +#[case::table_row_with_more_cells("| a | b |\n|---|---|\n| 1 | 2 | 3 |\n")] +#[case::paragraph_after_footnote("[^1]: Note text.\n\nParagraph after.\n")] +#[case::list_in_empty_item("- - a\n - b\n")] +#[case::empty_code_with_blank_line("```\n\n```\n")] +#[case::paragraph_after_list_in_empty_item("- - a\n\n Own Id: X\n")] +#[case::paragraph_after_nested_list("- a\n - s\n\n b\n")] +#[case::code_after_nested_list("- a\n - s\n ```\n code\n ```\n")] +#[case::nested_lists_around_paragraph("- a\n - s\n\n b\n - t\n")] +#[case::ordered_item_with_nested_list_and_paragraph("1. x\n - s\n\n para\n2. y\n")] +fn rendering_reads_back_as_the_same_html(#[case] input: &str) { + let rendered = Markdown::from_markdown_str(input).unwrap().to_string(); + assert_eq!( + mq_markdown::to_html(&rendered), + mq_markdown::to_html(input), + "{rendered:?}" + ); +} + +/// What an MDX document starts its blocks with: `import` and `export` lines are one block up to the +/// next blank line. +#[rstest] +#[case::import_and_export("import a from \"b\"\nexport const x = 1\n\nx\n", &["esm:import a from \"b\"\nexport const x = 1", "text:x"])] +#[case::export_default("export default d\n", &["esm:export default d"])] +#[case::indented_is_text(" import a from \"b\"\n", &["text:import a from \"b\""])] +#[case::no_space_is_text("import{a} from \"b\"\n", &["text:import", "expr:a", "text: from \"b\""])] +#[case::a_word_is_text("importx a\n", &["text:importx a"])] +#[case::in_a_list_is_text("- import a from \"b\"\n", &["item"])] +#[case::does_not_interrupt_a_paragraph("text\nimport a from \"b\"\n", &["text:text\nimport a from \"b\""])] +fn mdx_esm(#[case] input: &str, #[case] expected: &[&str]) { + let nodes = Markdown::from_mdx_str(input).unwrap().nodes; + let described = nodes + .iter() + .map(|node| match node { + Node::MdxJsEsm(esm) => format!("esm:{}", esm.value), + Node::MdxTextExpression(expression) => format!("expr:{}", expression.value), + Node::Text(text) => format!("text:{}", text.value), + Node::List(_) => "item".to_string(), + other => format!("{other:?}"), + }) + .collect::<Vec<_>>(); + assert_eq!(described, expected); +} + +/// Without MDX, `import` is text. +#[test] +fn import_is_text_in_markdown() { + let nodes = Markdown::from_markdown_str("import a from \"b\"\n").unwrap().nodes; + assert!(matches!(nodes.as_slice(), [Node::Text(_)])); +} diff --git a/crates/mq-run/src/cli.rs b/crates/mq-run/src/cli.rs index 605614fd9..b278d0969 100644 --- a/crates/mq-run/src/cli.rs +++ b/crates/mq-run/src/cli.rs @@ -710,9 +710,9 @@ struct OutputArgs { #[clap(long, default_value_t = false, conflicts_with = "quiet")] unbuffered: bool, - /// Set the list style for markdown output - #[clap(long, value_enum, default_value_t = ListStyle::Dash, conflicts_with = "quiet")] - list_style: ListStyle, + /// Set the list style for markdown output (default: keep the original) + #[clap(long, value_enum, conflicts_with = "quiet")] + list_style: Option<ListStyle>, /// Set the link title surround style for markdown output #[clap(long, value_enum, default_value_t = LinkTitleStyle::Double, conflicts_with = "quiet")] @@ -3048,11 +3048,11 @@ impl Cli { let mut markdown = mq_markdown::Markdown::new(runtime_values.iter().flat_map(Self::runtime_value_to_nodes).collect()); markdown.set_options(mq_markdown::RenderOptions { - list_style: match self.output.list_style.clone() { + list_style: self.output.list_style.clone().map(|style| match style { ListStyle::Dash => mq_markdown::ListStyle::Dash, ListStyle::Plus => mq_markdown::ListStyle::Plus, ListStyle::Star => mq_markdown::ListStyle::Star, - }, + }), link_title_style: match self.output.link_title_style.clone() { LinkTitleStyle::Double => mq_markdown::TitleSurroundStyle::Double, LinkTitleStyle::Single => mq_markdown::TitleSurroundStyle::Single, @@ -4107,7 +4107,7 @@ mod tests { let cli = Cli { input: InputArgs::default(), output: OutputArgs { - list_style: style.clone(), + list_style: Some(style.clone()), ..Default::default() }, commands: None, diff --git a/crates/mq-run/tests/integration_tests.rs b/crates/mq-run/tests/integration_tests.rs index e57fc3da7..a58da0958 100644 --- a/crates/mq-run/tests/integration_tests.rs +++ b/crates/mq-run/tests/integration_tests.rs @@ -230,7 +230,8 @@ In {year}, the snowfall was above average. <Chart color="#fcb32c" year={year} /> <Component />"##, - Some(r##"{Chart} + Some(r##"import {Chart} from './snowfall.js' +export const year = 2023 {year} <Chart color="#fcb32c" year={year} /> <Component /> @@ -796,7 +797,9 @@ In {year}, the snowfall was above average. assert .success() .code(0) - .stdout("{Chart}\n{year}\n<Chart color=\"#fcb32c\" year={year} />\n<Component />\n"); + .stdout( + "import {Chart} from './snowfall.js'\nexport const year = 2023\n{year}\n<Chart color=\"#fcb32c\" year={year} />\n<Component />\n", + ); Ok(()) } diff --git a/crates/mq-wasm/src/script.rs b/crates/mq-wasm/src/script.rs index b91e8532c..da956962d 100644 --- a/crates/mq-wasm/src/script.rs +++ b/crates/mq-wasm/src/script.rs @@ -688,14 +688,11 @@ pub async fn run(code: &str, content: &str, options: JsValue) -> Result<String, .collect(), ); markdown.set_options(mq_markdown::RenderOptions { - list_style: options - .list_style - .map(|style| match style { - ListStyle::Dash => mq_markdown::ListStyle::Dash, - ListStyle::Plus => mq_markdown::ListStyle::Plus, - ListStyle::Star => mq_markdown::ListStyle::Star, - }) - .unwrap_or_default(), + list_style: options.list_style.map(|style| match style { + ListStyle::Dash => mq_markdown::ListStyle::Dash, + ListStyle::Plus => mq_markdown::ListStyle::Plus, + ListStyle::Star => mq_markdown::ListStyle::Star, + }), link_title_style: options .link_title_style .map(|style| match style { From c53a1fb208e7ce27d3756bdf886a39c83b2f645f Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Thu, 8 Oct 2026 13:05:43 +0900 Subject: [PATCH 08/88] =?UTF-8?q?=F0=9F=90=9B=20fix(markdown):=20fix=20cod?= =?UTF-8?q?e=20language=20class,=20raw=20HTML=20end=20tags=20and=20task=20?= =?UTF-8?q?markers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Render the fence language as `class="language-*"` on `<code>` and compare spec HTML directly - End raw HTML blocks only at the closing tag of the opening element - Detect a task marker on the first non-blank line after an empty list marker - Use the tab-expanded column for the position of text after a task marker --- crates/mq-markdown/src/parser.rs | 19 +------------------ crates/mq-markdown/src/parser/block.rs | 2 +- crates/mq-markdown/src/parser/html_flow.rs | 18 ++++++++---------- crates/mq-markdown/src/parser/render_html.rs | 6 +++--- crates/mq-markdown/src/parser/snapshots.txt | 10 +++++----- crates/mq-markdown/src/parser/tree.rs | 5 +++-- .../mq-markdown/tests/html_rendering_tests.rs | 18 +++++++++++++++--- .../tests/parser_structure_tests.rs | 1 + 8 files changed, 37 insertions(+), 42 deletions(-) diff --git a/crates/mq-markdown/src/parser.rs b/crates/mq-markdown/src/parser.rs index f1aa01353..435985701 100644 --- a/crates/mq-markdown/src/parser.rs +++ b/crates/mq-markdown/src/parser.rs @@ -1154,30 +1154,13 @@ mod tests { } } - /// The HTML in the form of the spec, which gives the language of code as a class and not as `lang`. - fn spec_html(html: &str) -> String { - let mut result = String::with_capacity(html.len()); - let mut rest = html; - while let Some(start) = rest.find("<pre lang=\"") { - let (before, tag) = rest.split_at(start); - let value_start = "<pre lang=\"".len(); - let end = tag[value_start..].find("\"><code>").map(|end| value_start + end); - let Some(end) = end else { break }; - result.push_str(before); - result.push_str(&format!("<pre><code class=\"language-{}\">", &tag[value_start..end])); - rest = &tag[end + "\"><code>".len()..]; - } - result.push_str(rest); - result - } - /// The numbers of the examples of `text` that are not rendered as in the spec. fn differing_examples(text: &str, minimum: usize) -> Vec<usize> { let examples = spec_examples(text); assert!(examples.len() > minimum, "only {} examples were found", examples.len()); examples .iter() - .filter(|(_, markdown, html)| spec_html(&to_html(markdown)) != *html) + .filter(|(_, markdown, html)| to_html(markdown) != *html) .map(|(number, ..)| *number) .collect() } diff --git a/crates/mq-markdown/src/parser/block.rs b/crates/mq-markdown/src/parser/block.rs index 9d80ef14c..557babc9d 100644 --- a/crates/mq-markdown/src/parser/block.rs +++ b/crates/mq-markdown/src/parser/block.rs @@ -654,7 +654,7 @@ fn list_item( index += 1; } - let checkbox = task_checkbox(&first); + let checkbox = inner.iter().find(|line| !line.is_blank()).and_then(task_checkbox); // Trailing blank lines belong to whatever follows the item, but still extend its position. let reaches_end = index == lines.len(); diff --git a/crates/mq-markdown/src/parser/html_flow.rs b/crates/mq-markdown/src/parser/html_flow.rs index a2ee6c3e2..be65e3256 100644 --- a/crates/mq-markdown/src/parser/html_flow.rs +++ b/crates/mq-markdown/src/parser/html_flow.rs @@ -4,7 +4,7 @@ #[derive(Clone, Copy, PartialEq)] pub(super) enum Kind { /// `<script`, `<pre`, `<style` or `<textarea`: ends at a line with the matching closing tag. - Raw, + Raw(&'static str), /// `<!--`: ends at a line with `-->`. Comment, /// `<?`: ends at a line with `?>`. @@ -125,8 +125,8 @@ fn tag(rest: &str, from: usize, closing: bool) -> Option<Kind> { let slash = terminator == Some(b'/'); let name = rest[from..end].to_ascii_lowercase(); - if !slash && !closing && RAW_NAMES.contains(&name.as_str()) { - Some(Kind::Raw) + if let Some(raw) = RAW_NAMES.iter().find(|raw| !slash && !closing && **raw == name) { + Some(Kind::Raw(raw)) } else if BLOCK_NAMES.contains(&name.as_str()) { // A slash has to be the end of a self-closing tag. (!slash || bytes.get(end + 1) == Some(&b'>')).then_some(Kind::Basic) @@ -206,7 +206,7 @@ pub(super) fn first_line_offset(kind: Kind) -> usize { Kind::Instruction => 1, Kind::Declaration => 3, Kind::Cdata => 9, - Kind::Raw | Kind::Basic | Kind::Complete => 1, + Kind::Raw(_) | Kind::Basic | Kind::Complete => 1, } } @@ -217,21 +217,19 @@ pub(super) fn ends_in(kind: Kind, text: &str) -> bool { Kind::Instruction => text.contains("?>"), Kind::Declaration => text.contains('>'), Kind::Cdata => text.contains("]]>"), - Kind::Raw => has_raw_end(text), + Kind::Raw(name) => has_raw_end(text, name), Kind::Basic | Kind::Complete => false, } } -/// Whether `text` has a closing tag of a raw element, such as `</script>`. -fn has_raw_end(text: &str) -> bool { +/// Whether `text` has the closing tag of the raw element `name`, such as `</script>`. +fn has_raw_end(text: &str, name: &str) -> bool { let bytes = text.as_bytes(); let mut index = 0; while let Some(found) = text[index..].find("</") { let from = index + found + 2; let letters = bytes[from..].iter().take_while(|b| b.is_ascii_alphabetic()).count(); - if bytes.get(from + letters) == Some(&b'>') - && RAW_NAMES.contains(&text[from..from + letters].to_ascii_lowercase().as_str()) - { + if bytes.get(from + letters) == Some(&b'>') && text[from..from + letters].eq_ignore_ascii_case(name) { return true; } index = from; diff --git a/crates/mq-markdown/src/parser/render_html.rs b/crates/mq-markdown/src/parser/render_html.rs index ba5eb1f77..ac432a4c1 100644 --- a/crates/mq-markdown/src/parser/render_html.rs +++ b/crates/mq-markdown/src/parser/render_html.rs @@ -272,13 +272,13 @@ impl Html { fn code(&mut self, code: &Code, lines: usize, closed: bool) { self.line_ending_if_needed(); - self.out.push_str("<pre"); + self.out.push_str("<pre><code"); if let Some(lang) = code.lang.as_deref().filter(|lang| !lang.is_empty()) { - self.out.push_str(" lang=\""); + self.out.push_str(" class=\"language-"); encode(&mut self.out, lang); self.out.push('"'); } - self.out.push_str("><code>"); + self.out.push('>'); self.raw_flow(&code.value, lines, closed); } diff --git a/crates/mq-markdown/src/parser/snapshots.txt b/crates/mq-markdown/src/parser/snapshots.txt index 9c4716d49..58220126e 100644 --- a/crates/mq-markdown/src/parser/snapshots.txt +++ b/crates/mq-markdown/src/parser/snapshots.txt @@ -84,7 +84,7 @@ nodes: [Text(Text { value: "**", @1:1-1:3 })] html: "<p>**</p>\n" ## fence_lang_meta "```rust title\nlet a;\n```\n" nodes: [Code(Code { value: "let a;", lang: Some("rust"), @1:1-3:4, meta: Some("title"), fence: true })] -html: "<pre lang=\"rust\"><code>let a;\n</code></pre>\n" +html: "<pre><code class=\"language-rust\">let a;\n</code></pre>\n" ## fence_indented " ```\n a\n b\n ```\n" nodes: [Code(Code { value: "a\n b", lang: None, @1:3-4:6, meta: None, fence: true })] html: "<pre><code>a\n b\n</code></pre>\n" @@ -1020,7 +1020,7 @@ nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 }), Link(Link { ur html: "<ul>\n<li>\n<p>a</p>\n<p><a href=\"h\">P</a> x</p>\n</li>\n</ul>\n" ## fence_after_partial_tab "- a\n\n\t```ts\n\t\tx\n\t```\n" nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 }), Code(Code { value: "\tx", lang: Some("ts"), @3:5-5:8, meta: None, fence: true })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-5:8 })] -html: "<ul>\n<li>\n<p>a</p>\n<pre lang=\"ts\"><code>\tx\n</code></pre>\n</li>\n</ul>\n" +html: "<ul>\n<li>\n<p>a</p>\n<pre><code class=\"language-ts\">\tx\n</code></pre>\n</li>\n</ul>\n" ## math_after_partial_tab "- a\n\n\t$$\n\tm\n\t$$\n" nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 }), Math(Math { value: "m", @3:5-5:7 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-5:7 })] html: "<ul>\n<li>\n<p>a</p>\n<pre><code class=\"language-math math-display\">m\n</code></pre>\n</li>\n</ul>\n" @@ -1092,13 +1092,13 @@ nodes: [Text(Text { value: "👨\u{200d}👩\u{200d}👧 ", @1:1-1:20 }), Emphas html: "<p>👨\u{200d}👩\u{200d}👧 <em>a</em></p>" ## fence_cjk_info "```あ\n日本語\n```\n" nodes: [Code(Code { value: "日本語", lang: Some("あ"), @1:1-3:4, meta: None, fence: true })] -html: "<pre lang=\"あ\"><code>日本語\n</code></pre>\n" +html: "<pre><code class=\"language-あ\">日本語\n</code></pre>\n" ## fence_in_quote_emoji "> ```🎉\n> x\n> ```\n" nodes: [Blockquote(Blockquote { values: [Code(Code { value: "x", lang: Some("🎉"), @1:3-3:6, meta: None, fence: true })], @1:1-3:6 })] -html: "<blockquote>\n<pre lang=\"🎉\"><code>x\n</code></pre>\n</blockquote>\n" +html: "<blockquote>\n<pre><code class=\"language-🎉\">x\n</code></pre>\n</blockquote>\n" ## fence_in_list_cjk "- ```あ\n 日本語\n ```\n" nodes: [List(List { values: [Code(Code { value: "日本語", lang: Some("あ"), @1:3-3:6, meta: None, fence: true })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-3:6 })] -html: "<ul>\n<li>\n<pre lang=\"あ\"><code>日本語\n</code></pre>\n</li>\n</ul>\n" +html: "<ul>\n<li>\n<pre><code class=\"language-あ\">日本語\n</code></pre>\n</li>\n</ul>\n" ## math_cjk "$$\nあ\n$$\n\n$い$\n" nodes: [Math(Math { value: "あ", @1:1-3:3 }), MathInline(MathInline { value: "い", @5:1-5:6 })] html: "<pre><code class=\"language-math math-display\">あ\n</code></pre>\n<p><code class=\"language-math math-inline\">い</code></p>\n" diff --git a/crates/mq-markdown/src/parser/tree.rs b/crates/mq-markdown/src/parser/tree.rs index bf553cc1f..d8e26d2dc 100644 --- a/crates/mq-markdown/src/parser/tree.rs +++ b/crates/mq-markdown/src/parser/tree.rs @@ -43,16 +43,17 @@ impl InlineSource { pub(super) fn remove_prefix(&mut self, marker: usize) { let rest = &self.text[marker..]; let end = self.text.len() - rest.trim_start_matches([' ', '\t', '\n', '\r']).len(); + let new_point = self.point(end); self.text.replace_range(..end, ""); self.tabs = self.text.contains('\t'); let index = self.lines.partition_point(|(start, _)| *start <= end).saturating_sub(1); - let (start, point) = &self.lines[index]; + let point = &self.lines[index].1; let first = ( 0, Point { line: point.line, - column: point.column + (end - start), + column: new_point.column, }, ); let rest = self.lines[index + 1..] diff --git a/crates/mq-markdown/tests/html_rendering_tests.rs b/crates/mq-markdown/tests/html_rendering_tests.rs index 704f6b14b..8e6b9bc87 100644 --- a/crates/mq-markdown/tests/html_rendering_tests.rs +++ b/crates/mq-markdown/tests/html_rendering_tests.rs @@ -67,6 +67,18 @@ use rstest::rstest; "- [ ] \n b\n", "<ul>\n<li><input type=\"checkbox\" disabled=\"\" /> b</li>\n</ul>\n" )] +#[case::task_after_empty_marker( + "-\n [ ] b\n", + "<ul>\n<li><input type=\"checkbox\" disabled=\"\" /> b</li>\n</ul>\n" +)] +#[case::checked_task_after_empty_marker( + "-\n [x] b\n", + "<ul>\n<li><input type=\"checkbox\" checked=\"\" disabled=\"\" /> b</li>\n</ul>\n" +)] +#[case::raw_html_ignores_other_closing_tags( + "<script>\n</style>\n*hello*\n</script>\n", + "&lt;script>\n&lt;/style>\n*hello*\n&lt;/script>\n" +)] #[case::footnote_with_blank_line_after_marker( "a[^1]\n\n[^1]:\n\n text\n", "<p>a<sup><a href=\"#user-content-fn-1\" id=\"user-content-fnref-1\" data-footnote-ref=\"\" aria-describedby=\"footnote-label\">1</a></sup></p>\n<section data-footnotes=\"\" class=\"footnotes\"><h2 id=\"footnote-label\" class=\"sr-only\">Footnotes</h2>\n<ol>\n<li id=\"user-content-fn-1\">\n<p>text <a href=\"#user-content-fnref-1\" data-footnote-backref=\"\" aria-label=\"Back to content\" class=\"data-footnote-backref\">↩</a></p>\n</li>\n</ol>\n</section>\n" @@ -182,9 +194,9 @@ fn links_and_images(#[case] input: &str, #[case] expected: &str) { #[rstest] #[case::fenced("```\na\n```\n", "<pre><code>a\n</code></pre>\n")] -#[case::fenced_language("```rust\na\n```\n", "<pre lang=\"rust\"><code>a\n</code></pre>\n")] -#[case::fenced_meta_is_dropped("```rust title=x\na\n```\n", "<pre lang=\"rust\"><code>a\n</code></pre>\n")] -#[case::fenced_language_escaped("```a\"b\nc\n```\n", "<pre lang=\"a&quot;b\"><code>c\n</code></pre>\n")] +#[case::fenced_language("```rust\na\n```\n", "<pre><code class=\"language-rust\">a\n</code></pre>\n")] +#[case::fenced_meta_is_dropped("```rust title=x\na\n```\n", "<pre><code class=\"language-rust\">a\n</code></pre>\n")] +#[case::fenced_language_escaped("```a\"b\nc\n```\n", "<pre><code class=\"language-a&quot;b\">c\n</code></pre>\n")] #[case::fenced_empty("```\n```\n", "<pre><code></code></pre>\n")] #[case::fenced_blank_line("```\n\n```\n", "<pre><code>\n</code></pre>\n")] #[case::fenced_blank_lines_kept("```\na\n\n\n```\n", "<pre><code>a\n\n\n</code></pre>\n")] diff --git a/crates/mq-markdown/tests/parser_structure_tests.rs b/crates/mq-markdown/tests/parser_structure_tests.rs index 72800ce49..ce40d6940 100644 --- a/crates/mq-markdown/tests/parser_structure_tests.rs +++ b/crates/mq-markdown/tests/parser_structure_tests.rs @@ -136,6 +136,7 @@ fn attributes(#[case] input: &str, #[case] name: &str, #[case] expected: &str) { // The position of an item is the one of its content. #[case::list_item("- a", 0, (1, 3), (1, 4))] #[case::emphasis_position("a *b*", 1, (1, 3), (1, 6))] +#[case::task_after_tab("- [ ]\tfoo", 0, (1, 9), (1, 12))] #[case::crlf("a\r\nb", 0, (1, 1), (2, 2))] #[case::cr("a\rb", 0, (1, 1), (2, 2))] #[case::second_line_after_crlf("a\r\n\r\nb", 1, (3, 1), (3, 2))] From 6e5472d092f47e3a9c26d47bfd1bc5c5669fbf63 Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Thu, 8 Oct 2026 13:05:49 +0900 Subject: [PATCH 09/88] =?UTF-8?q?=F0=9F=93=A6=20build(markdown):=20fix=20c?= =?UTF-8?q?lippy=20errors=20and=20warnings=20without=20all=20features?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Import `ListStyle` in the `markdown.rs` tests - Require `html-to-markdown` for the `html_convert` example - Gate the benchmark documents behind the features of their benchmarks --- crates/mq-markdown/Cargo.toml | 4 ++++ crates/mq-markdown/benches/benchmark.rs | 5 +++++ crates/mq-markdown/src/markdown.rs | 2 +- 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/crates/mq-markdown/Cargo.toml b/crates/mq-markdown/Cargo.toml index 0dd25572f..b00ee8c68 100644 --- a/crates/mq-markdown/Cargo.toml +++ b/crates/mq-markdown/Cargo.toml @@ -35,6 +35,10 @@ ureq = {workspace = true} harness = false name = "benchmark" +[[example]] +name = "html_convert" +required-features = ["html-to-markdown"] + [features] callout = [] color = [] diff --git a/crates/mq-markdown/benches/benchmark.rs b/crates/mq-markdown/benches/benchmark.rs index c73382a1b..5f5577fa4 100644 --- a/crates/mq-markdown/benches/benchmark.rs +++ b/crates/mq-markdown/benches/benchmark.rs @@ -3,6 +3,7 @@ fn main() { } // Build 100-paragraph document with wikilinks in each paragraph. +#[cfg(feature = "wikilink")] fn wikilink_doc() -> String { (0..100) .map(|i| format!("# Heading {i}\n\nText with [[target{i}]] and [[another{i}|Display {i}]].\n\n")) @@ -10,6 +11,7 @@ fn wikilink_doc() -> String { } // Same shape but no [[...]] patterns. +#[cfg(feature = "wikilink")] fn plain_doc() -> String { (0..100) .map(|i| format!("# Heading {i}\n\nText without any wikilink patterns here.\n\n")) @@ -17,6 +19,7 @@ fn plain_doc() -> String { } // 100 callouts of various kinds. +#[cfg(feature = "callout")] fn callout_doc() -> String { (0..100) .map(|i| { @@ -27,6 +30,7 @@ fn callout_doc() -> String { } // 100 plain blockquotes (no callout header). +#[cfg(feature = "callout")] fn plain_blockquote_doc() -> String { (0..100) .map(|i| format!("> Plain blockquote {i} without any callout header.\n\n")) @@ -34,6 +38,7 @@ fn plain_blockquote_doc() -> String { } // 100 embeds spread through paragraphs. +#[cfg(feature = "embed")] fn embed_doc() -> String { (0..100) .map(|i| format!("# Heading {i}\n\nSee ![[note{i}.md]] for details.\n\n")) diff --git a/crates/mq-markdown/src/markdown.rs b/crates/mq-markdown/src/markdown.rs index 6268253c3..f92be4479 100644 --- a/crates/mq-markdown/src/markdown.rs +++ b/crates/mq-markdown/src/markdown.rs @@ -346,7 +346,7 @@ pub fn to_html(s: &str) -> String { mod tests { use rstest::rstest; - use crate::{TitleSurroundStyle, UrlSurroundStyle}; + use crate::{ListStyle, TitleSurroundStyle, UrlSurroundStyle}; use super::*; From 28d7348dfca50afbbc6a69aece05f219ad84d3c5 Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Thu, 8 Oct 2026 13:05:55 +0900 Subject: [PATCH 10/88] =?UTF-8?q?=F0=9F=90=9B=20fix(markdown):=20parse=20c?= =?UTF-8?q?allout=20headers=20before=20inline=20parsing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A callout title followed by a blank line absorbed the next paragraph, and a title made of an embed was dropped. The header line is now split off the first paragraph before its inline content is parsed, so the title is the line as written and the body starts on the next line. Add exhaustive and property-based tests for callout titles and bodies. --- crates/mq-markdown/src/node.rs | 200 --------------------- crates/mq-markdown/src/parser.rs | 2 + crates/mq-markdown/src/parser/callout.rs | 51 ++++++ crates/mq-markdown/src/parser/resolve.rs | 39 +++- crates/mq-markdown/src/parser/tree.rs | 18 ++ crates/mq-markdown/tests/obsidian_tests.rs | 160 +++++++++++++++++ 6 files changed, 263 insertions(+), 207 deletions(-) create mode 100644 crates/mq-markdown/src/parser/callout.rs diff --git a/crates/mq-markdown/src/node.rs b/crates/mq-markdown/src/node.rs index a61365e02..a2c90a80e 100644 --- a/crates/mq-markdown/src/node.rs +++ b/crates/mq-markdown/src/node.rs @@ -2684,127 +2684,6 @@ impl Node { } } - /// Builds a blockquote node, or a callout node when the `callout` feature is enabled and the content is one. - pub(crate) fn from_blockquote(values: Vec<Node>, position: Option<Position>) -> Node { - #[cfg(feature = "callout")] - { - Self::try_parse_callout(values, position) - } - #[cfg(not(feature = "callout"))] - { - Self::Blockquote(Blockquote { values, position }) - } - } - - /// Tries to parse a `Blockquote`'s converted nodes as an Obsidian callout. - /// - /// Returns a `Callout` node when the first text starts with `[!TYPE]`, otherwise - /// falls back to a plain `Blockquote`. `TYPE` is letters, digits, `-` and `_`. A `+` or `-` right after - /// it is the fold marker, and the rest of the line is the title, inline markup included. - #[cfg(feature = "callout")] - fn try_parse_callout(values: Vec<Node>, position: Option<Position>) -> Node { - // Peek without cloning so plain blockquotes pay no allocation cost. - let valid = matches!(values.first(), Some(Node::Text(t)) if Self::callout_header(&t.value).is_some()); - if !valid { - return Node::Blockquote(Blockquote { values, position }); - } - - let mut iter = values.into_iter(); - let first_text = match iter.next() { - Some(Node::Text(t)) => t.value, - _ => unreachable!("checked above"), - }; - let (kind, fold, after_header) = - Self::callout_header(&first_text).unwrap_or_else(|| unreachable!("checked above")); - - // The title is the rest of the first line, which markup in it can spread over several nodes. - let mut title = String::new(); - let mut rest = iter.collect::<Vec<_>>().into_iter().peekable(); - let mut body_start = String::new(); - match after_header.split_once('\n') { - Some((line, remaining)) => { - title.push_str(line); - body_start = remaining.to_string(); - } - None => { - title.push_str(after_header); - // Text right after the first text is the next paragraph, markup in the title is not text. - while rest.next_if(Self::is_title_markup).is_some_and(|node| { - title.push_str(&node.to_string_with(&RenderOptions::default())); - true - }) { - // What follows markup goes on with the title, up to a line ending. - let Some(Node::Text(t)) = rest.peek() else { continue }; - if let Some((line, remaining)) = t.value.split_once('\n') { - title.push_str(line); - body_start = remaining.to_string(); - rest.next(); - break; - } - title.push_str(&t.value); - rest.next(); - } - } - } - let title = title.trim(); - - let mut body = Vec::new(); - if !body_start.trim().is_empty() { - body.push(Node::Text(Text { - value: body_start, - position: None, - })); - } - body.extend(rest); - - Node::Callout(Callout { - kind, - fold, - title: (!title.is_empty()).then(|| title.to_string()), - values: body, - position, - }) - } - - /// Whether `node` is markup that can be part of the title of a callout. - #[cfg(feature = "callout")] - fn is_title_markup(node: &Node) -> bool { - matches!( - node, - Node::Emphasis(_) - | Node::Strong(_) - | Node::Delete(_) - | Node::CodeInline(_) - | Node::MathInline(_) - | Node::Link(_) - | Node::LinkRef(_) - | Node::Image(_) - | Node::ImageRef(_) - | Node::FootnoteRef(_) - ) || { - #[cfg(feature = "wikilink")] - let wiki = matches!(node, Node::WikiLink(_)); - #[cfg(not(feature = "wikilink"))] - let wiki = false; - wiki - } - } - - /// Splits `[!TYPE]` and the fold marker off the start of `text`, returning the type, the marker and - /// what follows. - #[cfg(feature = "callout")] - fn callout_header(text: &str) -> Option<(String, Option<char>, &str)> { - let rest = text.strip_prefix("[!")?; - let end = rest.find(']')?; - let kind = &rest[..end]; - if kind.is_empty() || !kind.chars().all(|c| c.is_alphanumeric() || matches!(c, '-' | '_')) { - return None; - } - let after = &rest[end + 1..]; - let fold = after.chars().next().filter(|c| matches!(c, '+' | '-')); - Some((kind.to_string(), fold, &after[fold.map_or(0, char::len_utf8)..])) - } - /// Returns true if any `Text` descendant contains `[[`, covering both /// embed (`![[`) and wikilink (`[[`) patterns. #[cfg(any(feature = "embed", feature = "wikilink"))] @@ -5986,85 +5865,6 @@ mod tests { // --- Callout tests --- - #[cfg(feature = "callout")] - #[rstest] - // basic [!NOTE] with body on next line - #[case( - vec![Node::Text(Text { value: "[!NOTE]\nbody text".to_string(), position: None })], - Node::Callout(Callout { - fold: None, kind: "NOTE".to_string(), - title: None, - values: vec![Node::Text(Text { value: "body text".to_string(), position: None })], - position: None, - }) - )] - // [!WARNING] with custom title - #[case( - vec![Node::Text(Text { value: "[!WARNING] My Title\nbody".to_string(), position: None })], - Node::Callout(Callout { - fold: None, kind: "WARNING".to_string(), - title: Some("My Title".to_string()), - values: vec![Node::Text(Text { value: "body".to_string(), position: None })], - position: None, - }) - )] - // [!TIP] with no body - #[case( - vec![Node::Text(Text { value: "[!TIP]".to_string(), position: None })], - Node::Callout(Callout { fold: None, kind: "TIP".to_string(), title: None, values: vec![], position: None }) - )] - // lowercase kind is preserved as-is - #[case( - vec![Node::Text(Text { value: "[!note]\ncontent".to_string(), position: None })], - Node::Callout(Callout { - fold: None, kind: "note".to_string(), - title: None, - values: vec![Node::Text(Text { value: "content".to_string(), position: None })], - position: None, - }) - )] - // not a callout — plain text stays Blockquote - #[case( - vec![Node::Text(Text { value: "plain quote".to_string(), position: None })], - Node::Blockquote(Blockquote { - values: vec![Node::Text(Text { value: "plain quote".to_string(), position: None })], - position: None, - }) - )] - // [! without closing ] — stays Blockquote - #[case( - vec![Node::Text(Text { value: "[!UNCLOSED".to_string(), position: None })], - Node::Blockquote(Blockquote { - values: vec![Node::Text(Text { value: "[!UNCLOSED".to_string(), position: None })], - position: None, - }) - )] - // first node is not Text — stays Blockquote - #[case( - vec![Node::Code(Code { value: "code".to_string(), lang: None, fence: true, meta: None, position: None })], - Node::Blockquote(Blockquote { - values: vec![Node::Code(Code { value: "code".to_string(), lang: None, fence: true, meta: None, position: None })], - position: None, - }) - )] - // multi-paragraph callout (header-only node + separate body node) - #[case( - vec![ - Node::Text(Text { value: "[!NOTE]".to_string(), position: None }), - Node::Text(Text { value: "second paragraph".to_string(), position: None }), - ], - Node::Callout(Callout { - fold: None, kind: "NOTE".to_string(), - title: None, - values: vec![Node::Text(Text { value: "second paragraph".to_string(), position: None })], - position: None, - }) - )] - fn test_try_parse_callout(#[case] values: Vec<Node>, #[case] expected: Node) { - let result = Node::try_parse_callout(values, None); - assert_eq!(result, expected); - } - #[cfg(feature = "callout")] #[rstest] // basic render diff --git a/crates/mq-markdown/src/parser.rs b/crates/mq-markdown/src/parser.rs index 435985701..ee97ba263 100644 --- a/crates/mq-markdown/src/parser.rs +++ b/crates/mq-markdown/src/parser.rs @@ -7,6 +7,8 @@ //! It reads `CommonMark`, GFM, frontmatter, math and MDX (without a JavaScript parser, so expressions //! only need balanced braces). HTML is rendered by [`render_html`]. mod block; +#[cfg(feature = "callout")] +mod callout; mod code; mod definition; mod html_flow; diff --git a/crates/mq-markdown/src/parser/callout.rs b/crates/mq-markdown/src/parser/callout.rs new file mode 100644 index 000000000..d756b5a8c --- /dev/null +++ b/crates/mq-markdown/src/parser/callout.rs @@ -0,0 +1,51 @@ +//! Obsidian-style callouts: a block quote whose first line is `[!TYPE]`, an optional fold marker and an +//! optional title. + +use super::tree::{Block, InlineBlock, InlineKind}; + +/// The header line of a callout. +pub(super) struct Header { + pub(super) kind: String, + pub(super) fold: Option<char>, + pub(super) title: Option<String>, +} + +/// Splits `[!TYPE]` and the fold marker off the start of `text`, returning the type, the marker and +/// what follows. `TYPE` is letters, digits, `-` and `_`, and a `+` or `-` right after it is the marker. +fn split(text: &str) -> Option<(&str, Option<char>, &str)> { + let rest = text.strip_prefix("[!")?; + let end = rest.find(']')?; + let kind = &rest[..end]; + if kind.is_empty() || !kind.chars().all(|c| c.is_alphanumeric() || matches!(c, '-' | '_')) { + return None; + } + let after = &rest[end + 1..]; + let fold = after.chars().next().filter(|c| matches!(c, '+' | '-')); + Some((kind, fold, &after[fold.map_or(0, char::len_utf8)..])) +} + +/// Takes the header line off the first paragraph of a quote, if it is one. The title is the rest of that +/// line as written, so markup in it never mixes with the body, which starts on the next line. +pub(super) fn take_header(children: &mut Vec<Block>) -> Option<Header> { + let Some(Block::Inline(InlineBlock { + source, + kind: InlineKind::Paragraph, + })) = children.first_mut() + else { + return None; + }; + split(&source.text)?; + + let line = source.take_first_line(); + let (kind, fold, title) = split(&line)?; + let title = title.trim(); + let header = Header { + kind: kind.to_string(), + fold, + title: (!title.is_empty()).then(|| title.to_string()), + }; + if source.text.is_empty() { + children.remove(0); + } + Some(header) +} diff --git a/crates/mq-markdown/src/parser/resolve.rs b/crates/mq-markdown/src/parser/resolve.rs index 56860ea3f..6bb1e72f0 100644 --- a/crates/mq-markdown/src/parser/resolve.rs +++ b/crates/mq-markdown/src/parser/resolve.rs @@ -3,8 +3,10 @@ use super::inline; use super::mdx::TagKind; -use super::tree::{Block, InlineBlock, InlineKind, JsxTag, ListBlock, TableItem}; -use crate::node::{Footnote, Heading, Level, List, MdxJsxFlowElement, Node, Position, TableAlign, TableCell}; +use super::tree::{Block, InlineBlock, InlineKind, JsxTag, ListBlock, QuoteBlock, TableItem}; +use crate::node::{ + Blockquote, Footnote, Heading, Level, List, MdxJsxFlowElement, Node, Position, TableAlign, TableCell, +}; use rustc_hash::FxHashSet; /// The identifiers that references can resolve to. @@ -114,11 +116,7 @@ fn flatten(blocks: Vec<Block>, references: &References, nodes: &mut Vec<Node>) - inline_block(block, references, &mut out)?; out.into_iter().for_each(|node| frames.push(node)); } - Block::Quote(quote) => { - let mut values = Vec::new(); - flatten(quote.children, references, &mut values)?; - frames.push(Node::from_blockquote(values, Some(quote.position))); - } + Block::Quote(quote) => frames.push(quote_node(quote, references)?), Block::List(list) => { let mut out = Vec::new(); list_nodes(list, 0, references, &mut out)?; @@ -176,6 +174,33 @@ fn flatten(blocks: Vec<Block>, references: &References, nodes: &mut Vec<Node>) - Ok(()) } +/// A block quote, or a callout when the `callout` feature is enabled and its first line is a callout header. +fn quote_node(quote: QuoteBlock, references: &References) -> Result<Node, String> { + #[cfg(feature = "callout")] + let (header, children) = { + let mut children = quote.children; + (super::callout::take_header(&mut children), children) + }; + #[cfg(not(feature = "callout"))] + let children = quote.children; + + let mut values = Vec::new(); + flatten(children, references, &mut values)?; + let position = Some(quote.position); + + #[cfg(feature = "callout")] + if let Some(header) = header { + return Ok(Node::Callout(crate::node::Callout { + kind: header.kind, + fold: header.fold, + title: header.title, + values, + position, + })); + } + Ok(Node::Blockquote(Blockquote { values, position })) +} + fn inline_block(block: InlineBlock, references: &References, nodes: &mut Vec<Node>) -> Result<(), String> { let values = inline::parse(&block.source, references)?; match block.kind { diff --git a/crates/mq-markdown/src/parser/tree.rs b/crates/mq-markdown/src/parser/tree.rs index d8e26d2dc..ed661c639 100644 --- a/crates/mq-markdown/src/parser/tree.rs +++ b/crates/mq-markdown/src/parser/tree.rs @@ -43,6 +43,24 @@ impl InlineSource { pub(super) fn remove_prefix(&mut self, marker: usize) { let rest = &self.text[marker..]; let end = self.text.len() - rest.trim_start_matches([' ', '\t', '\n', '\r']).len(); + self.drain_to(end); + } + + /// Removes the first line with its line ending and returns it without the line ending. + #[cfg(feature = "callout")] + pub(super) fn take_first_line(&mut self) -> String { + let Some(eol) = self.text.find(['\n', '\r']) else { + self.lines.clear(); + return std::mem::take(&mut self.text); + }; + let line = self.text[..eol].to_string(); + let end = eol + if self.text[eol..].starts_with("\r\n") { 2 } else { 1 }; + self.drain_to(end); + line + } + + /// Removes the first `end` bytes, keeping the position of every line that is left. + fn drain_to(&mut self, end: usize) { let new_point = self.point(end); self.text.replace_range(..end, ""); self.tabs = self.text.contains('\t'); diff --git a/crates/mq-markdown/tests/obsidian_tests.rs b/crates/mq-markdown/tests/obsidian_tests.rs index 1b7804b88..f9c262275 100644 --- a/crates/mq-markdown/tests/obsidian_tests.rs +++ b/crates/mq-markdown/tests/obsidian_tests.rs @@ -3,6 +3,7 @@ #![cfg(feature = "obsidian")] use mq_markdown::{Markdown, Node}; +use proptest::prelude::*; use rstest::rstest; fn describe(node: &Node) -> String { @@ -103,6 +104,11 @@ fn embeds(#[case] input: &str, #[case] expected: &str) { #[case::custom_type("> [!my-custom_1] T", "callout(my-custom_1||T)")] #[case::no_space_after_the_quote(">[!note] x", "callout(note||x)")] #[case::title_with_emphasis("> [!note] *Title* more\n> b", "callout(note||*Title* more)[text(b)]")] +#[case::title_markup_then_paragraph("> [!note] *Title*\n>\n> Body", "callout(note||*Title*)[text(Body)]")] +#[case::title_markup_then_markup_paragraph( + "> [!note] *Title*\n>\n> *Body* x", + "callout(note||*Title*)[em[text(Body)], text( x)]" +)] #[case::title_with_code("> [!note] a `c` b", "callout(note||a `c` b)")] #[case::list_body("> [!note]\n> - a", "callout(note||)[item[text(a)]]")] #[case::code_body("> [!note]\n> ```\n> x\n> ```", "callout(note||)[codeblock]")] @@ -127,3 +133,157 @@ fn writing_and_reading_back_keeps_the_nodes(#[case] input: &str) { assert_eq!(parse(&written), parse(input), "{written:?}"); assert_eq!(written, input); } + +/// Title lines, and the title they have to give: markup in a title spreads it over several nodes. +const TITLES: &[(&str, &str)] = &[ + ("Title", "Title"), + ("Two words", "Two words"), + ("*Title*", "*Title*"), + ("**Title**", "**Title**"), + ("`Title`", "`Title`"), + ("~~Title~~", "~~Title~~"), + ("[Title](u)", "[Title](u)"), + ("![Title](u)", "![Title](u)"), + ("[[Title]]", "[[Title]]"), + ("![[Title]]", "![[Title]]"), + ("*A* b", "*A* b"), + ("a *B*", "a *B*"), + ("*A* b *C*", "*A* b *C*"), + ("*A* **B**", "*A* **B**"), + ("**A** `b` [[c]]", "**A** `b` [[c]]"), +]; + +/// Bodies that mean the same alone as after a title, in any separator. +const BODIES: &[&str] = &[ + "Body", + "Body words", + "*Body*", + "*Body* x", + "x *Body*", + "**Body**", + "`Body`", + "[Body](u)", + "[[Body]]", + "![[Body]]", + "![Body](u)", + "a\nb", + "*a*\nb", + "- a", + "1. a", + "# h", + "```\nx\n```", + "> q", + "> [!tip] T\n> inner", +]; + +/// Bodies that are only right after a blank line, since after a title line they would run into it. +const PARAGRAPH_BODIES: &[&str] = &["Body\n\nSecond", "*Body*\n\n*Second*", "a\n\n- b", "---"]; + +/// Puts `prefix` before the first line and `rest` before the others. +fn indent(text: &str, first: &str, rest: &str) -> String { + text.lines() + .enumerate() + .map(|(i, line)| { + let prefix = if i == 0 { first } else { rest }; + if line.is_empty() && !prefix.trim().is_empty() && i > 0 { + prefix.trim_end().to_string() + } else { + format!("{prefix}{line}") + } + }) + .collect::<Vec<_>>() + .join("\n") +} + +/// What `body` parses to as the content of a plain quote. +fn body_alone(body: &str) -> Vec<String> { + let nodes = Markdown::from_markdown_str(&indent(body, "> ", "> ")).unwrap().nodes; + match nodes.as_slice() { + [Node::Blockquote(q)] => q.values.iter().map(describe).collect(), + [Node::Callout(c)] => c.values.iter().map(describe).collect(), + other => panic!("{body:?} is not a quote: {other:?}"), + } +} + +/// The callout at the top, or in the first list item. +fn find_callout(nodes: &[Node]) -> Option<&mq_markdown::Callout> { + nodes.iter().find_map(|node| match node { + Node::Callout(c) => Some(c), + Node::List(l) => find_callout(&l.values), + _ => None, + }) +} + +/// Parses a callout made of a header and a body, as a top-level quote and inside a list item, and checks +/// that the title is the header line and the body is what the same text is in a plain quote. +fn check_callout(kind: &str, fold: &str, title_source: &str, separator: &str, body: &str) { + let source = format!("> [!{kind}]{fold} {title_source}{separator}{}", indent(body, "", "> ")); + let title = title_source.trim(); + for source in [source.clone(), indent(&source, "- ", " ")] { + let nodes = Markdown::from_markdown_str(&source).unwrap().nodes; + let callout = find_callout(&nodes).unwrap_or_else(|| panic!("no callout in {source:?}")); + assert_eq!( + callout.title.as_deref(), + (!title.is_empty()).then_some(title), + "title of {source:?}" + ); + assert_eq!(callout.kind, kind, "type of {source:?}"); + assert_eq!(callout.fold, fold.chars().next(), "fold of {source:?}"); + let got: Vec<String> = callout.values.iter().map(describe).collect(); + assert_eq!(got, body_alone(body), "body of {source:?}"); + } +} + +/// Whatever the title is, and however the body is separated from it, the title is the header line and +/// the body is what the same text is in a plain quote. +#[test] +fn the_title_ends_with_the_header_line() { + let separators = [("blank", "\n>\n> "), ("direct", "\n> ")]; + let mut cases = 0; + for kind in ["note", "NOTE", "my-custom_1"] { + for fold in ["", "+", "-"] { + for &(title_source, title) in TITLES { + assert_eq!(title_source, title, "titles are written as they are expected"); + let bodies = BODIES + .iter() + .map(|b| (*b, true)) + .chain(PARAGRAPH_BODIES.iter().map(|b| (*b, false))); + for (body, runs_into_title) in bodies { + for (name, separator) in separators { + if name == "direct" && !runs_into_title { + continue; + } + check_callout(kind, fold, title_source, separator, body); + cases += 1; + } + } + } + } + } + assert!(cases > 500, "only {cases} cases ran"); +} + +proptest! { + /// The same for any type, any title made of characters that mean something in a title, and any number of + /// blank lines between the header and the body. + #[test] + fn any_title_ends_with_the_header_line( + kind in "[a-zA-Z0-9_-]{1,8}", + fold in prop::sample::select(vec!["", "+", "-"]), + title in "[a-zA-Z0-9 *_`~\\[\\]()!<>&#|.\\\\あ🎉-]{0,24}", + body in prop::sample::select(BODIES.iter().chain(PARAGRAPH_BODIES).copied().collect::<Vec<_>>()), + blank_lines in 1usize..4, + ) { + let separator = format!("\n{}> ", ">\n".repeat(blank_lines)); + check_callout(&kind, fold, &title, &separator, body); + } + + /// Without a blank line, a body that is not a paragraph still starts on the line after the title. + #[test] + fn any_title_ends_at_the_line_ending( + title in "[a-zA-Z0-9 *_`~\\[\\]()!<>&#|.\\\\あ🎉-]{0,24}", + body in prop::sample::select(BODIES.to_vec()), + ) { + check_callout("note", "", &title, "\n> ", body); + } +} From 75bea90e84b193ee3ea09cb899d62b6bad24ccc1 Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Thu, 8 Oct 2026 13:06:01 +0900 Subject: [PATCH 11/88] =?UTF-8?q?=E2=9C=A8=20feat(markdown):=20render=20Ob?= =?UTF-8?q?sidian=20syntax=20as=20HTML=20and=20harden=20the=20native=20par?= =?UTF-8?q?ser?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Obsidian syntax in HTML output - Render wikilinks as `<a class="internal-link">`, embeds as img, audio, video, embed or a link by extension, and callouts as `div.callout` or `details.callout` with the classes Obsidian uses. Robustness and performance - Cap inline nesting at 128 levels and report deeper MDX elements as an error, so deeply nested input no longer overflows the stack. - Match emphasis in linear time with a delimiter stack and a linked list, stop scanning for `]]` at the first bracket or line ending, and cap reference labels at 999 characters. - Stop opening containers on a line at the depth limit and decide thematic breaks in constant time from per-line precomputed offsets. Lines of repeated list markers no longer take minutes. Fixes - Keep a dashed rule apart from the paragraph text above it, so it is not read back as a setext heading. - Update the known GFM round trip failures: 16 of them now pass. Types - Add `ListMarker` and `HorizontalRuleMarker` in place of `char` markers. Their JSON form is unchanged. `ListMarker::Continuation` replaces the NUL sentinel used to render the rest of an item after nested items. Tests cover the limits, the Obsidian HTML, the markers and the rules. --- crates/mq-markdown/src/lib.rs | 11 +- crates/mq-markdown/src/markdown.rs | 19 +- crates/mq-markdown/src/markdown/list_order.rs | 9 +- crates/mq-markdown/src/node.rs | 265 +++++++++++++++++- crates/mq-markdown/src/parser/block.rs | 105 ++++--- crates/mq-markdown/src/parser/callout.rs | 12 + crates/mq-markdown/src/parser/inline.rs | 59 +++- .../mq-markdown/src/parser/inline/emphasis.rs | 194 ++++++++++--- crates/mq-markdown/src/parser/inline/link.rs | 25 +- crates/mq-markdown/src/parser/inline/tail.rs | 4 +- crates/mq-markdown/src/parser/line.rs | 21 ++ crates/mq-markdown/src/parser/render_html.rs | 12 + .../src/parser/render_html/obsidian.rs | 173 ++++++++++++ crates/mq-markdown/src/parser/resolve.rs | 5 +- crates/mq-markdown/src/parser/snapshots.txt | 158 +++++------ crates/mq-markdown/src/parser/tree.rs | 7 +- .../tests/gfm_roundtrip_fidelity.rs | 14 +- crates/mq-markdown/tests/obsidian_tests.rs | 113 ++++++++ .../mq-markdown/tests/parser_limits_tests.rs | 118 ++++++++ 19 files changed, 1096 insertions(+), 228 deletions(-) create mode 100644 crates/mq-markdown/src/parser/render_html/obsidian.rs create mode 100644 crates/mq-markdown/tests/parser_limits_tests.rs diff --git a/crates/mq-markdown/src/lib.rs b/crates/mq-markdown/src/lib.rs index 084c61901..b5e995aae 100644 --- a/crates/mq-markdown/src/lib.rs +++ b/crates/mq-markdown/src/lib.rs @@ -13,6 +13,9 @@ //! - **Configurable Rendering**: Customize output formatting and styles //! - **Built-in parser**: CommonMark, GFM (tables, strikethrough, autolink literals, task lists, //! footnotes), frontmatter, math and MDX are parsed, and rendered to HTML, by a parser of this crate. +//! - **Obsidian syntax**: with the `wikilink`, `embed` and `callout` features, `[[links]]`, `![[embeds]]` and +//! `> [!callouts]` are parsed, and rendered to HTML with the classes Obsidian uses (`internal-link`, `callout`). +//! Link targets are used as URLs as written. //! //! ## Quick Start //! @@ -88,10 +91,10 @@ mod parser; pub use markdown::{Markdown, to_html}; pub use node::{ Blockquote, Break, Code, CodeInline, ColorTheme, Definition, Delete, Emphasis, Footnote, FootnoteRef, Fragment, - Heading, HorizontalRule, Html, Image, ImageRef, Link, LinkRef, List, ListStyle, Math, MathInline, - MdxFlowExpression, MdxJsEsm, MdxJsxFlowElement, MdxJsxTextElement, MdxTextExpression, Node, Point, Position, - RenderOptions, Strong, TableAlign, TableAlignKind, TableCell, TableRow, Text, Title, TitleSurroundStyle, Toml, Url, - UrlSurroundStyle, Yaml, attr_value::AttrValue, + Heading, HorizontalRule, HorizontalRuleMarker, Html, Image, ImageRef, Link, LinkRef, List, ListMarker, ListStyle, + Math, MathInline, MdxFlowExpression, MdxJsEsm, MdxJsxFlowElement, MdxJsxTextElement, MdxTextExpression, Node, + Point, Position, RenderOptions, Strong, TableAlign, TableAlignKind, TableCell, TableRow, Text, Title, + TitleSurroundStyle, Toml, Url, UrlSurroundStyle, Yaml, attr_value::AttrValue, }; pub use node_kind::{AttrSpec, AttrType, NodeKind}; diff --git a/crates/mq-markdown/src/markdown.rs b/crates/mq-markdown/src/markdown.rs index f92be4479..c9718fd76 100644 --- a/crates/mq-markdown/src/markdown.rs +++ b/crates/mq-markdown/src/markdown.rs @@ -5,10 +5,9 @@ use crate::html_to_markdown::ConversionOptions; #[cfg(feature = "html-to-markdown")] use crate::node::ListStyle; use crate::node::{ - Code, ColorTheme, Node, Position, RenderOptions, TableAlign, TableCell, indent_lines, list_own_prefix_width, - reindent_all_lines, render_cell_values, render_values_block, + Code, ColorTheme, ListMarker, Node, Position, RenderOptions, TableAlign, TableCell, indent_lines, + list_own_prefix_width, reindent_all_lines, render_cell_values, render_values_block, }; -use list_order::CONTINUATION; #[cfg(any(feature = "json", feature = "html-to-markdown"))] use miette::miette; use std::{fmt, str::FromStr}; @@ -157,7 +156,7 @@ impl Markdown { let prev_node = i.checked_sub(1).and_then(|j| nodes.get(j)); let value = if let Node::List(list) = node - && list.marker == Some(CONTINUATION) + && list.marker == Some(ListMarker::Continuation) { // The rest of an item after its nested items, aligned with the content of the item. list_indent_stack.truncate(list.level as usize + 1); @@ -205,6 +204,15 @@ impl Markdown { new_line_count = 2; } + // A line of dashes right under paragraph text reads back as a setext heading. + if new_line_count < 2 + && value.starts_with("---") + && matches!(node, Node::HorizontalRule(_)) + && prev_node.is_some_and(Node::is_paragraph_text) + { + new_line_count = 2; + } + // A footnote ends where the next line starts, so a block after it needs a blank line. if new_line_count < 2 && matches!(prev_node, Some(Node::Footnote(_))) @@ -519,6 +527,9 @@ mod tests { #[case::fence_switches_to_tilde_on_backtick_in_info("~~~ aa ``` ~~~\nfoo\n~~~", 1, "~~~aa ``` ~~~\nfoo\n~~~\n")] // a sibling right after a block quote needs a blank line, or it's read as lazy continuation #[case::blockquote_sibling_gets_blank_line_separator("> bar\n>\nbaz", 2, "> bar\n\nbaz\n")] + #[case::dash_rule_under_text_stays_a_rule("Foo\n--- -\n", 2, "Foo\n\n---\n")] + #[case::dash_rule_under_emphasis_stays_a_rule("*Foo*\n- - -\n", 2, "*Foo*\n\n---\n")] + #[case::star_rule_under_text_needs_no_blank_line("Foo\n***\n", 2, "Foo\n***\n")] // items whose marker sits alone on its own line must stay tight, not go loose #[case::list_tight_despite_marker_on_own_line("-\n foo\n-\n bar", 2, "- foo\n- bar\n")] // an indented code block right after a list must not read back as list continuation diff --git a/crates/mq-markdown/src/markdown/list_order.rs b/crates/mq-markdown/src/markdown/list_order.rs index 969c2bf97..115ce3eec 100644 --- a/crates/mq-markdown/src/markdown/list_order.rs +++ b/crates/mq-markdown/src/markdown/list_order.rs @@ -4,11 +4,7 @@ //! that item come after them in the source, like a paragraph that follows a nested list. Rendering //! the nodes as they are would write those children before the nested items. -use crate::node::{List, Node, Position}; - -/// The marker of a node that holds the children of an item that follow items nested in it. Such a -/// node is rendered as its content, indented to the content of the item, without a marker. -pub(super) const CONTINUATION: char = '\0'; +use crate::node::{List, ListMarker, Node, Position}; /// Returns the nodes in source order, or `None` when they already are. pub(super) fn reorder(nodes: &[Node]) -> Option<Vec<Node>> { @@ -110,7 +106,8 @@ fn continuation(item: &List, values: &[Node]) -> Node { Node::List(List { values: values.to_vec(), position: span(values), - marker: Some(CONTINUATION), + // Rendered as its content, indented to the content of the item, without a marker. + marker: Some(ListMarker::Continuation), ..item.clone() }) } diff --git a/crates/mq-markdown/src/node.rs b/crates/mq-markdown/src/node.rs index a2c90a80e..87f8933af 100644 --- a/crates/mq-markdown/src/node.rs +++ b/crates/mq-markdown/src/node.rs @@ -172,6 +172,114 @@ impl Display for ListStyle { } } +/// The marker of a list item as it is written in the source. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[cfg_attr(feature = "json", derive(serde::Serialize, serde::Deserialize))] +pub enum ListMarker { + /// `-`, a bullet. + #[cfg_attr(feature = "json", serde(rename = "-"))] + Dash, + /// `+`, a bullet. + #[cfg_attr(feature = "json", serde(rename = "+"))] + Plus, + /// `*`, a bullet. + #[cfg_attr(feature = "json", serde(rename = "*"))] + Star, + /// `.` after the number of an ordered item. + #[cfg_attr(feature = "json", serde(rename = "."))] + Period, + /// `)` after the number of an ordered item. + #[cfg_attr(feature = "json", serde(rename = ")"))] + Paren, + /// What the rest of an item that follows the items nested in it is rendered with. It has no marker + /// of its own, and the parser never produces it. + #[doc(hidden)] + #[cfg_attr(feature = "json", serde(rename = "\0"))] + Continuation, +} + +impl ListMarker { + /// The character the marker is written with, or `None` for [`ListMarker::Continuation`]. + pub fn as_char(self) -> Option<char> { + match self { + Self::Dash => Some('-'), + Self::Plus => Some('+'), + Self::Star => Some('*'), + Self::Period => Some('.'), + Self::Paren => Some(')'), + Self::Continuation => None, + } + } + + /// The marker written with `char`: `-`, `+`, `*`, `.` or `)`. + pub fn from_char(char: char) -> Option<Self> { + match char { + '-' => Some(Self::Dash), + '+' => Some(Self::Plus), + '*' => Some(Self::Star), + '.' => Some(Self::Period), + ')' => Some(Self::Paren), + _ => None, + } + } + + /// The bullet style of the marker, if it is a bullet. + pub fn style(self) -> Option<ListStyle> { + match self { + Self::Dash => Some(ListStyle::Dash), + Self::Plus => Some(ListStyle::Plus), + Self::Star => Some(ListStyle::Star), + Self::Period | Self::Paren | Self::Continuation => None, + } + } +} + +impl From<ListStyle> for ListMarker { + fn from(style: ListStyle) -> Self { + match style { + ListStyle::Dash => Self::Dash, + ListStyle::Plus => Self::Plus, + ListStyle::Star => Self::Star, + } + } +} + +/// The character a horizontal rule is written with in the source. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[cfg_attr(feature = "json", derive(serde::Serialize, serde::Deserialize))] +pub enum HorizontalRuleMarker { + /// `*` + #[cfg_attr(feature = "json", serde(rename = "*"))] + Star, + /// `-` + #[cfg_attr(feature = "json", serde(rename = "-"))] + Dash, + /// `_` + #[cfg_attr(feature = "json", serde(rename = "_"))] + Underscore, +} + +impl HorizontalRuleMarker { + /// The character the rule is written with. + pub fn as_char(self) -> char { + match self { + Self::Star => '*', + Self::Dash => '-', + Self::Underscore => '_', + } + } + + /// The marker written with `char`: `*`, `-` or `_`. + pub fn from_char(char: char) -> Option<Self> { + match char { + '*' => Some(Self::Star), + '-' => Some(Self::Dash), + '_' => Some(Self::Underscore), + _ => None, + } + } +} + #[derive(Debug, Clone, PartialEq, Default)] #[cfg_attr( feature = "json", @@ -290,9 +398,9 @@ pub struct List { /// Starting number for an ordered list (e.g. `5` in `5. foo`); `None` means 1. #[cfg_attr(feature = "json", serde(skip_serializing_if = "Option::is_none"))] pub start: Option<u32>, - /// Source marker: `-`, `+` or `*` for bullets, `.` or `)` for ordered lists. + /// Source marker. `None` keeps the default of the list kind: `-` for bullets, `.` for ordered lists. #[cfg_attr(feature = "json", serde(default, skip_serializing_if = "Option::is_none"))] - pub marker: Option<char>, + pub marker: Option<ListMarker>, #[cfg_attr(feature = "json", serde(skip_serializing_if = "Option::is_none"))] pub position: Option<Position>, } @@ -771,9 +879,9 @@ pub struct Break { serde(rename_all = "camelCase", tag = "type") )] pub struct HorizontalRule { - /// Source marker: `*`, `-` or `_`. + /// Source marker. `None` renders as `***`. #[cfg_attr(feature = "json", serde(default, skip_serializing_if = "Option::is_none"))] - pub marker: Option<char>, + pub marker: Option<HorizontalRuleMarker>, #[cfg_attr(feature = "json", serde(skip_serializing_if = "Option::is_none"))] pub position: Option<Position>, } @@ -1089,15 +1197,12 @@ impl Node { .. }) => { let marker = if *ordered { - let delimiter = marker.filter(|c| *c == ')').unwrap_or('.'); + let delimiter = if *marker == Some(ListMarker::Paren) { ')' } else { '.' }; format!("{}{}", start.unwrap_or(1) as usize + *index, delimiter) } else if let Some(style) = &options.list_style { style.to_string() } else { - marker - .filter(|c| matches!(c, '-' | '+' | '*')) - .unwrap_or('-') - .to_string() + marker.and_then(ListMarker::style).unwrap_or_default().to_string() }; let checkbox = (*checked).map(|it| if it { "[x] " } else { "[ ] " }).unwrap_or(""); let prefix_width = *level as usize * 2 + list_own_prefix_width(*ordered, *index, *start); @@ -1454,7 +1559,11 @@ impl Node { Self::Break(_) => "\\\n".to_string(), Self::HorizontalRule(HorizontalRule { marker, .. }) => { let (hs, he) = &theme.horizontal_rule; - let marker = marker.filter(|c| matches!(c, '-' | '_')).unwrap_or('*'); + let marker = match marker { + Some(HorizontalRuleMarker::Dash) => '-', + Some(HorizontalRuleMarker::Underscore) => '_', + _ => '*', + }; format!("{hs}{marker}{marker}{marker}{he}") } Self::Fragment(Fragment { values }) => values @@ -1860,6 +1969,24 @@ impl Node { matches!(self, Self::Blockquote(_)) } + /// True for the nodes that make up the text of a paragraph. + pub(crate) fn is_paragraph_text(&self) -> bool { + matches!( + self, + Self::Text(_) + | Self::Emphasis(_) + | Self::Strong(_) + | Self::Delete(_) + | Self::CodeInline(_) + | Self::MathInline(_) + | Self::Link(_) + | Self::LinkRef(_) + | Self::Image(_) + | Self::ImageRef(_) + | Self::FootnoteRef(_) + ) + } + /// True for block quotes and (when enabled) Obsidian-style callouts. pub(crate) fn is_blockquote_like(&self) -> bool { match self { @@ -3458,6 +3585,124 @@ mod tests { use proptest::prelude::*; use rstest::rstest; + #[rstest] + #[case::dash('-', Some(ListMarker::Dash))] + #[case::plus('+', Some(ListMarker::Plus))] + #[case::star('*', Some(ListMarker::Star))] + #[case::period('.', Some(ListMarker::Period))] + #[case::paren(')', Some(ListMarker::Paren))] + #[case::underscore('_', None)] + #[case::digit('1', None)] + fn list_marker_from_char(#[case] char: char, #[case] expected: Option<ListMarker>) { + assert_eq!(ListMarker::from_char(char), expected); + assert_eq!(expected.and_then(ListMarker::as_char), expected.map(|_| char)); + } + + #[rstest] + #[case::dash(ListMarker::Dash, Some(ListStyle::Dash))] + #[case::plus(ListMarker::Plus, Some(ListStyle::Plus))] + #[case::star(ListMarker::Star, Some(ListStyle::Star))] + #[case::period(ListMarker::Period, None)] + #[case::paren(ListMarker::Paren, None)] + #[case::continuation(ListMarker::Continuation, None)] + fn list_marker_style(#[case] marker: ListMarker, #[case] expected: Option<ListStyle>) { + assert_eq!(marker.style(), expected); + } + + #[rstest] + #[case::dash(ListStyle::Dash, ListMarker::Dash)] + #[case::plus(ListStyle::Plus, ListMarker::Plus)] + #[case::star(ListStyle::Star, ListMarker::Star)] + fn list_marker_from_style(#[case] style: ListStyle, #[case] expected: ListMarker) { + assert_eq!(ListMarker::from(style), expected); + } + + #[rstest] + #[case::star('*', Some(HorizontalRuleMarker::Star))] + #[case::dash('-', Some(HorizontalRuleMarker::Dash))] + #[case::underscore('_', Some(HorizontalRuleMarker::Underscore))] + #[case::plus('+', None)] + fn horizontal_rule_marker_from_char(#[case] char: char, #[case] expected: Option<HorizontalRuleMarker>) { + assert_eq!(HorizontalRuleMarker::from_char(char), expected); + assert_eq!(expected.map(HorizontalRuleMarker::as_char), expected.map(|_| char)); + } + + #[rstest] + #[case::bullet_dash(false, Some(ListMarker::Dash), "- a")] + #[case::bullet_plus(false, Some(ListMarker::Plus), "+ a")] + #[case::bullet_star(false, Some(ListMarker::Star), "* a")] + #[case::bullet_unknown(false, None, "- a")] + #[case::bullet_with_an_ordered_marker(false, Some(ListMarker::Paren), "- a")] + #[case::ordered_period(true, Some(ListMarker::Period), "1. a")] + #[case::ordered_paren(true, Some(ListMarker::Paren), "1) a")] + #[case::ordered_unknown(true, None, "1. a")] + #[case::ordered_with_a_bullet_marker(true, Some(ListMarker::Star), "1. a")] + fn list_renders_its_marker(#[case] ordered: bool, #[case] marker: Option<ListMarker>, #[case] expected: &str) { + let node = Node::List(List { + index: 0, + level: 0, + checked: None, + values: vec![Node::Text(Text { + value: "a".into(), + position: None, + })], + ordered, + start: None, + spread: false, + marker, + position: None, + }); + assert_eq!( + node.render_with_theme(&RenderOptions::default(), &ColorTheme::PLAIN), + expected + ); + } + + #[rstest] + #[case::star(Some(HorizontalRuleMarker::Star), "***")] + #[case::dash(Some(HorizontalRuleMarker::Dash), "---")] + #[case::underscore(Some(HorizontalRuleMarker::Underscore), "___")] + #[case::unknown(None, "***")] + fn horizontal_rule_renders_its_marker(#[case] marker: Option<HorizontalRuleMarker>, #[case] expected: &str) { + let node = Node::HorizontalRule(HorizontalRule { marker, position: None }); + assert_eq!( + node.render_with_theme(&RenderOptions::default(), &ColorTheme::PLAIN), + expected + ); + } + + /// The JSON of the markers is the character they are written with, as it was when they were chars. + #[cfg(feature = "json")] + #[rstest] + #[case::dash(ListMarker::Dash, "\"-\"")] + #[case::plus(ListMarker::Plus, "\"+\"")] + #[case::star(ListMarker::Star, "\"*\"")] + #[case::period(ListMarker::Period, "\".\"")] + #[case::paren(ListMarker::Paren, "\")\"")] + fn list_marker_json(#[case] marker: ListMarker, #[case] expected: &str) { + assert_eq!(serde_json::to_string(&marker).unwrap(), expected); + assert_eq!(serde_json::from_str::<ListMarker>(expected).unwrap(), marker); + } + + #[cfg(feature = "json")] + #[rstest] + #[case::star(HorizontalRuleMarker::Star, "\"*\"")] + #[case::dash(HorizontalRuleMarker::Dash, "\"-\"")] + #[case::underscore(HorizontalRuleMarker::Underscore, "\"_\"")] + fn horizontal_rule_marker_json(#[case] marker: HorizontalRuleMarker, #[case] expected: &str) { + assert_eq!(serde_json::to_string(&marker).unwrap(), expected); + assert_eq!(serde_json::from_str::<HorizontalRuleMarker>(expected).unwrap(), marker); + } + + #[cfg(feature = "json")] + #[test] + fn list_json_keeps_the_marker_as_a_string() { + let json = r#"{"type":"list","values":[],"index":0,"level":0,"ordered":false,"checked":null,"spread":false,"marker":"+"}"#; + let list = serde_json::from_str::<List>(json).unwrap(); + assert_eq!(list.marker, Some(ListMarker::Plus)); + assert!(serde_json::to_string(&list).unwrap().contains(r#""marker":"+""#)); + } + #[test] fn map_values_into_transforms_owned_fragments() { let node = Node::Fragment(Fragment { diff --git a/crates/mq-markdown/src/parser/block.rs b/crates/mq-markdown/src/parser/block.rs index 557babc9d..81d6882ab 100644 --- a/crates/mq-markdown/src/parser/block.rs +++ b/crates/mq-markdown/src/parser/block.rs @@ -14,7 +14,9 @@ use super::mdx_flow::{ }; use super::table; use super::tree::{Block, FootnoteBlock, InlineBlock, InlineKind, InlineSource, Item, ListBlock, QuoteBlock}; -use crate::node::{HorizontalRule, Html, MdxJsEsm, Node, Point, Position, Toml, Yaml}; +use crate::node::{ + HorizontalRule, HorizontalRuleMarker, Html, ListMarker, MdxJsEsm, Node, Point, Position, Toml, Yaml, +}; /// Lines indented by this many columns or more are code, not other blocks. pub(super) const CODE_INDENT: usize = 4; @@ -118,9 +120,9 @@ fn parse_blocks(lines: &[Line<'_>], depth: usize, interrupting: Interrupt) -> Ve } else if let Some(depth) = atx_depth(rest) { blocks.push(atx_heading(line, indent, depth)); index += 1; - } else if is_thematic_break(rest) { + } else if is_thematic_break(line, indent) { blocks.push(Block::Node(Node::HorizontalRule(HorizontalRule { - marker: rest.chars().next(), + marker: rest.chars().next().and_then(HorizontalRuleMarker::from_char), position: Some(Position { start: line.point(0), end: line.end(), @@ -131,7 +133,7 @@ fn parse_blocks(lines: &[Line<'_>], depth: usize, interrupting: Interrupt) -> Ve index = blockquote(lines, index, depth, interrupting, &mut blocks); after_container = true; } else if let Some(marker) = - ListMarker::parse(line).filter(|marker| containers && (!interrupting.list || marker.interrupts_paragraph())) + ItemMarker::parse(line).filter(|marker| containers && (!interrupting.list || marker.interrupts_paragraph())) { index = list(lines, index, &marker, depth, interrupting, &mut blocks); after_container = true; @@ -198,10 +200,13 @@ struct LeafState<'a> { paragraph_containers: usize, /// Restrictions on the next fed line, the first line of a container that interrupted a paragraph. restricted: Interrupt, + /// How many more containers a line can open before the depth limit, which no longer open any. + budget: usize, } impl<'a> LeafState<'a> { - fn new(restricted: Interrupt) -> Self { + /// `depth` is the depth of the container that collects the lines. + fn new(restricted: Interrupt, depth: usize) -> Self { Self { fence: None, html: None, @@ -210,6 +215,7 @@ impl<'a> LeafState<'a> { paragraph: false, paragraph_containers: 0, restricted, + budget: MAX_DEPTH.saturating_sub(depth + 1), } } @@ -288,7 +294,7 @@ impl<'a> LeafState<'a> { self.paragraph = false; return; } - if atx_depth(rest).is_some() || is_thematic_break(rest) { + if atx_depth(rest).is_some() || is_thematic_break(&line, indent) { self.paragraph = false; return; } @@ -301,16 +307,20 @@ impl<'a> LeafState<'a> { self.paragraph = false; return; } - if let Some(stripped) = strip_blockquote(line) { + if containers < self.budget + && let Some(stripped) = strip_blockquote(line) + { line = stripped; containers += 1; continue; } - if let Some(marker) = ListMarker::parse(&line).filter(|m| { - // A marker in or inside the container of the open paragraph has to be able to interrupt it. - let continues_paragraph = open && containers >= self.paragraph_containers; - !(restricted.list || continues_paragraph) || m.interrupts_paragraph() - }) { + if containers < self.budget + && let Some(marker) = ItemMarker::parse(&line).filter(|m| { + // A marker in or inside the container of the open paragraph has to be able to interrupt it. + let continues_paragraph = open && containers >= self.paragraph_containers; + !(restricted.list || continues_paragraph) || m.interrupts_paragraph() + }) + { line = marker.content(line); containers += 1; // The rest of the line starts a new item. @@ -368,7 +378,7 @@ fn blockquote( blocks: &mut Vec<Block>, ) -> usize { let mut inner = Vec::new(); - let mut state = LeafState::new(interrupting); + let mut state = LeafState::new(interrupting, depth); let mut index = start; while let Some(line) = lines.get(index) { @@ -402,10 +412,9 @@ fn blockquote( index } -struct ListMarker { +struct ItemMarker { ordered: bool, - /// `-`, `+` or `*` for bullets, `.` or `)` for ordered lists. - delimiter: u8, + kind: ListMarker, start: u32, /// Bytes from the start of the line to the end of the marker. after_marker: usize, @@ -417,7 +426,7 @@ struct ListMarker { empty: bool, } -impl ListMarker { +impl ItemMarker { fn parse(line: &Line<'_>) -> Option<Self> { let (columns, indent) = line.indent(); if columns >= line.code_indent() { @@ -425,7 +434,7 @@ impl ListMarker { } let rest = &line.text[indent..]; let bytes = rest.as_bytes(); - if is_thematic_break(rest) { + if is_thematic_break(line, indent) { return None; } @@ -440,7 +449,13 @@ impl ListMarker { } _ => return None, }; - let delimiter = bytes[marker_len - 1]; + let kind = match bytes[marker_len - 1] { + b'-' => ListMarker::Dash, + b'+' => ListMarker::Plus, + b'*' => ListMarker::Star, + b'.' => ListMarker::Period, + _ => ListMarker::Paren, + }; let after = &rest[marker_len..]; let (spaces, blanks) = line.skip(indent + marker_len).indent(); @@ -458,7 +473,7 @@ impl ListMarker { Some(Self { ordered, - delimiter, + kind, start, after_marker: indent + marker_len, gap, @@ -477,7 +492,7 @@ impl ListMarker { } fn same_list(&self, other: &Self) -> bool { - self.ordered == other.ordered && self.delimiter == other.delimiter + self.ordered == other.ordered && self.kind == other.kind } /// Whether this marker may interrupt a paragraph. @@ -489,7 +504,7 @@ impl ListMarker { fn list( lines: &[Line<'_>], start: usize, - first: &ListMarker, + first: &ItemMarker, depth: usize, interrupting: Interrupt, blocks: &mut Vec<Block>, @@ -499,7 +514,7 @@ fn list( let mut index = start; loop { - let marker = ListMarker::parse(&lines[index]).unwrap_or_else(|| unreachable!("checked by the caller")); + let marker = ItemMarker::parse(&lines[index]).unwrap_or_else(|| unreachable!("checked by the caller")); let (item, end) = list_item( lines, index, @@ -515,7 +530,7 @@ fn list( // Blank lines between items make the list loose. let next = (end..lines.len()).find(|&i| !lines[i].is_blank()); - match next.filter(|&i| ListMarker::parse(&lines[i]).is_some_and(|m| m.same_list(first))) { + match next.filter(|&i| ItemMarker::parse(&lines[i]).is_some_and(|m| m.same_list(first))) { Some(next) => { spread |= next > end; index = next; @@ -530,7 +545,7 @@ fn list( blocks.push(Block::List(ListBlock { ordered: first.ordered, start: first.ordered.then_some(first.start), - marker: first.delimiter as char, + marker: first.kind, spread, items, })); @@ -613,13 +628,13 @@ fn extend_quote(quote: &mut QuoteBlock, end: &Point) { fn list_item( lines: &[Line<'_>], start: usize, - marker: &ListMarker, + marker: &ItemMarker, depth: usize, interrupting: Interrupt, ) -> (Item, usize) { let first = marker.content(lines[start]); let mut inner = vec![first]; - let mut state = LeafState::new(interrupting); + let mut state = LeafState::new(interrupting, depth); state.feed(&first); let mut blanks = Vec::new(); let mut last_blank_has_whitespace = false; @@ -664,7 +679,7 @@ fn list_item( // another container, but not when a plain paragraph or thematic break follows. let next = lines.get(index); let item_end = next - .is_none_or(|line| line.is_blank() || blockquote_marker(line).is_some() || ListMarker::parse(line).is_some()); + .is_none_or(|line| line.is_blank() || blockquote_marker(line).is_some() || ItemMarker::parse(line).is_some()); if let Some(last) = inner.last_mut() { last.item_end = item_end; } @@ -778,19 +793,21 @@ fn offset_in(parent: &str, child: &str) -> usize { child.as_ptr() as usize - parent.as_ptr() as usize } -fn is_thematic_break(rest: &str) -> bool { - let Some(marker) = rest.chars().next().filter(|c| matches!(c, '*' | '-' | '_')) else { - return false; +/// Whether the rest of `line` after `indent` bytes is a thematic break. +fn is_thematic_break(line: &Line<'_>, indent: usize) -> bool { + let rest = &line.text[indent..]; + let marker = match rest.as_bytes().first() { + Some(b'*') => 0, + Some(b'-') => 1, + Some(b'_') => 2, + _ => return false, }; - let mut count = 0; - for c in rest.chars() { - if c == marker { - count += 1; - } else if !matches!(c, ' ' | '\t') { - return false; - } + // Anything else after this point rules it out without looking at the rest of the line. + if line.others[marker] > line.column + indent { + return false; } - count >= 3 + let marker = rest.as_bytes()[0]; + rest.bytes().filter(|&byte| byte == marker).take(3).count() == 3 } /// Returns the setext heading depth when `line` is a setext underline. @@ -811,7 +828,7 @@ fn setext_depth(line: &Line<'_>) -> Option<u8> { /// Whether a line without its container prefix can continue an open paragraph of that container. /// Any list marker ends the paragraph here, even one that could not interrupt it elsewhere. fn is_lazy_continuation(line: &Line<'_>) -> bool { - !interrupts_paragraph(line) && ListMarker::parse(line).is_none() + !interrupts_paragraph(line) && ItemMarker::parse(line).is_none() } /// Whether `line` ends the paragraph that is being collected. @@ -828,18 +845,18 @@ fn interrupts_paragraph(line: &Line<'_>) -> bool { columns < line.code_indent() && (Fence::open(rest, !line.mdx).is_some() || atx_depth(rest).is_some() - || is_thematic_break(rest) + || is_thematic_break(line, indent) || blockquote_marker(line).is_some() || footnote_marker(line).is_some() || html_start(line, rest).is_some_and(|kind| kind != HtmlKind::Complete) - || ListMarker::parse(line).is_some_and(|marker| marker.interrupts_paragraph())) + || ItemMarker::parse(line).is_some_and(|marker| marker.interrupts_paragraph())) } /// Whether `line` ends a table. Unlike a paragraph, a table is also ended by a list item that is empty /// or that is numbered from other than one. fn ends_table(line: &Line<'_>) -> bool { interrupts_paragraph(line) - || (!line.lazy && line.indent().0 < line.code_indent() && ListMarker::parse(line).is_some()) + || (!line.lazy && line.indent().0 < line.code_indent() && ItemMarker::parse(line).is_some()) } /// Returns the index after the block and what it restricts on the first line of a following container. @@ -1012,7 +1029,7 @@ fn footnote( ) -> usize { let first = lines[start].skip(marker.content); let mut inner = vec![first]; - let mut state = LeafState::new(interrupting); + let mut state = LeafState::new(interrupting, depth); state.feed(&first); let mut blanks = Vec::new(); let mut index = start + 1; diff --git a/crates/mq-markdown/src/parser/callout.rs b/crates/mq-markdown/src/parser/callout.rs index d756b5a8c..449756ca2 100644 --- a/crates/mq-markdown/src/parser/callout.rs +++ b/crates/mq-markdown/src/parser/callout.rs @@ -49,3 +49,15 @@ pub(super) fn take_header(children: &mut Vec<Block>) -> Option<Header> { } Some(header) } + +/// The header of a callout at the start of `children`, and what remains of the first paragraph after it. +/// Unlike [`take_header`] this leaves `children` as it is. +pub(super) fn peek_header(children: &[Block]) -> Option<(Header, Vec<Block>)> { + let Block::Inline(inline) = children.first()? else { + return None; + }; + split(&inline.source.text)?; + let mut head = vec![Block::Inline(inline.clone())]; + let header = take_header(&mut head)?; + Some((header, head)) +} diff --git a/crates/mq-markdown/src/parser/inline.rs b/crates/mq-markdown/src/parser/inline.rs index ff9f3ca7b..6c4fdaea9 100644 --- a/crates/mq-markdown/src/parser/inline.rs +++ b/crates/mq-markdown/src/parser/inline.rs @@ -51,8 +51,6 @@ pub(super) struct Opener { pub(super) image: bool, /// Offset of the `[`, or of the `!` of an image. pub(super) start: usize, - /// Links cannot contain links, so openers before a link are deactivated. - pub(super) active: bool, } pub(super) enum Item { @@ -61,7 +59,8 @@ pub(super) enum Item { end: usize, value: Value, }, - Node(Node), + /// A finished node and how deeply it nests: a leaf is 1. + Node(Node, usize), Delim(Delim), Open(Opener), /// An email address in plain text. It is not linked inside link text. @@ -83,6 +82,18 @@ pub(super) struct TextTag { end: usize, } +/// Emphasis and links nested deeper than this are text, and JSX elements are an error, which bounds the recursion of everything that +/// walks the nodes. +pub(super) const MAX_NESTING: usize = 128; + +/// How deeply the item nests: nodes carry it, and the rest are text. +pub(super) fn item_depth(item: &Item) -> usize { + match item { + Item::Node(_, depth) => *depth, + _ => 0, + } +} + /// What the scanner and its helpers share. #[derive(Clone, Copy)] pub(super) struct Context<'a> { @@ -129,6 +140,8 @@ pub(super) fn parse(source: &InlineSource, references: &References) -> Result<Ve context: &context, items: Vec::new(), openers: Vec::new(), + depths: Vec::new(), + inactive_below: 0, pos: 0, run: 0, }; @@ -136,7 +149,7 @@ pub(super) fn parse(source: &InlineSource, references: &References) -> Result<Ve scanner.flush(); let Scanner { mut items, .. } = scanner; - emphasis::process(&mut items, 0, &context); + emphasis::process(&mut items, &context); let nodes = to_nodes(items, &context); match error.into_inner() { @@ -189,7 +202,7 @@ pub(super) fn to_nodes(items: Vec<Item>, context: &Context<'_>) -> Vec<Node> { let end = opener.start + if opener.image { 2 } else { 1 }; (opener.start, end, Cow::Borrowed(&src[opener.start..end])) } - Item::Node(node) => { + Item::Node(node, _) => { finish(&mut current, &mut nodes); nodes.push(node); continue; @@ -197,6 +210,9 @@ pub(super) fn to_nodes(items: Vec<Item>, context: &Context<'_>) -> Vec<Node> { Item::Jsx(tag) => { finish(&mut current, &mut nodes); match tag.kind { + TagKind::Open if open.len() >= MAX_NESTING => { + context.fail(format!("Elements are nested deeper than {MAX_NESTING} levels")); + } TagKind::Open => open.push((tag, std::mem::take(&mut nodes))), TagKind::SelfClosing => { let end = tag.end; @@ -277,6 +293,10 @@ struct Scanner<'a> { items: Vec<Item>, /// Indexes of the `Item::Open` items that are still waiting for a `]`. openers: Vec<usize>, + /// For each opener, the deepest node pushed inside it so far. + depths: Vec<usize>, + /// Links cannot contain links, so the `[` openers below this many are deactivated. + inactive_below: usize, pos: usize, /// Start of the plain text that has not been pushed as an item yet. run: usize, @@ -356,11 +376,12 @@ impl Scanner<'_> { [b'[', b'[', ..] if cfg!(feature = "wikilink") => (false, start + 2), _ => return None, }; - let close = open + src[open..].find("]]")?; - let content = &src[open..close]; - if content.contains(['[', ']', '\n', '\r']) { + // The content holds no bracket or line ending, so the first one found must start the `]]`. + let close = open + src[open..].find(['[', ']', '\n', '\r'])?; + if !src[close..].starts_with("]]") { return None; } + let content = &src[open..close]; let (target, label) = match content.split_once('|') { Some((target, label)) => (target.trim(), Some(label.trim().to_string())), None => (content.trim(), None), @@ -417,13 +438,23 @@ impl Scanner<'_> { /// Pushes `item` after the pending text and continues scanning at `end`. fn push(&mut self, item: Item, end: usize) { self.flush(); + if let Item::Node(_, depth) = &item { + self.note_depth(*depth); + } self.items.push(item); self.pos = end; self.run = end; } + /// Records that a node of `depth` is inside the innermost open bracket. + fn note_depth(&mut self, depth: usize) { + if let Some(innermost) = self.depths.last_mut() { + *innermost = (*innermost).max(depth); + } + } + fn push_node(&mut self, node: Node, end: usize) { - self.push(Item::Node(node), end); + self.push(Item::Node(node, 1), end); } fn escape(&mut self) { @@ -493,7 +524,8 @@ impl Scanner<'_> { let node = Node::Break(Break { position: Some(self.context.position(start, pos + eol)), }); - self.items.push(Item::Node(node)); + self.note_depth(1); + self.items.push(Item::Node(node, 1)); } else { self.items.push(Item::Text { start: pos, @@ -591,14 +623,11 @@ impl Scanner<'_> { } fn open(&mut self, image: bool) { - let opener = Opener { - image, - start: self.pos, - active: true, - }; + let opener = Opener { image, start: self.pos }; let end = self.pos + if image { 2 } else { 1 }; self.flush(); self.openers.push(self.items.len()); + self.depths.push(0); self.items.push(Item::Open(opener)); self.pos = end; self.run = end; diff --git a/crates/mq-markdown/src/parser/inline/emphasis.rs b/crates/mq-markdown/src/parser/inline/emphasis.rs index 69dad388a..96e18af88 100644 --- a/crates/mq-markdown/src/parser/inline/emphasis.rs +++ b/crates/mq-markdown/src/parser/inline/emphasis.rs @@ -4,7 +4,7 @@ //! to other runs open and close, and in using the remaining length of a run for the rule of three. use super::punctuation::is_punctuation; -use super::{Context, Delim, Item, to_nodes}; +use super::{Context, Delim, Item, MAX_NESTING, item_depth, to_nodes}; use crate::node::{Delete, Emphasis, Node, Strong}; #[derive(PartialEq)] @@ -57,38 +57,131 @@ fn matches(opener: &Delim, closer: &Delim) -> bool { !(closer.ch == b'~' && (closer.count != opener.count || closer.count > 2)) } -/// Matches delimiters in `items[bottom..]`, replacing each pair and what is between them by a node. -pub(super) fn process(items: &mut Vec<Item>, bottom: usize, context: &Context<'_>) { - let mut closer = bottom; +/// A node of the list the delimiters are matched in, which lets matched items be taken out between +/// two delimiters without moving the items after them. +struct Slot { + item: Option<Item>, + prev: usize, + next: usize, +} + +const NIL: usize = usize::MAX; + +/// What decides whether a closer can match an opener: the openers a closer with the same key +/// rejected once are rejected for every later closer with it. +fn key(closer: &Delim) -> usize { + let ch = match closer.ch { + b'*' => 0, + b'_' => 1, + _ => 2, + }; + // Strikethrough also goes by the length of the run. + let count = if ch == 2 { closer.count.min(2) } else { 0 }; + ((ch * 2 + usize::from(closer.can_open)) * 3 + closer.original % 3) * 3 + count +} + +fn delim(slots: &[Slot], index: usize) -> &Delim { + match &slots[index].item { + Some(Item::Delim(delim)) => delim, + _ => unreachable!("the index is a delimiter"), + } +} + +fn delim_mut(slots: &mut [Slot], index: usize) -> &mut Delim { + match &mut slots[index].item { + Some(Item::Delim(delim)) => delim, + _ => unreachable!("the index is a delimiter"), + } +} + +fn unlink(slots: &mut [Slot], index: usize) { + let (prev, next) = (slots[index].prev, slots[index].next); + slots[prev].next = next; + if next != NIL { + slots[next].prev = prev; + } + slots[index].item = None; +} + +/// Matches the delimiters in `items`, replacing each pair and what is between them by a node. +/// +/// The openers that can still match are kept on a stack, and the openers a closer cannot match are +/// remembered by the offset they start at, so that the time is linear in the number of items. +pub(super) fn process(items: &mut Vec<Item>, context: &Context<'_>) { + if !items.iter().any(|item| matches!(item, Item::Delim(_))) { + return; + } + + let mut slots = Vec::with_capacity(items.len() + 1); + let total = items.len(); + slots.push(Slot { + item: None, + prev: NIL, + next: 1, + }); + for (index, item) in items.drain(..).enumerate() { + slots.push(Slot { + item: Some(item), + prev: index, + next: if index + 1 == total { NIL } else { index + 2 }, + }); + } + + let mut stack: Vec<usize> = Vec::new(); + let mut bottoms = [0usize; 54]; + let mut closer = slots[0].next; - while closer < items.len() { - let Item::Delim(current) = &items[closer] else { - closer += 1; + while closer != NIL { + if !matches!(slots[closer].item, Some(Item::Delim(_))) { + closer = slots[closer].next; continue; - }; - let opener = if current.can_close { - (bottom..closer).rev().find(|&index| match &items[index] { - Item::Delim(candidate) => matches(candidate, current), - _ => false, - }) - } else { - None - }; - let Some(opener) = opener else { - closer += 1; + } + let current = delim(&slots, closer); + let (can_open, can_close, key) = (current.can_open, current.can_close, key(current)); + + let mut found = None; + if can_close { + for at in (0..stack.len()).rev() { + let candidate = delim(&slots, stack[at]); + if candidate.start < bottoms[key] { + break; + } + if matches(candidate, current) { + found = Some(at); + break; + } + } + } + let found = found.filter(|&at| fits(&slots, stack[at], closer)); + + let Some(at) = found else { + if can_close { + bottoms[key] = delim(&slots, closer).start; + } + if can_open { + stack.push(closer); + } + closer = slots[closer].next; continue; }; - let (open, close) = match (&items[opener], &items[closer]) { - (Item::Delim(open), Item::Delim(close)) => (open, close), - _ => unreachable!("both indexes are delimiters"), - }; + let opener = stack[at]; + let (open, close) = (delim(&slots, opener), delim(&slots, closer)); let used = if open.count > 1 && close.count > 1 { 2 } else { 1 }; let ch = close.ch; let start = open.start + open.count - used; let end = close.start + used; - let children = items.drain(opener + 1..closer).collect::<Vec<_>>(); + let mut children = Vec::new(); + let mut depth = 0; + let mut index = slots[opener].next; + while index != closer { + if let Some(item) = slots[index].item.take() { + depth = depth.max(item_depth(&item)); + children.push(item); + } + index = slots[index].next; + } let values = to_nodes(children, context); let position = Some(context.position(start, end)); let node = match (ch, used) { @@ -97,26 +190,49 @@ pub(super) fn process(items: &mut Vec<Item>, bottom: usize, context: &Context<'_ _ => Node::Emphasis(Emphasis { values, position }), }; - // The drained items are gone, so the closer is now right after the new node. - closer = opener + 1; - items.insert(closer, Item::Node(node)); - closer += 1; + let node_index = slots.len(); + slots.push(Slot { + item: Some(Item::Node(node, depth + 1)), + prev: opener, + next: closer, + }); + slots[opener].next = node_index; + slots[closer].prev = node_index; - if let Item::Delim(open) = &mut items[opener] { - open.count -= used; + delim_mut(&mut slots, opener).count -= used; + let close = delim_mut(&mut slots, closer); + close.start += used; + close.count -= used; + + // Delimiters inside the new node cannot open anything else. + stack.truncate(at + 1); + if delim(&slots, opener).count == 0 { + unlink(&mut slots, opener); + stack.pop(); } - if let Item::Delim(close) = &mut items[closer] { - close.start += used; - close.count -= used; + if delim(&slots, closer).count == 0 { + let next = slots[closer].next; + unlink(&mut slots, closer); + closer = next; } + } - // Delimiters inside the new node cannot open anything else, and those used up are removed. - if matches!(&items[closer], Item::Delim(close) if close.count == 0) { - items.remove(closer); - } - if matches!(&items[opener], Item::Delim(open) if open.count == 0) { - items.remove(opener); - closer -= 1; + let mut index = slots[0].next; + while index != NIL { + items.extend(slots[index].item.take()); + index = slots[index].next; + } +} + +/// Whether the node for the items between `opener` and `closer` stays within the nesting limit. +fn fits(slots: &[Slot], opener: usize, closer: usize) -> bool { + let mut depth = 0; + let mut index = slots[opener].next; + while index != closer { + if let Some(item) = &slots[index].item { + depth = depth.max(item_depth(item)); } + index = slots[index].next; } + depth < MAX_NESTING } diff --git a/crates/mq-markdown/src/parser/inline/link.rs b/crates/mq-markdown/src/parser/inline/link.rs index 444b80a6e..c9f8c8650 100644 --- a/crates/mq-markdown/src/parser/inline/link.rs +++ b/crates/mq-markdown/src/parser/inline/link.rs @@ -2,7 +2,7 @@ use super::entity::{remove_line_indent, unescape}; use super::tail::{Tail, inline_tail, reference}; -use super::{Context, Item, Scanner, Value, emphasis, to_nodes}; +use super::{Context, Item, MAX_NESTING, Scanner, Value, emphasis, item_depth, to_nodes}; use crate::node::{FootnoteRef, Image, ImageRef, Link, LinkRef, Node, Position, Title, Url}; #[cfg(feature = "wikilink")] use crate::node::{RenderOptions, Text}; @@ -58,9 +58,14 @@ pub(super) fn close(scanner: &mut Scanner<'_>) -> bool { let Item::Open(opener) = &scanner.items[opener_index] else { return false; }; - let (image, active, opener_start) = (opener.image, opener.active, opener.start); + let (image, opener_start) = (opener.image, opener.start); + let active = image || scanner.openers.len() > scanner.inactive_below; scanner.openers.pop(); - if !active { + scanner.inactive_below = scanner.inactive_below.min(scanner.openers.len()); + let inner_depth = scanner.depths.pop().unwrap_or(0); + // The brackets stay text when they do not close, and what is inside them is still inside the outer ones. + scanner.note_depth(inner_depth); + if !active || inner_depth >= MAX_NESTING { return false; } @@ -108,23 +113,19 @@ pub(super) fn close(scanner: &mut Scanner<'_>) -> bool { emails: false, ..*context }; - emphasis::process(&mut content, 0, &inner); + emphasis::process(&mut content, &inner); + let depth = content.iter().map(item_depth).max().unwrap_or(0) + 1; let values = to_nodes(content, &inner); let (node, end) = build(kind, image, values, after, |end| context.position(opener_start, end)); if !image { // Links cannot contain links: earlier `[` openers can no longer become links. - for &index in &scanner.openers { - if let Item::Open(earlier) = &mut scanner.items[index] - && !earlier.image - { - earlier.active = false; - } - } + scanner.inactive_below = scanner.openers.len(); } - scanner.items.push(Item::Node(node)); + scanner.note_depth(depth); + scanner.items.push(Item::Node(node, depth)); scanner.pos = end; scanner.run = end; true diff --git a/crates/mq-markdown/src/parser/inline/tail.rs b/crates/mq-markdown/src/parser/inline/tail.rs index 87d76bb7d..af4bc86e6 100644 --- a/crates/mq-markdown/src/parser/inline/tail.rs +++ b/crates/mq-markdown/src/parser/inline/tail.rs @@ -15,7 +15,9 @@ pub(super) struct Tail { /// Returns the label, the offset after the reference, and whether the label is the text itself. pub(super) fn reference(context: &Context<'_>, text: &str, after: usize) -> Option<(String, usize, bool)> { let src = context.src(); - let defined = |label: &str| context.references.definitions.contains(&normalize(label)); + // A label has at most 999 characters, which also keeps the normalizing of long text away. + let defined = + |label: &str| label.chars().nth(999).is_none() && context.references.definitions.contains(&normalize(label)); if src[after..].starts_with('[') { match parse_label(src, after) { diff --git a/crates/mq-markdown/src/parser/line.rs b/crates/mq-markdown/src/parser/line.rs index ff27447e2..eb7720864 100644 --- a/crates/mq-markdown/src/parser/line.rs +++ b/crates/mq-markdown/src/parser/line.rs @@ -25,6 +25,10 @@ pub(super) struct Line<'a> { pub(super) pad: usize, /// Whether the document is MDX, which has no indented code, HTML, autolinks or GFM. pub(super) mdx: bool, + /// For `*`, `-` and `_`: one past the offset in `origin` of the last byte that is neither that + /// character, nor a space or tab, or 0 if there is none. A thematic break is a run of one of them + /// and whitespace to the end of the line, so this tells at once, from any offset, that it is not. + pub(super) others: [usize; 3], } impl<'a> Line<'a> { @@ -166,6 +170,21 @@ pub(super) fn visual_column(chars: impl Iterator<Item = char>, start: usize) -> }) } +/// The `Line::others` of `text`. +fn last_others(text: &str) -> [usize; 3] { + let mut last = [0; 3]; + for (index, byte) in text.bytes().enumerate() { + match byte { + b' ' | b'\t' => {} + b'*' => (last[1], last[2]) = (index + 1, index + 1), + b'-' => (last[0], last[2]) = (index + 1, index + 1), + b'_' => (last[0], last[1]) = (index + 1, index + 1), + _ => last = [index + 1; 3], + } + } + last +} + /// Splits `src` into lines on `\n`, `\r\n` and `\r`. A trailing terminator does not add a line. pub(super) fn split_lines(src: &str, mdx: bool) -> Vec<Line<'_>> { let bytes = src.as_bytes(); @@ -194,6 +213,7 @@ pub(super) fn split_lines(src: &str, mdx: bool) -> Vec<Line<'_>> { tabs: src[start..end].contains('\t'), pad: 0, mdx, + others: last_others(&src[start..end]), }); start = index; } @@ -211,6 +231,7 @@ pub(super) fn split_lines(src: &str, mdx: bool) -> Vec<Line<'_>> { tabs: src[start..].contains('\t'), pad: 0, mdx, + others: last_others(&src[start..]), }); } diff --git a/crates/mq-markdown/src/parser/render_html.rs b/crates/mq-markdown/src/parser/render_html.rs index ac432a4c1..dc602a552 100644 --- a/crates/mq-markdown/src/parser/render_html.rs +++ b/crates/mq-markdown/src/parser/render_html.rs @@ -10,6 +10,9 @@ use super::tree::{Block, FencedBlock, InlineBlock, InlineKind, InlineSource, Lis use crate::node::{Code, Node, TableAlignKind}; use rustc_hash::FxHashMap; +#[cfg(any(feature = "wikilink", feature = "embed", feature = "callout"))] +mod obsidian; + /// Protocols that a link may have, others are dropped. const SAFE_PROTOCOL_HREF: [&str; 6] = ["http", "https", "irc", "ircs", "mailto", "xmpp"]; /// Protocols that an image source may have. @@ -221,6 +224,11 @@ impl Html { Block::Fenced(fenced) => self.fenced(fenced), Block::Inline(block) => self.inline_block(block), Block::Quote(quote) => { + #[cfg(feature = "callout")] + if let Some((header, head)) = super::callout::peek_header(&quote.children) { + self.callout(&header, &head, &quote.children[1..]); + return; + } self.tight.push(false); self.line_ending_if_needed(); self.out.push_str("<blockquote>"); @@ -471,6 +479,10 @@ impl Html { self.image(&url, &image.alt, title.as_deref()); } Node::FootnoteRef(reference) => self.footnote_call(&reference.ident), + #[cfg(feature = "wikilink")] + Node::WikiLink(link) => self.wikilink(&link.target, link.text.as_deref()), + #[cfg(feature = "embed")] + Node::Embed(embed) => self.embed(&embed.target, embed.display.as_deref()), _ => {} } } diff --git a/crates/mq-markdown/src/parser/render_html/obsidian.rs b/crates/mq-markdown/src/parser/render_html/obsidian.rs new file mode 100644 index 000000000..8fef3378d --- /dev/null +++ b/crates/mq-markdown/src/parser/render_html/obsidian.rs @@ -0,0 +1,173 @@ +//! HTML for the Obsidian syntax: wikilinks, embeds and callouts. +//! +//! The classes and attributes are the ones Obsidian writes, so styles made for it apply. A link +//! target is used as the URL as written, because the files of a vault are not known here. + +#[cfg(feature = "callout")] +use super::super::callout::Header; +#[cfg(feature = "callout")] +use super::super::tree::Block; +#[cfg(feature = "embed")] +use super::SAFE_PROTOCOL_SRC; +use super::{Html, encode}; +#[cfg(any(feature = "wikilink", feature = "embed"))] +use super::{SAFE_PROTOCOL_HREF, sanitize_with_protocols}; + +/// What Obsidian shows for a link without text: `note#Heading` is `note > Heading`. +#[cfg(any(feature = "wikilink", feature = "embed"))] +fn shown_target(target: &str) -> String { + match target.split_once('#') { + Some(("", heading)) => heading.to_string(), + Some((page, heading)) => format!("{page} > {heading}"), + None => target.to_string(), + } +} + +/// How an embedded file is shown, by its extension. +#[cfg(feature = "embed")] +enum Media { + Image, + Audio, + Video, + Pdf, + Note, +} + +#[cfg(feature = "embed")] +fn media(target: &str) -> Media { + let path = target.split(['#', '?']).next().unwrap_or_default(); + let Some((_, extension)) = path.rsplit_once('.') else { + return Media::Note; + }; + match extension.to_ascii_lowercase().as_str() { + "png" | "jpg" | "jpeg" | "gif" | "bmp" | "svg" | "webp" | "avif" => Media::Image, + "mp3" | "wav" | "m4a" | "ogg" | "3gp" | "flac" => Media::Audio, + "mp4" | "webm" | "ogv" | "mov" | "mkv" => Media::Video, + "pdf" => Media::Pdf, + _ => Media::Note, + } +} + +/// The width and the height from a display like `100` or `100x200`. +#[cfg(feature = "embed")] +fn size(display: &str) -> Option<(&str, Option<&str>)> { + let digits = |value: &str| !value.is_empty() && value.bytes().all(|b| b.is_ascii_digit()); + match display.split_once('x') { + Some((width, height)) if digits(width) && digits(height) => Some((width, Some(height))), + None if digits(display) => Some((display, None)), + _ => None, + } +} + +impl Html { + /// `<a>` to the target, which is plain text inside another link. + #[cfg(any(feature = "wikilink", feature = "embed"))] + fn internal_link(&mut self, class: &str, target: &str, shown: &str) { + if self.in_link { + encode(&mut self.out, shown); + return; + } + self.out.push_str(&format!( + "<a class=\"{class}\" href=\"{}\" data-href=\"", + sanitize_with_protocols(target, &SAFE_PROTOCOL_HREF) + )); + encode(&mut self.out, target); + self.out.push_str("\">"); + encode(&mut self.out, shown); + self.out.push_str("</a>"); + } + + /// `[[target|text]]`. + #[cfg(feature = "wikilink")] + pub(super) fn wikilink(&mut self, target: &str, text: Option<&str>) { + let shown = text.map_or_else(|| shown_target(target), str::to_string); + self.internal_link("internal-link", target, &shown); + } + + /// `![[target|display]]`: an image, audio, video or PDF where the browser can show the file, and a + /// link where it cannot, which is a note. + #[cfg(feature = "embed")] + pub(super) fn embed(&mut self, target: &str, display: Option<&str>) { + let kind = media(target); + if matches!(kind, Media::Note) { + return self.internal_link("internal-link embed", target, &shown_target(target)); + } + + let src = sanitize_with_protocols(target, &SAFE_PROTOCOL_SRC); + let dimensions = display.and_then(size); + let mut attributes = String::new(); + if let Some((width, height)) = dimensions { + attributes.push_str(&format!(" width=\"{width}\"")); + if let Some(height) = height { + attributes.push_str(&format!(" height=\"{height}\"")); + } + } + + match kind { + Media::Image => { + self.out.push_str(&format!("<img src=\"{src}\" alt=\"")); + encode( + &mut self.out, + display.filter(|_| dimensions.is_none()).unwrap_or(target), + ); + self.out.push_str(&format!("\"{attributes} />")); + } + Media::Audio => self.out.push_str(&format!("<audio controls src=\"{src}\"></audio>")), + Media::Video => self + .out + .push_str(&format!("<video controls src=\"{src}\"{attributes}></video>")), + Media::Pdf => self + .out + .push_str(&format!("<embed type=\"application/pdf\" src=\"{src}\"{attributes} />")), + Media::Note => {} + } + } + + /// A callout: `<details>` when it can be folded, and a `<div>` otherwise. `head` is what remains of + /// the first paragraph after the header line, and `rest` the other blocks of the quote. + #[cfg(feature = "callout")] + pub(super) fn callout(&mut self, header: &Header, head: &[Block], rest: &[Block]) { + let kind = header.kind.to_lowercase(); + let title = header.title.clone().unwrap_or_else(|| { + let mut chars = kind.chars(); + chars.next().map_or_else(String::new, |first| { + first.to_uppercase().chain(chars).collect::<String>() + }) + }); + + self.line_ending_if_needed(); + let mut data = String::new(); + encode(&mut data, &kind); + let closing = if header.fold.is_some() { + let open = if header.fold == Some('+') { " open" } else { "" }; + self.out + .push_str(&format!("<details class=\"callout\" data-callout=\"{data}\"{open}>")); + self.line_ending(); + self.out.push_str("<summary class=\"callout-title\">"); + encode(&mut self.out, &title); + self.out.push_str("</summary>"); + "</details>" + } else { + self.out + .push_str(&format!("<div class=\"callout\" data-callout=\"{data}\">")); + self.line_ending(); + self.out.push_str("<div class=\"callout-title\">"); + encode(&mut self.out, &title); + self.out.push_str("</div>"); + "</div>" + }; + + if !head.is_empty() || !rest.is_empty() { + self.line_ending(); + self.out.push_str("<div class=\"callout-content\">"); + self.tight.push(false); + self.blocks(head); + self.blocks(rest); + self.tight.pop(); + self.line_ending_if_needed(); + self.out.push_str("</div>"); + } + self.line_ending(); + self.out.push_str(closing); + } +} diff --git a/crates/mq-markdown/src/parser/resolve.rs b/crates/mq-markdown/src/parser/resolve.rs index 6bb1e72f0..6b079d944 100644 --- a/crates/mq-markdown/src/parser/resolve.rs +++ b/crates/mq-markdown/src/parser/resolve.rs @@ -1,7 +1,7 @@ //! Turns the block tree into nodes: parses the inline content, pairs JSX tags and flattens lists and //! tables. -use super::inline; +use super::inline::{self, MAX_NESTING}; use super::mdx::TagKind; use super::tree::{Block, InlineBlock, InlineKind, JsxTag, ListBlock, QuoteBlock, TableItem}; use crate::node::{ @@ -66,6 +66,9 @@ impl Frames { fn tag(&mut self, tag: JsxTag) -> Result<(), String> { match tag.kind { + TagKind::Open if self.open.len() >= MAX_NESTING => { + return Err(format!("Elements are nested deeper than {MAX_NESTING} levels")); + } TagKind::Open => self.open.push((tag, Vec::new())), TagKind::SelfClosing => self.push(element(tag, Vec::new(), None)), TagKind::Close => { diff --git a/crates/mq-markdown/src/parser/snapshots.txt b/crates/mq-markdown/src/parser/snapshots.txt index 58220126e..d66b5ba9e 100644 --- a/crates/mq-markdown/src/parser/snapshots.txt +++ b/crates/mq-markdown/src/parser/snapshots.txt @@ -71,13 +71,13 @@ html: "<h2>a</h2>\n" nodes: [Heading(Heading { depth: 2, values: [Text(Text { value: "a\nb", @1:1-2:2 })], @1:1-3:4 })] html: "<h2>a\nb</h2>\n" ## thematic_star "***\n" -nodes: [HorizontalRule(HorizontalRule { marker: Some('*'), @1:1-1:4 })] +nodes: [HorizontalRule(HorizontalRule { marker: Some(Star), @1:1-1:4 })] html: "<hr />\n" ## thematic_dash_spaced "- - -\n" -nodes: [HorizontalRule(HorizontalRule { marker: Some('-'), @1:1-1:6 })] +nodes: [HorizontalRule(HorizontalRule { marker: Some(Dash), @1:1-1:6 })] html: "<hr />\n" ## thematic_underscore " ___ \n" -nodes: [HorizontalRule(HorizontalRule { marker: Some('_'), @1:1-1:8 })] +nodes: [HorizontalRule(HorizontalRule { marker: Some(Underscore), @1:1-1:8 })] html: "<hr />\n" ## thematic_two "**\n" nodes: [Text(Text { value: "**", @1:1-1:3 })] @@ -149,124 +149,124 @@ html: "<blockquote>\n<pre><code>a\n</code></pre>\n</blockquote>\n<p>b</p>\n" nodes: [Text(Text { value: "a", @1:1-1:2 }), Blockquote(Blockquote { values: [Text(Text { value: "b", @2:3-2:4 })], @2:1-2:4 })] html: "<p>a</p>\n<blockquote>\n<p>b</p>\n</blockquote>\n" ## quote_thematic_not_lazy "> a\n---\n" -nodes: [Blockquote(Blockquote { values: [Text(Text { value: "a", @1:3-1:4 })], @1:1-1:4 }), HorizontalRule(HorizontalRule { marker: Some('-'), @2:1-2:4 })] +nodes: [Blockquote(Blockquote { values: [Text(Text { value: "a", @1:3-1:4 })], @1:1-1:4 }), HorizontalRule(HorizontalRule { marker: Some(Dash), @2:1-2:4 })] html: "<blockquote>\n<p>a</p>\n</blockquote>\n<hr />\n" ## quote_multibyte "> あ\n> い\n" nodes: [Blockquote(Blockquote { values: [Text(Text { value: "あ\nい", @1:3-2:6 })], @1:1-2:6 })] html: "<blockquote>\n<p>あ\nい</p>\n</blockquote>\n" ## bullet "- a\n- b\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-1:4 }), List(List { values: [Text(Text { value: "b", @2:3-2:4 })], index: 1, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @2:3-2:4 })] +nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-1:4 }), List(List { values: [Text(Text { value: "b", @2:3-2:4 })], index: 1, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @2:3-2:4 })] html: "<ul>\n<li>a</li>\n<li>b</li>\n</ul>\n" ## bullet_loose "- a\n\n- b\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 })], index: 0, level: 0, ordered: false, checked: None, spread: true, start: None, marker: Some('-'), @1:3-1:4 }), List(List { values: [Text(Text { value: "b", @3:3-3:4 })], index: 1, level: 0, ordered: false, checked: None, spread: true, start: None, marker: Some('-'), @3:3-3:4 })] +nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 })], index: 0, level: 0, ordered: false, checked: None, spread: true, start: None, marker: Some(Dash), @1:3-1:4 }), List(List { values: [Text(Text { value: "b", @3:3-3:4 })], index: 1, level: 0, ordered: false, checked: None, spread: true, start: None, marker: Some(Dash), @3:3-3:4 })] html: "<ul>\n<li>\n<p>a</p>\n</li>\n<li>\n<p>b</p>\n</li>\n</ul>\n" ## bullet_indent " - a\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:4-1:5 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:4-1:5 })] +nodes: [List(List { values: [Text(Text { value: "a", @1:4-1:5 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:4-1:5 })] html: "<ul>\n<li>a</li>\n</ul>\n" ## bullet_mixed_markers "* a\n+ b\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('*'), @1:3-1:4 }), List(List { values: [Text(Text { value: "b", @2:3-2:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('+'), @2:3-2:4 })] +nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Star), @1:3-1:4 }), List(List { values: [Text(Text { value: "b", @2:3-2:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Plus), @2:3-2:4 })] html: "<ul>\n<li>a</li>\n</ul>\n<ul>\n<li>b</li>\n</ul>\n" ## ordered "1. a\n2. b\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:4-1:5 })], index: 0, level: 0, ordered: true, checked: None, spread: false, start: Some(1), marker: Some('.'), @1:4-1:5 }), List(List { values: [Text(Text { value: "b", @2:4-2:5 })], index: 1, level: 0, ordered: true, checked: None, spread: false, start: Some(1), marker: Some('.'), @2:4-2:5 })] +nodes: [List(List { values: [Text(Text { value: "a", @1:4-1:5 })], index: 0, level: 0, ordered: true, checked: None, spread: false, start: Some(1), marker: Some(Period), @1:4-1:5 }), List(List { values: [Text(Text { value: "b", @2:4-2:5 })], index: 1, level: 0, ordered: true, checked: None, spread: false, start: Some(1), marker: Some(Period), @2:4-2:5 })] html: "<ol>\n<li>a</li>\n<li>b</li>\n</ol>\n" ## ordered_paren "5) a\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:4-1:5 })], index: 0, level: 0, ordered: true, checked: None, spread: false, start: Some(5), marker: Some(')'), @1:4-1:5 })] +nodes: [List(List { values: [Text(Text { value: "a", @1:4-1:5 })], index: 0, level: 0, ordered: true, checked: None, spread: false, start: Some(5), marker: Some(Paren), @1:4-1:5 })] html: "<ol start=\"5\">\n<li>a</li>\n</ol>\n" ## ordered_zero "0. a\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:4-1:5 })], index: 0, level: 0, ordered: true, checked: None, spread: false, start: Some(0), marker: Some('.'), @1:4-1:5 })] +nodes: [List(List { values: [Text(Text { value: "a", @1:4-1:5 })], index: 0, level: 0, ordered: true, checked: None, spread: false, start: Some(0), marker: Some(Period), @1:4-1:5 })] html: "<ol start=\"0\">\n<li>a</li>\n</ol>\n" ## ordered_too_long "1234567890. a\n" nodes: [Text(Text { value: "1234567890. a", @1:1-1:14 })] html: "<p>1234567890. a</p>\n" ## ordered_mixed_delimiters "1. a\n2) b\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:4-1:5 })], index: 0, level: 0, ordered: true, checked: None, spread: false, start: Some(1), marker: Some('.'), @1:4-1:5 }), List(List { values: [Text(Text { value: "b", @2:4-2:5 })], index: 0, level: 0, ordered: true, checked: None, spread: false, start: Some(2), marker: Some(')'), @2:4-2:5 })] +nodes: [List(List { values: [Text(Text { value: "a", @1:4-1:5 })], index: 0, level: 0, ordered: true, checked: None, spread: false, start: Some(1), marker: Some(Period), @1:4-1:5 }), List(List { values: [Text(Text { value: "b", @2:4-2:5 })], index: 0, level: 0, ordered: true, checked: None, spread: false, start: Some(2), marker: Some(Paren), @2:4-2:5 })] html: "<ol>\n<li>a</li>\n</ol>\n<ol start=\"2\">\n<li>b</li>\n</ol>\n" ## task "- [ ] x\n- [x] y\n- [X] z\n" -nodes: [List(List { values: [Text(Text { value: "x", @1:7-1:8 })], index: 0, level: 0, ordered: false, checked: Some(false), spread: false, start: None, marker: Some('-'), @1:7-1:8 }), List(List { values: [Text(Text { value: "y", @2:7-2:8 })], index: 1, level: 0, ordered: false, checked: Some(true), spread: false, start: None, marker: Some('-'), @2:7-2:8 }), List(List { values: [Text(Text { value: "z", @3:7-3:8 })], index: 2, level: 0, ordered: false, checked: Some(true), spread: false, start: None, marker: Some('-'), @3:7-3:8 })] +nodes: [List(List { values: [Text(Text { value: "x", @1:7-1:8 })], index: 0, level: 0, ordered: false, checked: Some(false), spread: false, start: None, marker: Some(Dash), @1:7-1:8 }), List(List { values: [Text(Text { value: "y", @2:7-2:8 })], index: 1, level: 0, ordered: false, checked: Some(true), spread: false, start: None, marker: Some(Dash), @2:7-2:8 }), List(List { values: [Text(Text { value: "z", @3:7-3:8 })], index: 2, level: 0, ordered: false, checked: Some(true), spread: false, start: None, marker: Some(Dash), @3:7-3:8 })] html: "<ul>\n<li><input type=\"checkbox\" disabled=\"\" /> x</li>\n<li><input type=\"checkbox\" checked=\"\" disabled=\"\" /> y</li>\n<li><input type=\"checkbox\" checked=\"\" disabled=\"\" /> z</li>\n</ul>\n" ## task_empty "- [ ]\n" -nodes: [List(List { values: [Text(Text { value: "[ ]", @1:3-1:6 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-1:6 })] +nodes: [List(List { values: [Text(Text { value: "[ ]", @1:3-1:6 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-1:6 })] html: "<ul>\n<li>[ ]</li>\n</ul>\n" ## task_no_space "- [ ]x\n" -nodes: [List(List { values: [Text(Text { value: "[ ]x", @1:3-1:7 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-1:7 })] +nodes: [List(List { values: [Text(Text { value: "[ ]x", @1:3-1:7 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-1:7 })] html: "<ul>\n<li>[ ]x</li>\n</ul>\n" ## nested "- a\n - b\n c\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-1:4 }), List(List { values: [Text(Text { value: "b\nc", @2:5-3:6 })], index: 0, level: 1, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @2:5-3:6 })] +nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-1:4 }), List(List { values: [Text(Text { value: "b\nc", @2:5-3:6 })], index: 0, level: 1, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @2:5-3:6 })] html: "<ul>\n<li>a\n<ul>\n<li>b\nc</li>\n</ul>\n</li>\n</ul>\n" ## nested_ordered "1. a\n 1. b\n 2. c\n2. d\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:4-1:5 })], index: 0, level: 0, ordered: true, checked: None, spread: false, start: Some(1), marker: Some('.'), @1:4-1:5 }), List(List { values: [Text(Text { value: "b", @2:7-2:8 })], index: 0, level: 1, ordered: true, checked: None, spread: false, start: Some(1), marker: Some('.'), @2:7-2:8 }), List(List { values: [Text(Text { value: "c", @3:7-3:8 })], index: 1, level: 1, ordered: true, checked: None, spread: false, start: Some(1), marker: Some('.'), @3:7-3:8 }), List(List { values: [Text(Text { value: "d", @4:4-4:5 })], index: 1, level: 0, ordered: true, checked: None, spread: false, start: Some(1), marker: Some('.'), @4:4-4:5 })] +nodes: [List(List { values: [Text(Text { value: "a", @1:4-1:5 })], index: 0, level: 0, ordered: true, checked: None, spread: false, start: Some(1), marker: Some(Period), @1:4-1:5 }), List(List { values: [Text(Text { value: "b", @2:7-2:8 })], index: 0, level: 1, ordered: true, checked: None, spread: false, start: Some(1), marker: Some(Period), @2:7-2:8 }), List(List { values: [Text(Text { value: "c", @3:7-3:8 })], index: 1, level: 1, ordered: true, checked: None, spread: false, start: Some(1), marker: Some(Period), @3:7-3:8 }), List(List { values: [Text(Text { value: "d", @4:4-4:5 })], index: 1, level: 0, ordered: true, checked: None, spread: false, start: Some(1), marker: Some(Period), @4:4-4:5 })] html: "<ol>\n<li>a\n<ol>\n<li>b</li>\n<li>c</li>\n</ol>\n</li>\n<li>d</li>\n</ol>\n" ## nested_dedent "- a\n - b\n- c\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-1:4 }), List(List { values: [Text(Text { value: "b", @2:5-2:6 })], index: 0, level: 1, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @2:5-2:6 }), List(List { values: [Text(Text { value: "c", @3:3-3:4 })], index: 1, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @3:3-3:4 })] +nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-1:4 }), List(List { values: [Text(Text { value: "b", @2:5-2:6 })], index: 0, level: 1, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @2:5-2:6 }), List(List { values: [Text(Text { value: "c", @3:3-3:4 })], index: 1, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @3:3-3:4 })] html: "<ul>\n<li>a\n<ul>\n<li>b</li>\n</ul>\n</li>\n<li>c</li>\n</ul>\n" ## two_paragraphs "- a\n\n b\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 }), Text(Text { value: "b", @3:3-3:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-3:4 })] +nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 }), Text(Text { value: "b", @3:3-3:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-3:4 })] html: "<ul>\n<li>\n<p>a</p>\n<p>b</p>\n</li>\n</ul>\n" ## two_paragraphs_then_item "- a\n\n b\n- c\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 }), Text(Text { value: "b", @3:3-3:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-3:4 }), List(List { values: [Text(Text { value: "c", @4:3-4:4 })], index: 1, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @4:3-4:4 })] +nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 }), Text(Text { value: "b", @3:3-3:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-3:4 }), List(List { values: [Text(Text { value: "c", @4:3-4:4 })], index: 1, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @4:3-4:4 })] html: "<ul>\n<li>\n<p>a</p>\n<p>b</p>\n</li>\n<li>\n<p>c</p>\n</li>\n</ul>\n" ## empty_item "-\n" -nodes: [List(List { values: [], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:1-1:2 })] +nodes: [List(List { values: [], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:1-1:2 })] html: "<ul>\n<li></li>\n</ul>\n" ## empty_item_content "-\n foo\n" -nodes: [List(List { values: [Text(Text { value: "foo", @2:3-2:6 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @2:3-2:6 })] +nodes: [List(List { values: [Text(Text { value: "foo", @2:3-2:6 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @2:3-2:6 })] html: "<ul>\n<li>foo</li>\n</ul>\n" ## empty_item_blank_then_text "-\n\n foo\n" -nodes: [List(List { values: [], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:1-1:2 }), Text(Text { value: "foo", @3:3-3:6 })] +nodes: [List(List { values: [], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:1-1:2 }), Text(Text { value: "foo", @3:3-3:6 })] html: "<ul>\n<li></li>\n</ul>\n<p>foo</p>\n" ## empty_items "-\n-\n" -nodes: [List(List { values: [], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:1-1:2 }), List(List { values: [], index: 1, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @2:1-2:2 })] +nodes: [List(List { values: [], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:1-1:2 }), List(List { values: [], index: 1, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @2:1-2:2 })] html: "<ul>\n<li></li>\n<li></li>\n</ul>\n" ## wide_marker_gap "- a\n b\n" -nodes: [List(List { values: [Text(Text { value: "a\nb", @1:5-2:6 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:5-2:6 })] +nodes: [List(List { values: [Text(Text { value: "a\nb", @1:5-2:6 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:5-2:6 })] html: "<ul>\n<li>a\nb</li>\n</ul>\n" ## code_in_item "- a\n" -nodes: [List(List { values: [Code(Code { value: "a", lang: None, @1:3-1:8, meta: None, fence: false })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-1:8 })] +nodes: [List(List { values: [Code(Code { value: "a", lang: None, @1:3-1:8, meta: None, fence: false })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-1:8 })] html: "<ul>\n<li>\n<pre><code>a\n</code></pre>\n</li>\n</ul>\n" ## lazy "- a\nb\n" -nodes: [List(List { values: [Text(Text { value: "a\nb", @1:3-2:2 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-2:2 })] +nodes: [List(List { values: [Text(Text { value: "a\nb", @1:3-2:2 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-2:2 })] html: "<ul>\n<li>a\nb</li>\n</ul>\n" ## lazy_after_blank "- a\n\nb\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-1:4 }), Text(Text { value: "b", @3:1-3:2 })] +nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-1:4 }), Text(Text { value: "b", @3:1-3:2 })] html: "<ul>\n<li>a</li>\n</ul>\n<p>b</p>\n" ## list_interrupts_paragraph "a\n- b\n" -nodes: [Text(Text { value: "a", @1:1-1:2 }), List(List { values: [Text(Text { value: "b", @2:3-2:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @2:3-2:4 })] +nodes: [Text(Text { value: "a", @1:1-1:2 }), List(List { values: [Text(Text { value: "b", @2:3-2:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @2:3-2:4 })] html: "<p>a</p>\n<ul>\n<li>b</li>\n</ul>\n" ## ordered_two_no_interrupt "a\n2. b\n" nodes: [Text(Text { value: "a\n2. b", @1:1-2:5 })] html: "<p>a\n2. b</p>\n" ## ordered_one_interrupts "a\n1. b\n" -nodes: [Text(Text { value: "a", @1:1-1:2 }), List(List { values: [Text(Text { value: "b", @2:4-2:5 })], index: 0, level: 0, ordered: true, checked: None, spread: false, start: Some(1), marker: Some('.'), @2:4-2:5 })] +nodes: [Text(Text { value: "a", @1:1-1:2 }), List(List { values: [Text(Text { value: "b", @2:4-2:5 })], index: 0, level: 0, ordered: true, checked: None, spread: false, start: Some(1), marker: Some(Period), @2:4-2:5 })] html: "<p>a</p>\n<ol>\n<li>b</li>\n</ol>\n" ## empty_item_no_interrupt "a\n-\n" nodes: [Heading(Heading { depth: 2, values: [Text(Text { value: "a", @1:1-1:2 })], @1:1-2:2 })] html: "<h2>a</h2>\n" ## thematic_over_list "- - -\n" -nodes: [HorizontalRule(HorizontalRule { marker: Some('-'), @1:1-1:6 })] +nodes: [HorizontalRule(HorizontalRule { marker: Some(Dash), @1:1-1:6 })] html: "<hr />\n" ## list_in_quote "> - a\n> - b\n" -nodes: [Blockquote(Blockquote { values: [List(List { values: [Text(Text { value: "a", @1:5-1:6 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:5-1:6 }), List(List { values: [Text(Text { value: "b", @2:5-2:6 })], index: 1, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @2:5-2:6 })], @1:1-2:6 })] +nodes: [Blockquote(Blockquote { values: [List(List { values: [Text(Text { value: "a", @1:5-1:6 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:5-1:6 }), List(List { values: [Text(Text { value: "b", @2:5-2:6 })], index: 1, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @2:5-2:6 })], @1:1-2:6 })] html: "<blockquote>\n<ul>\n<li>a</li>\n<li>b</li>\n</ul>\n</blockquote>\n" ## quote_in_list "- > a\n > b\n" -nodes: [List(List { values: [Blockquote(Blockquote { values: [Text(Text { value: "a\nb", @1:5-2:6 })], @1:3-2:6 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-2:6 })] +nodes: [List(List { values: [Blockquote(Blockquote { values: [Text(Text { value: "a\nb", @1:5-2:6 })], @1:3-2:6 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-2:6 })] html: "<ul>\n<li>\n<blockquote>\n<p>a\nb</p>\n</blockquote>\n</li>\n</ul>\n" ## quote_in_list_level "- a\n > - b\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 }), Blockquote(Blockquote { values: [List(List { values: [Text(Text { value: "b", @2:7-2:8 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @2:7-2:8 })], @2:3-2:8 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-2:8 })] +nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 }), Blockquote(Blockquote { values: [List(List { values: [Text(Text { value: "b", @2:7-2:8 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @2:7-2:8 })], @2:3-2:8 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-2:8 })] html: "<ul>\n<li>a\n<blockquote>\n<ul>\n<li>b</li>\n</ul>\n</blockquote>\n</li>\n</ul>\n" ## heading_in_item "- # h\n text\n" -nodes: [List(List { values: [Heading(Heading { depth: 1, values: [Text(Text { value: "h", @1:5-1:6 })], @1:3-1:6 }), Text(Text { value: "text", @2:3-2:7 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-2:7 })] +nodes: [List(List { values: [Heading(Heading { depth: 1, values: [Text(Text { value: "h", @1:5-1:6 })], @1:3-1:6 }), Text(Text { value: "text", @2:3-2:7 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-2:7 })] html: "<ul>\n<li>\n<h1>h</h1>\ntext</li>\n</ul>\n" ## fence_in_item "- ```\n a\n ```\n" -nodes: [List(List { values: [Code(Code { value: "a", lang: None, @1:3-3:6, meta: None, fence: true })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-3:6 })] +nodes: [List(List { values: [Code(Code { value: "a", lang: None, @1:3-3:6, meta: None, fence: true })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-3:6 })] html: "<ul>\n<li>\n<pre><code>a\n</code></pre>\n</li>\n</ul>\n" ## multibyte_item "- あ\n- い\n" -nodes: [List(List { values: [Text(Text { value: "あ", @1:3-1:6 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-1:6 }), List(List { values: [Text(Text { value: "い", @2:3-2:6 })], index: 1, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @2:3-2:6 })] +nodes: [List(List { values: [Text(Text { value: "あ", @1:3-1:6 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-1:6 }), List(List { values: [Text(Text { value: "い", @2:3-2:6 })], index: 1, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @2:3-2:6 })] html: "<ul>\n<li>あ</li>\n<li>い</li>\n</ul>\n" ## list_after_paragraph_blank "a\n\n- b\n" -nodes: [Text(Text { value: "a", @1:1-1:2 }), List(List { values: [Text(Text { value: "b", @3:3-3:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @3:3-3:4 })] +nodes: [Text(Text { value: "a", @1:1-1:2 }), List(List { values: [Text(Text { value: "b", @3:3-3:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @3:3-3:4 })] html: "<p>a</p>\n<ul>\n<li>b</li>\n</ul>\n" ## two_lists_split "- a\n\n\n- b\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 })], index: 0, level: 0, ordered: false, checked: None, spread: true, start: None, marker: Some('-'), @1:3-1:4 }), List(List { values: [Text(Text { value: "b", @4:3-4:4 })], index: 1, level: 0, ordered: false, checked: None, spread: true, start: None, marker: Some('-'), @4:3-4:4 })] +nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 })], index: 0, level: 0, ordered: false, checked: None, spread: true, start: None, marker: Some(Dash), @1:3-1:4 }), List(List { values: [Text(Text { value: "b", @4:3-4:4 })], index: 1, level: 0, ordered: false, checked: None, spread: true, start: None, marker: Some(Dash), @4:3-4:4 })] html: "<ul>\n<li>\n<p>a</p>\n</li>\n<li>\n<p>b</p>\n</li>\n</ul>\n" ## deep_quote ">>>>>> a\n" nodes: [Blockquote(Blockquote { values: [Blockquote(Blockquote { values: [Blockquote(Blockquote { values: [Blockquote(Blockquote { values: [Blockquote(Blockquote { values: [Blockquote(Blockquote { values: [Text(Text { value: "a", @1:8-1:9 })], @1:6-1:9 })], @1:5-1:9 })], @1:4-1:9 })], @1:3-1:9 })], @1:2-1:9 })], @1:1-1:9 })] @@ -275,10 +275,10 @@ html: "<blockquote>\n<blockquote>\n<blockquote>\n<blockquote>\n<blockquote>\n<bl nodes: [Code(Code { value: "a", lang: None, @1:1-2:2, meta: None, fence: true })] html: "<pre><code>a\n</code></pre>\n" ## fence_unclosed_in_item "- ```\n a\n" -nodes: [List(List { values: [Code(Code { value: "a", lang: None, @1:3-3:1, meta: None, fence: true })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-3:1 })] +nodes: [List(List { values: [Code(Code { value: "a", lang: None, @1:3-3:1, meta: None, fence: true })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-3:1 })] html: "<ul>\n<li>\n<pre><code>a\n</code></pre>\n</li>\n</ul>" ## fence_unclosed_in_item_no_eol "- ```\n a" -nodes: [List(List { values: [Code(Code { value: "a", lang: None, @1:3-2:4, meta: None, fence: true })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-2:4 })] +nodes: [List(List { values: [Code(Code { value: "a", lang: None, @1:3-2:4, meta: None, fence: true })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-2:4 })] html: "<ul>\n<li>\n<pre><code>a\n</code></pre>\n</li>\n</ul>" ## fence_unclosed_in_quote_no_eol "> ```\n> a" nodes: [Blockquote(Blockquote { values: [Code(Code { value: "a", lang: None, @1:3-2:4, meta: None, fence: true })], @1:1-2:4 })] @@ -290,13 +290,13 @@ html: "<blockquote>\n<pre><code>a\n</code></pre>\n</blockquote>" nodes: [Code(Code { value: "a\n", lang: None, @1:1-4:1, meta: None, fence: true })] html: "<pre><code>a\n\n</code></pre>\n" ## fence_unclosed_item_then_para "- ```\n a\nb\n" -nodes: [List(List { values: [Code(Code { value: "a", lang: None, @1:3-2:4, meta: None, fence: true })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-2:4 }), Text(Text { value: "b", @3:1-3:2 })] +nodes: [List(List { values: [Code(Code { value: "a", lang: None, @1:3-2:4, meta: None, fence: true })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-2:4 }), Text(Text { value: "b", @3:1-3:2 })] html: "<ul>\n<li>\n<pre><code>a\n</code></pre>\n</li>\n</ul>\n<p>b</p>\n" ## item_code_then_unindented_text "- ===\n\n a\n===" -nodes: [List(List { values: [Text(Text { value: "===", @1:3-1:6 }), Code(Code { value: "a", lang: None, @3:3-3:8, meta: None, fence: false })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-3:8 }), Text(Text { value: "===", @4:1-4:4 })] +nodes: [List(List { values: [Text(Text { value: "===", @1:3-1:6 }), Code(Code { value: "a", lang: None, @3:3-3:8, meta: None, fence: false })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-3:8 }), Text(Text { value: "===", @4:1-4:4 })] html: "<ul>\n<li>\n<p>===</p>\n<pre><code>a\n</code></pre>\n</li>\n</ul>\n<p>===</p>" ## quote_setext_dashes "> a\n> ---\n> ---\n> a" -nodes: [Blockquote(Blockquote { values: [Heading(Heading { depth: 2, values: [Text(Text { value: "a", @1:3-1:4 })], @1:3-2:6 }), HorizontalRule(HorizontalRule { marker: Some('-'), @3:3-3:6 }), Text(Text { value: "a", @4:3-4:4 })], @1:1-4:4 })] +nodes: [Blockquote(Blockquote { values: [Heading(Heading { depth: 2, values: [Text(Text { value: "a", @1:3-1:4 })], @1:3-2:6 }), HorizontalRule(HorizontalRule { marker: Some(Dash), @3:3-3:6 }), Text(Text { value: "a", @4:3-4:4 })], @1:1-4:4 })] html: "<blockquote>\n<h2>a</h2>\n<hr />\n<p>a</p>\n</blockquote>" ## table_basic "| a | b |\n|---|:-:|\n| 1 | 2 |\n" nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:3-1:4 })]:0, row: 0, @1:1-1:5 }), TableCell(TableCell { values: [Text(Text { value: "b", @1:7-1:8 })]:1, row: 0, @1:5-1:10 }), TableAlign(TableAlign { align: [None, Center], @2:1-2:1 }), TableCell(TableCell { values: [Text(Text { value: "1", @3:3-3:4 })]:0, row: 1, @3:1-3:5 }), TableCell(TableCell { values: [Text(Text { value: "2", @3:7-3:8 })]:1, row: 1, @3:5-3:10 })] @@ -341,7 +341,7 @@ html: "<table>\n<thead>\n<tr>\n<th>a</th>\n</tr>\n</thead>\n</table>\n<h1>h</h1> nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:2-1:3 })]:0, row: 0, @1:1-1:4 }), TableAlign(TableAlign { align: [None], @2:1-2:1 }), Code(Code { value: "x", lang: None, @3:1-5:4, meta: None, fence: true })] html: "<table>\n<thead>\n<tr>\n<th>a</th>\n</tr>\n</thead>\n</table>\n<pre><code>x\n</code></pre>\n" ## table_then_list "|a|\n|-|\n- x\n" -nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:2-1:3 })]:0, row: 0, @1:1-1:4 }), TableAlign(TableAlign { align: [None], @2:1-2:1 }), List(List { values: [Text(Text { value: "x", @3:3-3:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @3:3-3:4 })] +nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:2-1:3 })]:0, row: 0, @1:1-1:4 }), TableAlign(TableAlign { align: [None], @2:1-2:1 }), List(List { values: [Text(Text { value: "x", @3:3-3:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @3:3-3:4 })] html: "<table>\n<thead>\n<tr>\n<th>a</th>\n</tr>\n</thead>\n</table>\n<ul>\n<li>x</li>\n</ul>\n" ## table_two "|a|\n|-|\n\n|b|\n|-|\n" nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:2-1:3 })]:0, row: 0, @1:1-1:4 }), TableAlign(TableAlign { align: [None], @2:1-2:1 }), TableCell(TableCell { values: [Text(Text { value: "b", @4:2-4:3 })]:0, row: 0, @4:1-4:4 }), TableAlign(TableAlign { align: [None], @5:1-5:1 })] @@ -464,7 +464,7 @@ html: "<h1>a <em>b</em> <code>c</code></h1>\n" nodes: [Heading(Heading { depth: 1, values: [Text(Text { value: "a ", @1:1-1:3 }), Emphasis(Emphasis { values: [Text(Text { value: "b", @1:4-1:5 })], @1:3-1:6 }), Text(Text { value: "\nc", @1:6-2:2 })], @1:1-3:4 })] html: "<h1>a <em>b</em>\nc</h1>\n" ## inline_list_item "- a *b*\n c **d**\n" -nodes: [List(List { values: [Text(Text { value: "a ", @1:3-1:5 }), Emphasis(Emphasis { values: [Text(Text { value: "b", @1:6-1:7 })], @1:5-1:8 }), Text(Text { value: "\nc ", @1:8-2:5 }), Strong(Strong { values: [Text(Text { value: "d", @2:7-2:8 })], @2:5-2:10 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-2:10 })] +nodes: [List(List { values: [Text(Text { value: "a ", @1:3-1:5 }), Emphasis(Emphasis { values: [Text(Text { value: "b", @1:6-1:7 })], @1:5-1:8 }), Text(Text { value: "\nc ", @1:8-2:5 }), Strong(Strong { values: [Text(Text { value: "d", @2:7-2:8 })], @2:5-2:10 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-2:10 })] html: "<ul>\n<li>a <em>b</em>\nc <strong>d</strong></li>\n</ul>\n" ## inline_table_cell "| a *b* | `c\\|d` |\n|-|-|\n" nodes: [TableCell(TableCell { values: [Text(Text { value: "a ", @1:3-1:5 }), Emphasis(Emphasis { values: [Text(Text { value: "b", @1:6-1:7 })], @1:5-1:8 })]:0, row: 0, @1:1-1:9 }), TableCell(TableCell { values: [CodeInline(CodeInline { value: "c|d", @1:11-1:17 })]:1, row: 0, @1:9-1:19 }), TableAlign(TableAlign { align: [None, None], @2:1-2:1 })] @@ -512,7 +512,7 @@ html: "<p>text\n[a]: b</p>" nodes: [Definition(Definition { @1:1-1:7, url: Url("b"), title: None, ident: "a", label: Some("a") }), Text(Text { value: "===", @2:1-2:4 })] html: "<p>===</p>" ## def_then_setext_dashes "[a]: b\n---" -nodes: [Definition(Definition { @1:1-1:7, url: Url("b"), title: None, ident: "a", label: Some("a") }), HorizontalRule(HorizontalRule { marker: Some('-'), @2:1-2:4 })] +nodes: [Definition(Definition { @1:1-1:7, url: Url("b"), title: None, ident: "a", label: Some("a") }), HorizontalRule(HorizontalRule { marker: Some(Dash), @2:1-2:4 })] html: "<hr />" ## def_indent1 " [a]: b" nodes: [Definition(Definition { @1:1-1:8, url: Url("b"), title: None, ident: "a", label: Some("a") })] @@ -527,7 +527,7 @@ html: "<pre><code>[a]: b\n</code></pre>" nodes: [Blockquote(Blockquote { values: [Definition(Definition { @1:3-1:9, url: Url("b"), title: None, ident: "a", label: Some("a") })], @1:1-1:9 })] html: "<blockquote>\n</blockquote>" ## def_in_list "- [a]: b" -nodes: [List(List { values: [Definition(Definition { @1:3-1:9, url: Url("b"), title: None, ident: "a", label: Some("a") })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-1:9 })] +nodes: [List(List { values: [Definition(Definition { @1:3-1:9, url: Url("b"), title: None, ident: "a", label: Some("a") })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-1:9 })] html: "<ul>\n<li></li>\n</ul>" ## def_no_dest "[a]:" nodes: [Text(Text { value: "[a]:", @1:1-1:5 })] @@ -632,19 +632,19 @@ html: "<pre><code>[^a]: b\n</code></pre>" nodes: [Blockquote(Blockquote { values: [Footnote(Footnote { ident: "a", values: [Text(Text { value: "b", @1:9-1:10 })], @1:3-1:10 })], @1:1-1:10 })] html: "<blockquote>\n</blockquote>" ## fn_in_list "- [^a]: b" -nodes: [List(List { values: [Footnote(Footnote { ident: "a", values: [Text(Text { value: "b", @1:9-1:10 })], @1:3-1:10 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-1:10 })] +nodes: [List(List { values: [Footnote(Footnote { ident: "a", values: [Text(Text { value: "b", @1:9-1:10 })], @1:3-1:10 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-1:10 })] html: "<ul>\n<li></li>\n</ul>" ## fn_then_quote "[^a]: b\n> q" nodes: [Footnote(Footnote { ident: "a", values: [Text(Text { value: "b", @1:7-1:8 })], @1:1-1:8 }), Blockquote(Blockquote { values: [Text(Text { value: "q", @2:3-2:4 })], @2:1-2:4 })] html: "<blockquote>\n<p>q</p>\n</blockquote>" ## fn_then_list "[^a]: b\n- x" -nodes: [Footnote(Footnote { ident: "a", values: [Text(Text { value: "b", @1:7-1:8 })], @1:1-1:8 }), List(List { values: [Text(Text { value: "x", @2:3-2:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @2:3-2:4 })] +nodes: [Footnote(Footnote { ident: "a", values: [Text(Text { value: "b", @1:7-1:8 })], @1:1-1:8 }), List(List { values: [Text(Text { value: "x", @2:3-2:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @2:3-2:4 })] html: "<ul>\n<li>x</li>\n</ul>" ## fn_heading "[^a]: # h" nodes: [Footnote(Footnote { ident: "a", values: [Heading(Heading { depth: 1, values: [Text(Text { value: "h", @1:9-1:10 })], @1:7-1:10 })], @1:1-1:10 })] html: "" ## fn_list "[^a]: - x\n - y" -nodes: [Footnote(Footnote { ident: "a", values: [List(List { values: [Text(Text { value: "x", @1:9-1:10 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:9-1:10 }), List(List { values: [Text(Text { value: "y", @2:7-2:8 })], index: 1, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @2:7-2:8 })], @1:1-2:8 })] +nodes: [Footnote(Footnote { ident: "a", values: [List(List { values: [Text(Text { value: "x", @1:9-1:10 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:9-1:10 }), List(List { values: [Text(Text { value: "y", @2:7-2:8 })], index: 1, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @2:7-2:8 })], @1:1-2:8 })] html: "" ## fn_fence "[^a]: b\n ```\n x\n ```" nodes: [Footnote(Footnote { ident: "a", values: [Text(Text { value: "b", @1:7-1:8 }), Code(Code { value: "x", lang: None, @2:5-4:8, meta: None, fence: true })], @1:1-4:8 })] @@ -659,7 +659,7 @@ html: "" nodes: [Text(Text { value: "text", @1:1-1:5 }), Footnote(Footnote { ident: "a", values: [Text(Text { value: "b", @2:7-2:8 })], @2:1-2:8 })] html: "<p>text</p>\n" ## fn_then_dashes "[^a]: b\n---" -nodes: [Footnote(Footnote { ident: "a", values: [Text(Text { value: "b", @1:7-1:8 })], @1:1-1:8 }), HorizontalRule(HorizontalRule { marker: Some('-'), @2:1-2:4 })] +nodes: [Footnote(Footnote { ident: "a", values: [Text(Text { value: "b", @1:7-1:8 })], @1:1-1:8 }), HorizontalRule(HorizontalRule { marker: Some(Dash), @2:1-2:4 })] html: "<hr />" ## fn_then_equals "[^a]: b\n===" nodes: [Footnote(Footnote { ident: "a", values: [Text(Text { value: "b\n===", @1:7-2:4 })], @1:1-2:4 })] @@ -731,7 +731,7 @@ html: "<div\nclass=\"a\">\nz" nodes: [Blockquote(Blockquote { values: [Html(Html { value: "<div>\nx", @1:3-2:4 })], @1:1-2:4 })] html: "<blockquote>\n<div>\nx\n</blockquote>" ## html_in_list "- <div>\n x" -nodes: [List(List { values: [Html(Html { value: "<div>\nx", @1:3-2:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-2:4 })] +nodes: [List(List { values: [Html(Html { value: "<div>\nx", @1:3-2:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-2:4 })] html: "<ul>\n<li>\n<div>\nx\n</li>\n</ul>" ## html_uppercase "<DIV>\nx" nodes: [Html(Html { value: "<DIV>\nx", @1:1-2:2 })] @@ -794,7 +794,7 @@ html: "" nodes: [Yaml(Yaml { value: "", @1:1-2:4 })] html: "" ## fm_unclosed "---\na\n" -nodes: [HorizontalRule(HorizontalRule { marker: Some('-'), @1:1-1:4 }), Text(Text { value: "a", @2:1-2:2 })] +nodes: [HorizontalRule(HorizontalRule { marker: Some(Dash), @1:1-1:4 }), Text(Text { value: "a", @2:1-2:2 })] html: "<hr />\n<p>a</p>\n" ## fm_blank_content "---\n\n---\nx" nodes: [Yaml(Yaml { value: "", @1:1-3:4 }), Text(Text { value: "x", @4:1-4:2 })] @@ -803,13 +803,13 @@ html: "<p>x</p>" nodes: [Toml(Toml { value: "a = 1", @1:1-3:4 }), Text(Text { value: "x", @4:1-4:2 })] html: "<p>x</p>" ## fm_indented " ---\na\n---" -nodes: [HorizontalRule(HorizontalRule { marker: Some('-'), @1:1-1:5 }), Heading(Heading { depth: 2, values: [Text(Text { value: "a", @2:1-2:2 })], @2:1-3:4 })] +nodes: [HorizontalRule(HorizontalRule { marker: Some(Dash), @1:1-1:5 }), Heading(Heading { depth: 2, values: [Text(Text { value: "a", @2:1-2:2 })], @2:1-3:4 })] html: "<hr />\n<h2>a</h2>" ## fm_close_trailing_space "---\na\n--- \nx" nodes: [Yaml(Yaml { value: "a", @1:1-3:5 }), Text(Text { value: "x", @4:1-4:2 })] html: "<p>x</p>" ## fm_close_longer "---\na\n----\nx" -nodes: [HorizontalRule(HorizontalRule { marker: Some('-'), @1:1-1:4 }), Heading(Heading { depth: 2, values: [Text(Text { value: "a", @2:1-2:2 })], @2:1-3:5 }), Text(Text { value: "x", @4:1-4:2 })] +nodes: [HorizontalRule(HorizontalRule { marker: Some(Dash), @1:1-1:4 }), Heading(Heading { depth: 2, values: [Text(Text { value: "a", @2:1-2:2 })], @2:1-3:5 }), Text(Text { value: "x", @4:1-4:2 })] html: "<hr />\n<h2>a</h2>\n<p>x</p>" ## fm_open_trailing_space "--- \na\n---" nodes: [Yaml(Yaml { value: "a", @1:1-3:4 })] @@ -821,13 +821,13 @@ html: "<h2>---x\na</h2>" nodes: [Heading(Heading { depth: 2, values: [Text(Text { value: "text", @1:1-1:5 })], @1:1-2:4 }), Heading(Heading { depth: 2, values: [Text(Text { value: "a", @3:1-3:2 })], @3:1-4:4 })] html: "<h2>text</h2>\n<h2>a</h2>" ## fm_blank_before "\n---\na\n---" -nodes: [HorizontalRule(HorizontalRule { marker: Some('-'), @2:1-2:4 }), Heading(Heading { depth: 2, values: [Text(Text { value: "a", @3:1-3:2 })], @3:1-4:4 })] +nodes: [HorizontalRule(HorizontalRule { marker: Some(Dash), @2:1-2:4 }), Heading(Heading { depth: 2, values: [Text(Text { value: "a", @3:1-3:2 })], @3:1-4:4 })] html: "<hr />\n<h2>a</h2>" ## fm_multiline_content "---\na: 1\n\nb: 2\n---\n# h" nodes: [Yaml(Yaml { value: "a: 1\n\nb: 2", @1:1-5:4 }), Heading(Heading { depth: 1, values: [Text(Text { value: "h", @6:3-6:4 })], @6:1-6:4 })] html: "<h1>h</h1>" ## fm_hr_after "---\na\n---\n\n---" -nodes: [Yaml(Yaml { value: "a", @1:1-3:4 }), HorizontalRule(HorizontalRule { marker: Some('-'), @5:1-5:4 })] +nodes: [Yaml(Yaml { value: "a", @1:1-3:4 }), HorizontalRule(HorizontalRule { marker: Some(Dash), @5:1-5:4 })] html: "<hr />" ## math_basic "$$\na\n$$" nodes: [Math(Math { value: "a", @1:1-3:3 })] @@ -863,7 +863,7 @@ html: "<pre><code class=\"language-math math-display\"></code></pre>" nodes: [Blockquote(Blockquote { values: [Math(Math { value: "a", @1:3-3:5 })], @1:1-3:5 })] html: "<blockquote>\n<pre><code class=\"language-math math-display\">a\n</code></pre>\n</blockquote>" ## math_in_list "- $$\n a\n $$" -nodes: [List(List { values: [Math(Math { value: "a", @1:3-3:5 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-3:5 })] +nodes: [List(List { values: [Math(Math { value: "a", @1:3-3:5 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-3:5 })] html: "<ul>\n<li>\n<pre><code class=\"language-math math-display\">a\n</code></pre>\n</li>\n</ul>" ## math_close_junk "$$\na\n$$ x" nodes: [Math(Math { value: "a\n$$ x", @1:1-3:5 })] @@ -905,13 +905,13 @@ html: "<pre><code class=\"language-math math-display\">\n</code></pre>" nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:3-1:4 })]:0, row: 0, @1:1-1:4 }), TableCell(TableCell { values: [Text(Text { value: "b", @1:5-1:6 })]:1, row: 0, @1:4-1:6 }), TableAlign(TableAlign { align: [None, Center], @2:1-2:1 })] html: "<table>\n<thead>\n<tr>\n<th>a</th>\n<th align=\"center\">b</th>\n</tr>\n</thead>\n</table>" ## tab_nested_list "- a\n\t- b\n\t\t- c\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-1:4 }), List(List { values: [Text(Text { value: "b", @2:7-2:8 })], index: 0, level: 1, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @2:7-2:8 }), List(List { values: [Text(Text { value: "c", @3:11-3:12 })], index: 0, level: 2, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @3:11-3:12 })] +nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-1:4 }), List(List { values: [Text(Text { value: "b", @2:7-2:8 })], index: 0, level: 1, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @2:7-2:8 }), List(List { values: [Text(Text { value: "c", @3:11-3:12 })], index: 0, level: 2, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @3:11-3:12 })] html: "<ul>\n<li>a\n<ul>\n<li>b\n<ul>\n<li>c</li>\n</ul>\n</li>\n</ul>\n</li>\n</ul>\n" ## tab_nested_ordered "1. a\n\t1. b\n\t\t1. c\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:4-1:5 })], index: 0, level: 0, ordered: true, checked: None, spread: false, start: Some(1), marker: Some('.'), @1:4-1:5 }), List(List { values: [Text(Text { value: "b", @2:8-2:9 })], index: 0, level: 1, ordered: true, checked: None, spread: false, start: Some(1), marker: Some('.'), @2:8-2:9 }), List(List { values: [Text(Text { value: "c", @3:12-3:13 })], index: 0, level: 2, ordered: true, checked: None, spread: false, start: Some(1), marker: Some('.'), @3:12-3:13 })] +nodes: [List(List { values: [Text(Text { value: "a", @1:4-1:5 })], index: 0, level: 0, ordered: true, checked: None, spread: false, start: Some(1), marker: Some(Period), @1:4-1:5 }), List(List { values: [Text(Text { value: "b", @2:8-2:9 })], index: 0, level: 1, ordered: true, checked: None, spread: false, start: Some(1), marker: Some(Period), @2:8-2:9 }), List(List { values: [Text(Text { value: "c", @3:12-3:13 })], index: 0, level: 2, ordered: true, checked: None, spread: false, start: Some(1), marker: Some(Period), @3:12-3:13 })] html: "<ol>\n<li>a\n<ol>\n<li>b\n<ol>\n<li>c</li>\n</ol>\n</li>\n</ol>\n</li>\n</ol>\n" ## tab_list_code "- a\n\n\t\tcode\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 }), Code(Code { value: " code", lang: None, @3:3-3:13, meta: None, fence: false })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-3:13 })] +nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 }), Code(Code { value: " code", lang: None, @3:3-3:13, meta: None, fence: false })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-3:13 })] html: "<ul>\n<li>\n<p>a</p>\n<pre><code> code\n</code></pre>\n</li>\n</ul>\n" ## tab_in_text "a\tb\t*c*\n" nodes: [Text(Text { value: "a\tb\t", @1:1-1:9 }), Emphasis(Emphasis { values: [Text(Text { value: "c", @1:10-1:11 })], @1:9-1:12 })] @@ -920,7 +920,7 @@ html: "<p>a\tb\t<em>c</em></p>\n" nodes: [Code(Code { value: "code\n\tmore", lang: None, @1:1-2:13, meta: None, fence: false })] html: "<pre><code>code\n\tmore\n</code></pre>\n" ## tab_after_marker "-\ta\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:5-1:6 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:5-1:6 })] +nodes: [List(List { values: [Text(Text { value: "a", @1:5-1:6 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:5-1:6 })] html: "<ul>\n<li>a</li>\n</ul>\n" ## atx_leading_sequences "# # a" nodes: [Heading(Heading { depth: 1, values: [Text(Text { value: "# a", @1:3-1:6 })], @1:1-1:6 })] @@ -947,7 +947,7 @@ html: "<h1># a</h1>" nodes: [Heading(Heading { depth: 1, values: [Text(Text { value: "a # b", @1:3-1:8 })], @1:1-1:8 })] html: "<h1>a # b</h1>" ## quote_footnote_trailing_space "- a\n > [^a]: x\n " -nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 }), Blockquote(Blockquote { values: [Footnote(Footnote { ident: "a", values: [Text(Text { value: "x", @2:11-2:12 })], @2:5-3:3 })], @2:3-3:3 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-3:3 })] +nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 }), Blockquote(Blockquote { values: [Footnote(Footnote { ident: "a", values: [Text(Text { value: "x", @2:11-2:12 })], @2:5-3:3 })], @2:3-3:3 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-3:3 })] html: "<ul>\n<li>a\n<blockquote>\n</blockquote>\n</li>\n</ul>" ## indented_code_trailing_blank " code\n\n\n" nodes: [Code(Code { value: "code", lang: None, @1:1-1:9, meta: None, fence: false })] @@ -1007,28 +1007,28 @@ html: "<table>\n<thead>\n<tr>\n<th align=\"right\">a</th>\n</tr>\n</thead>\n<tbo nodes: [Text(Text { value: "a\n::-", @1:1-2:4 })] html: "<p>a\n::-</p>\n" ## table_body_empty_marker "a|b\n|-|-|\n*\n" -nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:1-1:2 })]:0, row: 0, @1:1-1:2 }), TableCell(TableCell { values: [Text(Text { value: "b", @1:3-1:4 })]:1, row: 0, @1:2-1:4 }), TableAlign(TableAlign { align: [None, None], @2:1-2:1 }), List(List { values: [], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('*'), @3:1-3:2 })] +nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:1-1:2 })]:0, row: 0, @1:1-1:2 }), TableCell(TableCell { values: [Text(Text { value: "b", @1:3-1:4 })]:1, row: 0, @1:2-1:4 }), TableAlign(TableAlign { align: [None, None], @2:1-2:1 }), List(List { values: [], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Star), @3:1-3:2 })] html: "<table>\n<thead>\n<tr>\n<th>a</th>\n<th>b</th>\n</tr>\n</thead>\n</table>\n<ul>\n<li></li>\n</ul>\n" ## table_body_ordered_marker "a|b\n|-|-|\n2. x\n" -nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:1-1:2 })]:0, row: 0, @1:1-1:2 }), TableCell(TableCell { values: [Text(Text { value: "b", @1:3-1:4 })]:1, row: 0, @1:2-1:4 }), TableAlign(TableAlign { align: [None, None], @2:1-2:1 }), List(List { values: [Text(Text { value: "x", @3:4-3:5 })], index: 0, level: 0, ordered: true, checked: None, spread: false, start: Some(2), marker: Some('.'), @3:4-3:5 })] +nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:1-1:2 })]:0, row: 0, @1:1-1:2 }), TableCell(TableCell { values: [Text(Text { value: "b", @1:3-1:4 })]:1, row: 0, @1:2-1:4 }), TableAlign(TableAlign { align: [None, None], @2:1-2:1 }), List(List { values: [Text(Text { value: "x", @3:4-3:5 })], index: 0, level: 0, ordered: true, checked: None, spread: false, start: Some(2), marker: Some(Period), @3:4-3:5 })] html: "<table>\n<thead>\n<tr>\n<th>a</th>\n<th>b</th>\n</tr>\n</thead>\n</table>\n<ol start=\"2\">\n<li>x</li>\n</ol>\n" ## table_body_empty_ordered "a|b\n|-|-|\n1.\n" -nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:1-1:2 })]:0, row: 0, @1:1-1:2 }), TableCell(TableCell { values: [Text(Text { value: "b", @1:3-1:4 })]:1, row: 0, @1:2-1:4 }), TableAlign(TableAlign { align: [None, None], @2:1-2:1 }), List(List { values: [], index: 0, level: 0, ordered: true, checked: None, spread: false, start: Some(1), marker: Some('.'), @3:1-3:3 })] +nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:1-1:2 })]:0, row: 0, @1:1-1:2 }), TableCell(TableCell { values: [Text(Text { value: "b", @1:3-1:4 })]:1, row: 0, @1:2-1:4 }), TableAlign(TableAlign { align: [None, None], @2:1-2:1 }), List(List { values: [], index: 0, level: 0, ordered: true, checked: None, spread: false, start: Some(1), marker: Some(Period), @3:1-3:3 })] html: "<table>\n<thead>\n<tr>\n<th>a</th>\n<th>b</th>\n</tr>\n</thead>\n</table>\n<ol>\n<li></li>\n</ol>\n" ## paragraph_after_partial_tab "- a\n\n\t[P](h) x\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 }), Link(Link { url: Url("h"), title: None, values: [Text(Text { value: "P", @3:6-3:7 })], @3:5-3:11 }), Text(Text { value: " x", @3:11-3:13 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-3:13 })] +nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 }), Link(Link { url: Url("h"), title: None, values: [Text(Text { value: "P", @3:6-3:7 })], @3:5-3:11 }), Text(Text { value: " x", @3:11-3:13 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-3:13 })] html: "<ul>\n<li>\n<p>a</p>\n<p><a href=\"h\">P</a> x</p>\n</li>\n</ul>\n" ## fence_after_partial_tab "- a\n\n\t```ts\n\t\tx\n\t```\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 }), Code(Code { value: "\tx", lang: Some("ts"), @3:5-5:8, meta: None, fence: true })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-5:8 })] +nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 }), Code(Code { value: "\tx", lang: Some("ts"), @3:5-5:8, meta: None, fence: true })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-5:8 })] html: "<ul>\n<li>\n<p>a</p>\n<pre><code class=\"language-ts\">\tx\n</code></pre>\n</li>\n</ul>\n" ## math_after_partial_tab "- a\n\n\t$$\n\tm\n\t$$\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 }), Math(Math { value: "m", @3:5-5:7 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-5:7 })] +nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 }), Math(Math { value: "m", @3:5-5:7 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-5:7 })] html: "<ul>\n<li>\n<p>a</p>\n<pre><code class=\"language-math math-display\">m\n</code></pre>\n</li>\n</ul>\n" ## quote_after_partial_tab "- a\n\n\t> q\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 }), Blockquote(Blockquote { values: [Text(Text { value: "q", @3:7-3:8 })], @3:3-3:8 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-3:8 })] +nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 }), Blockquote(Blockquote { values: [Text(Text { value: "q", @3:7-3:8 })], @3:3-3:8 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-3:8 })] html: "<ul>\n<li>\n<p>a</p>\n<blockquote>\n<p>q</p>\n</blockquote>\n</li>\n</ul>\n" ## heading_after_partial_tab "- a\n\n\t# h\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 }), Heading(Heading { depth: 1, values: [Text(Text { value: "h", @3:7-3:8 })], @3:3-3:8 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-3:8 })] +nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 }), Heading(Heading { depth: 1, values: [Text(Text { value: "h", @3:7-3:8 })], @3:3-3:8 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-3:8 })] html: "<ul>\n<li>\n<p>a</p>\n<h1>h</h1>\n</li>\n</ul>\n" ## control_character_references "&#1;&#x7f;&#x85;&#xB;&#xC;&#x9f;&#xa0;&#xd800;&#x110000;&#0;" nodes: [Text(Text { value: "����\u{c}�\u{a0}���", @1:1-1:61 })] @@ -1064,7 +1064,7 @@ html: "<p><a href=\"x%EF%BF%BDy\">a</a></p>" nodes: [Definition(Definition { @1:1-1:7, url: Url("b"), title: None, ident: "a", label: Some("a") }), Text(Text { value: "-", @2:1-2:2 })] html: "<p>-</p>" ## definition_then_item "[a]: b\n- x" -nodes: [Definition(Definition { @1:1-1:7, url: Url("b"), title: None, ident: "a", label: Some("a") }), List(List { values: [Text(Text { value: "x", @2:3-2:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @2:3-2:4 })] +nodes: [Definition(Definition { @1:1-1:7, url: Url("b"), title: None, ident: "a", label: Some("a") }), List(List { values: [Text(Text { value: "x", @2:3-2:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @2:3-2:4 })] html: "<ul>\n<li>x</li>\n</ul>" ## definition_then_ordered "[a]: b\n1." nodes: [Definition(Definition { @1:1-1:7, url: Url("b"), title: None, ident: "a", label: Some("a") }), Text(Text { value: "1.", @2:1-2:3 })] @@ -1097,7 +1097,7 @@ html: "<pre><code class=\"language-あ\">日本語\n</code></pre>\n" nodes: [Blockquote(Blockquote { values: [Code(Code { value: "x", lang: Some("🎉"), @1:3-3:6, meta: None, fence: true })], @1:1-3:6 })] html: "<blockquote>\n<pre><code class=\"language-🎉\">x\n</code></pre>\n</blockquote>\n" ## fence_in_list_cjk "- ```あ\n 日本語\n ```\n" -nodes: [List(List { values: [Code(Code { value: "日本語", lang: Some("あ"), @1:3-3:6, meta: None, fence: true })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-3:6 })] +nodes: [List(List { values: [Code(Code { value: "日本語", lang: Some("あ"), @1:3-3:6, meta: None, fence: true })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-3:6 })] html: "<ul>\n<li>\n<pre><code class=\"language-あ\">日本語\n</code></pre>\n</li>\n</ul>\n" ## math_cjk "$$\nあ\n$$\n\n$い$\n" nodes: [Math(Math { value: "あ", @1:1-3:3 }), MathInline(MathInline { value: "い", @5:1-5:6 })] @@ -1190,7 +1190,7 @@ nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [MdxJsxFlowElement(MdxJs ## mdx jsx_in_quote "> <a>\n> x\n> </a>" nodes: [Blockquote(Blockquote { values: [MdxJsxFlowElement(MdxJsxFlowElement { children: [Text(Text { value: "x", @2:3-2:4 })], @1:3-3:7, name: Some("a"), attributes: [] })], @1:1-3:7 })] ## mdx jsx_in_list "- <a>\n x\n </a>" -nodes: [List(List { values: [MdxJsxFlowElement(MdxJsxFlowElement { children: [Text(Text { value: "x", @2:3-2:4 })], @1:3-3:7, name: Some("a"), attributes: [] })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:3-3:7 })] +nodes: [List(List { values: [MdxJsxFlowElement(MdxJsxFlowElement { children: [Text(Text { value: "x", @2:3-2:4 })], @1:3-3:7, name: Some("a"), attributes: [] })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-3:7 })] ## mdx attr_quote_inside "<a b='c\"d' />" nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [], @1:1-1:14, name: Some("a"), attributes: [Property(MdxJsxAttribute { name: "b", value: Some(Literal("c\"d")) })] })] ## mdx two_tags "<a/><b/>" @@ -1258,7 +1258,7 @@ nodes: [MdxJsxTextElement(MdxJsxTextElement { children: [Emphasis(Emphasis { val ## mdx flow_element_with_emphasis "<a>\n*b*\n</a>" nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [Emphasis(Emphasis { values: [Text(Text { value: "b", @2:2-2:3 })], @2:1-2:4 })], @1:1-3:5, name: Some("a"), attributes: [] })] ## mdx flow_element_with_list "<a>\n- b\n</a>" -nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [List(List { values: [Text(Text { value: "b", @2:3-2:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @2:3-2:4 })], @1:1-3:5, name: Some("a"), attributes: [] })] +nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [List(List { values: [Text(Text { value: "b", @2:3-2:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @2:3-2:4 })], @1:1-3:5, name: Some("a"), attributes: [] })] ## mdx list_items_unbalanced "- <a>\n- </a>" nodes: error: Expected a closing tag for `<a>` (1:3) before the end of its container ## mdx spaced_equals "<a b = \"c\" />" @@ -1272,17 +1272,17 @@ nodes: [Heading(Heading { depth: 1, values: [Text(Text { value: "h", @1:8-1:9 }) ## mdx no_autolink "<http://a.b>" nodes: error: Unexpected character `/` (U+002F) before local name ## mdx no_gfm "| a |\n|-|\n\n~a~ www.a.b [^a]\n\n- [ ] a\n" -nodes: [Text(Text { value: "| a |\n|-|", @1:1-2:4 }), Text(Text { value: "~a~ www.a.b [^a]", @4:1-4:17 }), List(List { values: [Text(Text { value: "[ ] a", @6:3-6:8 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @6:3-6:8 })] +nodes: [Text(Text { value: "| a |\n|-|", @1:1-2:4 }), Text(Text { value: "~a~ www.a.b [^a]", @4:1-4:17 }), List(List { values: [Text(Text { value: "[ ] a", @6:3-6:8 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @6:3-6:8 })] ## mdx no_math "$a$\n\n$$\na\n$$\n" nodes: [Text(Text { value: "$a$", @1:1-1:4 }), Text(Text { value: "$$\na\n$$", @3:1-5:3 })] ## mdx no_frontmatter "---\na\n---\n" -nodes: [HorizontalRule(HorizontalRule { marker: Some('-'), @1:1-1:4 }), Heading(Heading { depth: 2, values: [Text(Text { value: "a", @2:1-2:2 })], @2:1-3:4 })] +nodes: [HorizontalRule(HorizontalRule { marker: Some(Dash), @1:1-1:4 }), Heading(Heading { depth: 2, values: [Text(Text { value: "a", @2:1-2:2 })], @2:1-3:4 })] ## mdx esm "import a from 'b'\n\nexport const c = 1\n" nodes: [MdxJsEsm(MdxJsEsm { value: "import a from 'b'", @1:1-1:18 }), MdxJsEsm(MdxJsEsm { value: "export const c = 1", @3:1-3:19 })] ## mdx definition "[a]: /u\n\n[a]\n" nodes: [Definition(Definition { @1:1-1:8, url: Url("/u"), title: None, ident: "a", label: Some("a") }), LinkRef(LinkRef { ident: "a", label: Some("a"), values: [Text(Text { value: "a", @3:2-3:3 })], @3:1-3:4 })] ## mdx list_deep_indent "- a\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:8-1:9 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some('-'), @1:8-1:9 })] +nodes: [List(List { values: [Text(Text { value: "a", @1:8-1:9 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:8-1:9 })] ## mdx fence "```rust\ncode\n```\n" nodes: [Code(Code { value: "code", lang: Some("rust"), @1:1-3:4, meta: None, fence: true })] ## mdx lone_cr_after_lt "<\r/-x" diff --git a/crates/mq-markdown/src/parser/tree.rs b/crates/mq-markdown/src/parser/tree.rs index ed661c639..79b878831 100644 --- a/crates/mq-markdown/src/parser/tree.rs +++ b/crates/mq-markdown/src/parser/tree.rs @@ -5,9 +5,10 @@ use super::line::visual_column; use super::mdx::TagKind; -use crate::node::{MdxAttributeContent, Node, Point, Position, TableAlignKind}; +use crate::node::{ListMarker, MdxAttributeContent, Node, Point, Position, TableAlignKind}; /// Raw inline content of a paragraph, heading or table cell, with the source position of each line. +#[derive(Clone)] pub(super) struct InlineSource { pub(super) text: String, /// For each line, the offset in `text` where it starts and its position in the document. @@ -141,11 +142,13 @@ pub(super) struct FencedBlock { pub(super) lines: usize, } +#[derive(Clone)] pub(super) struct InlineBlock { pub(super) source: InlineSource, pub(super) kind: InlineKind, } +#[derive(Clone)] pub(super) enum InlineKind { Paragraph, Heading { depth: u8, position: Position }, @@ -166,7 +169,7 @@ pub(super) struct FootnoteBlock { pub(super) struct ListBlock { pub(super) ordered: bool, pub(super) start: Option<u32>, - pub(super) marker: char, + pub(super) marker: ListMarker, pub(super) spread: bool, pub(super) items: Vec<Item>, } diff --git a/crates/mq-markdown/tests/gfm_roundtrip_fidelity.rs b/crates/mq-markdown/tests/gfm_roundtrip_fidelity.rs index 8b027ddda..7e8f68831 100644 --- a/crates/mq-markdown/tests/gfm_roundtrip_fidelity.rs +++ b/crates/mq-markdown/tests/gfm_roundtrip_fidelity.rs @@ -70,17 +70,9 @@ fn parse_spec_examples(text: &str) -> Vec<SpecExample> { /// number in the spec revision pinned above. Remove an entry once it's /// fixed; add one (with its section) if a new gap turns up. const KNOWN_FAILURES: &[usize] = &[ - 5, 6, 7, // Tabs - 229, 269, // List items - 271, 272, 295, // Lists - // 300, 306, 346, 479, 480, 599: URL-shaped text disables backslash - // escapes even in link text (GFM extended autolink scanning). - 300, 306, // Backslash escapes - 346, // Code spans - 479, 480, // Emphasis and strong emphasis - 522, 534, // Links - 570, 589, // Images - 599, 602, // Autolinks + 300, // Backslash escapes + 589, // Images + 602, // Autolinks ]; #[test] diff --git a/crates/mq-markdown/tests/obsidian_tests.rs b/crates/mq-markdown/tests/obsidian_tests.rs index f9c262275..e3f4b97d3 100644 --- a/crates/mq-markdown/tests/obsidian_tests.rs +++ b/crates/mq-markdown/tests/obsidian_tests.rs @@ -287,3 +287,116 @@ proptest! { check_callout("note", "", &title, "\n> ", body); } } + +#[rstest] +#[case::wikilink( + "[[foo]]\n", + "<p><a class=\"internal-link\" href=\"foo\" data-href=\"foo\">foo</a></p>\n" +)] +#[case::wikilink_with_text( + "a [[foo|bar]] b\n", + "<p>a <a class=\"internal-link\" href=\"foo\" data-href=\"foo\">bar</a> b</p>\n" +)] +#[case::wikilink_with_spaces( + "[[My Note]]\n", + "<p><a class=\"internal-link\" href=\"My%20Note\" data-href=\"My Note\">My Note</a></p>\n" +)] +#[case::wikilink_with_heading( + "[[note#Part one]]\n", + "<p><a class=\"internal-link\" href=\"note#Part%20one\" data-href=\"note#Part one\">note &gt; Part one</a></p>\n" +)] +#[case::wikilink_to_a_heading_of_the_page( + "[[#Part]]\n", + "<p><a class=\"internal-link\" href=\"#Part\" data-href=\"#Part\">Part</a></p>\n" +)] +#[case::wikilink_is_escaped( + "[[a&b|<c>]]\n", + "<p><a class=\"internal-link\" href=\"a&amp;b\" data-href=\"a&amp;b\">&lt;c&gt;</a></p>\n" +)] +#[case::wikilink_with_a_script_protocol( + "[[javascript:alert(1)]]\n", + "<p><a class=\"internal-link\" href=\"\" data-href=\"javascript:alert(1)\">javascript:alert(1)</a></p>\n" +)] +#[case::wikilink_inside_a_link_stays_text("[a [[b]]](c)\n", "<p><a href=\"c\">a [[b]]</a></p>\n")] +#[case::wikilink_in_a_heading( + "# [[a]]\n", + "<h1><a class=\"internal-link\" href=\"a\" data-href=\"a\">a</a></h1>\n" +)] +#[case::image("![[img.png]]\n", "<p><img src=\"img.png\" alt=\"img.png\" /></p>\n")] +#[case::image_with_width( + "![[img.png|100]]\n", + "<p><img src=\"img.png\" alt=\"img.png\" width=\"100\" /></p>\n" +)] +#[case::image_with_size( + "![[img.png|100x200]]\n", + "<p><img src=\"img.png\" alt=\"img.png\" width=\"100\" height=\"200\" /></p>\n" +)] +#[case::image_with_alt("![[img.png|a photo]]\n", "<p><img src=\"img.png\" alt=\"a photo\" /></p>\n")] +#[case::image_extension_is_case_insensitive("![[IMG.PNG]]\n", "<p><img src=\"IMG.PNG\" alt=\"IMG.PNG\" /></p>\n")] +#[case::image_with_spaces("![[my img.png]]\n", "<p><img src=\"my%20img.png\" alt=\"my img.png\" /></p>\n")] +#[case::image_with_a_script_protocol("![[javascript:a.png]]\n", "<p><img src=\"\" alt=\"javascript:a.png\" /></p>\n")] +#[case::audio("![[a.mp3]]\n", "<p><audio controls src=\"a.mp3\"></audio></p>\n")] +#[case::video( + "![[a.mp4|640x480]]\n", + "<p><video controls src=\"a.mp4\" width=\"640\" height=\"480\"></video></p>\n" +)] +#[case::pdf("![[a.pdf]]\n", "<p><embed type=\"application/pdf\" src=\"a.pdf\" /></p>\n")] +#[case::note( + "![[note]]\n", + "<p><a class=\"internal-link embed\" href=\"note\" data-href=\"note\">note</a></p>\n" +)] +#[case::note_with_heading( + "a ![[note#Part]] b\n", + "<p>a <a class=\"internal-link embed\" href=\"note#Part\" data-href=\"note#Part\">note &gt; Part</a> b</p>\n" +)] +fn html_renders_wikilinks_and_embeds(#[case] input: &str, #[case] expected: &str) { + assert_eq!(mq_markdown::to_html(input), expected); + assert_eq!(Markdown::from_markdown_str(input).unwrap().to_html(), expected); +} + +#[rstest] +#[case::note( + "> [!note]\n> Body\n", + "<div class=\"callout\" data-callout=\"note\">\n<div class=\"callout-title\">Note</div>\n<div class=\"callout-content\">\n<p>Body</p>\n</div>\n</div>\n" +)] +#[case::upper_case_kind( + "> [!WARNING]\n> Careful\n", + "<div class=\"callout\" data-callout=\"warning\">\n<div class=\"callout-title\">Warning</div>\n<div class=\"callout-content\">\n<p>Careful</p>\n</div>\n</div>\n" +)] +#[case::title( + "> [!tip] A <b> title\n> Body\n", + "<div class=\"callout\" data-callout=\"tip\">\n<div class=\"callout-title\">A &lt;b&gt; title</div>\n<div class=\"callout-content\">\n<p>Body</p>\n</div>\n</div>\n" +)] +#[case::without_body( + "> [!note] Only a title\n", + "<div class=\"callout\" data-callout=\"note\">\n<div class=\"callout-title\">Only a title</div>\n</div>\n" +)] +#[case::open_fold( + "> [!faq]+ Question\n> Answer\n", + "<details class=\"callout\" data-callout=\"faq\" open>\n<summary class=\"callout-title\">Question</summary>\n<div class=\"callout-content\">\n<p>Answer</p>\n</div>\n</details>\n" +)] +#[case::closed_fold( + "> [!faq]- Question\n> Answer\n", + "<details class=\"callout\" data-callout=\"faq\">\n<summary class=\"callout-title\">Question</summary>\n<div class=\"callout-content\">\n<p>Answer</p>\n</div>\n</details>\n" +)] +#[case::blocks_in_the_body( + "> [!note]\n> a\n>\n> - b\n", + "<div class=\"callout\" data-callout=\"note\">\n<div class=\"callout-title\">Note</div>\n<div class=\"callout-content\">\n<p>a</p>\n<ul>\n<li>b</li>\n</ul>\n</div>\n</div>\n" +)] +#[case::nested( + "> [!note]\n> > [!tip]\n> > Inner\n", + "<div class=\"callout\" data-callout=\"note\">\n<div class=\"callout-title\">Note</div>\n<div class=\"callout-content\">\n<div class=\"callout\" data-callout=\"tip\">\n<div class=\"callout-title\">Tip</div>\n<div class=\"callout-content\">\n<p>Inner</p>\n</div>\n</div>\n</div>\n</div>\n" +)] +#[case::wikilink_in_the_body( + "> [!note]\n> [[a]]\n", + "<div class=\"callout\" data-callout=\"note\">\n<div class=\"callout-title\">Note</div>\n<div class=\"callout-content\">\n<p><a class=\"internal-link\" href=\"a\" data-href=\"a\">a</a></p>\n</div>\n</div>\n" +)] +#[case::an_ordinary_quote("> [note]\n> Body\n", "<blockquote>\n<p>[note]\nBody</p>\n</blockquote>\n")] +#[case::after_a_paragraph( + "a\n\n> [!note]\n> b\n\nc\n", + "<p>a</p>\n<div class=\"callout\" data-callout=\"note\">\n<div class=\"callout-title\">Note</div>\n<div class=\"callout-content\">\n<p>b</p>\n</div>\n</div>\n<p>c</p>\n" +)] +fn html_renders_callouts(#[case] input: &str, #[case] expected: &str) { + assert_eq!(mq_markdown::to_html(input), expected); + assert_eq!(Markdown::from_markdown_str(input).unwrap().to_html(), expected); +} diff --git a/crates/mq-markdown/tests/parser_limits_tests.rs b/crates/mq-markdown/tests/parser_limits_tests.rs new file mode 100644 index 000000000..d3e80d2e1 --- /dev/null +++ b/crates/mq-markdown/tests/parser_limits_tests.rs @@ -0,0 +1,118 @@ +//! Input that nests or repeats constructs very deeply must parse and render without exhausting the +//! stack, and without spending time that grows faster than the input. + +use mq_markdown::Markdown; +use rstest::rstest; + +fn nested(open: &str, content: &str, close: &str, times: usize) -> String { + format!("{}{content}{}", open.repeat(times), close.repeat(times)) +} + +/// Runs `check` on a thread with the stack of a main thread: the test threads have less, and a debug +/// build uses far more of it than a release build does. +fn on_main_sized_stack(check: impl FnOnce() + Send + 'static) { + std::thread::Builder::new() + .stack_size(8 * 1024 * 1024) + .spawn(check) + .unwrap() + .join() + .unwrap(); +} + +/// Everything that walks the nodes, each of which recurses over the nesting. +fn exercise(markdown: Markdown) { + let _ = markdown.to_string(); + let _ = markdown.to_text(); + let _ = markdown.to_html(); + #[cfg(feature = "json")] + let _ = markdown.to_json(); + let copy = markdown.clone(); + drop(markdown); + drop(copy); +} + +#[rstest] +#[case::emphasis(nested("*a ", "b", " a*", 100_000))] +#[case::emphasis_and_strong(nested("*a **a ", "b", " a** a*", 50_000))] +#[case::underscore(nested("_a ", "b", " a_", 100_000))] +#[case::strikethrough(nested("~a ", "b", " a~", 100_000))] +#[case::links(nested("[a ", "![x](y)", "](z)", 50_000))] +#[case::images(nested("![a ", "b", "](z)", 50_000))] +#[case::link_labels(nested("[a ", "b", "]", 100_000))] +#[case::emphasis_in_links(nested("[*a ", "b", " a*](x)", 40_000))] +#[case::quotes(format!("{}a", "> ".repeat(10_000)))] +#[case::quotes_with_emphasis(nested(&"> ".repeat(100), "b", "", 1) + &nested("*a ", "b", " a*", 50_000))] +#[case::lists((0..300).map(|level| format!("{}- a\n", " ".repeat(level))).collect::<String>())] +fn deep_nesting_does_not_exhaust_the_stack(#[case] input: String) { + on_main_sized_stack(move || exercise(Markdown::from_markdown_str(&input).unwrap())); +} + +#[rstest] +#[case::inline(nested("<a>", "x", "</a>", 20_000))] +#[case::flow(nested("<a>\n\n", "x\n\n", "</a>\n\n", 20_000))] +fn deeply_nested_elements_are_an_error(#[case] input: String) { + let error = Markdown::from_mdx_str(&input).unwrap_err().to_string(); + assert!(error.contains("nested deeper"), "{error}"); +} + +#[test] +fn elements_nested_within_the_limit_parse() { + let input = nested("<a>", "x", "</a>", 100); + on_main_sized_stack(move || exercise(Markdown::from_mdx_str(&input).unwrap())); +} + +#[rstest] +#[case::wikilink_openers("[[a ".repeat(100_000))] +#[case::unmatched_openers("[".repeat(100_000))] +#[case::open_links_then_links(format!("{}{}", "[".repeat(50_000), "[a](b) ".repeat(50_000)))] +#[case::unmatched_emphasis("*a ".repeat(100_000))] +#[case::emphasis_that_cannot_match("*a_".repeat(100_000))] +#[case::list_markers_on_a_line(format!("{}a\n", "- ".repeat(20_000)).repeat(20))] +#[case::list_markers_ending_like_a_rule(format!("{}a -\n", "- ".repeat(20_000)).repeat(20))] +#[case::star_markers_ending_like_a_rule(format!("{}a *\n", "* ".repeat(20_000)).repeat(20))] +#[case::quote_markers_on_a_line(format!("{}a\n", "> ".repeat(20_000)).repeat(20))] +#[case::numbered_markers_on_a_line(format!("{}a\n", "1. ".repeat(20_000)).repeat(20))] +#[case::long_labels(format!("[a]: /u\n{}", "[b ".repeat(100_000)))] +fn repeated_constructs_parse(#[case] input: String) { + exercise(Markdown::from_markdown_str(&input).unwrap()); +} + +#[rstest] +#[case::emphasis( + "*a **a *a b a* a** a*\n", + "<p><em>a <strong>a <em>a b a</em> a</strong> a</em></p>\n" +)] +#[case::rule_of_three("*foo**bar**baz*\n", "<p><em>foo<strong>bar</strong>baz</em></p>\n")] +#[case::partly_used_runs("***a** b*\n", "<p><em><strong>a</strong> b</em></p>\n")] +#[case::unmatched_stay_text("*a **b\n", "<p>*a **b</p>\n")] +#[case::closer_also_opens("*a*b*c*\n", "<p><em>a</em>b<em>c</em></p>\n")] +#[case::strikethrough_lengths("~a~ ~~b~~ ~~c~\n", "<p><del>a</del> <del>b</del> ~~c~</p>\n")] +fn emphasis_matches_as_before(#[case] input: &str, #[case] expected: &str) { + assert_eq!(mq_markdown::to_html(input), expected); +} + +#[rstest] +#[case::dashes("---\n", "<hr />\n")] +#[case::spaced_dashes("- - -\n", "<hr />\n")] +#[case::spaced_stars("* * *\n", "<hr />\n")] +#[case::underscores("_ _ _\n", "<hr />\n")] +#[case::trailing_whitespace("-- - \t \n", "<hr />\n")] +#[case::in_a_quote("> - - -\n", "<blockquote>\n<hr />\n</blockquote>\n")] +#[case::in_a_list_item("- ***\n", "<ul>\n<li>\n<hr />\n</li>\n</ul>\n")] +#[case::two_markers_are_not_enough("- -\n", "<ul>\n<li>\n<ul>\n<li></li>\n</ul>\n</li>\n</ul>\n")] +#[case::mixed_markers( + "- * -\n", + "<ul>\n<li>\n<ul>\n<li>\n<ul>\n<li></li>\n</ul>\n</li>\n</ul>\n</li>\n</ul>\n" +)] +#[case::text_after_the_markers( + "- - - a\n", + "<ul>\n<li>\n<ul>\n<li>\n<ul>\n<li>a</li>\n</ul>\n</li>\n</ul>\n</li>\n</ul>\n" +)] +#[case::text_before_the_end( + "- - - a -\n", + "<ul>\n<li>\n<ul>\n<li>\n<ul>\n<li>a -</li>\n</ul>\n</li>\n</ul>\n</li>\n</ul>\n" +)] +#[case::rule_after_text_in_an_item("- a\n ---\n", "<ul>\n<li>\n<h2>a</h2>\n</li>\n</ul>\n")] +fn thematic_breaks_are_told_from_other_lines(#[case] input: &str, #[case] expected: &str) { + assert_eq!(mq_markdown::to_html(input), expected); +} From b0c4e771bbd6e40b3832ee32692aef52e57a5a19 Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Thu, 8 Oct 2026 13:06:07 +0900 Subject: [PATCH 12/88] =?UTF-8?q?=F0=9F=90=9B=20fix(markdown):=20fix=20das?= =?UTF-8?q?h=20rules=20after=20wikilinks=20and=20embed=20labels=20in=20HTM?= =?UTF-8?q?L?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Treat wikilinks and embeds as paragraph text so a dash rule after them keeps its blank line and does not read back as a setext heading. - Use the display text as the label of a note embed. - Split an image embed display into alt text and a trailing size, so `![[img.png|photo|200]]` keeps both. --- crates/mq-markdown/src/node.rs | 8 ++++++ .../src/parser/render_html/obsidian.rs | 25 ++++++++++++++----- crates/mq-markdown/tests/obsidian_tests.rs | 22 ++++++++++++++++ 3 files changed, 49 insertions(+), 6 deletions(-) diff --git a/crates/mq-markdown/src/node.rs b/crates/mq-markdown/src/node.rs index 87f8933af..eac17d494 100644 --- a/crates/mq-markdown/src/node.rs +++ b/crates/mq-markdown/src/node.rs @@ -1971,6 +1971,14 @@ impl Node { /// True for the nodes that make up the text of a paragraph. pub(crate) fn is_paragraph_text(&self) -> bool { + #[cfg(feature = "wikilink")] + if matches!(self, Self::WikiLink(_)) { + return true; + } + #[cfg(feature = "embed")] + if matches!(self, Self::Embed(_)) { + return true; + } matches!( self, Self::Text(_) diff --git a/crates/mq-markdown/src/parser/render_html/obsidian.rs b/crates/mq-markdown/src/parser/render_html/obsidian.rs index 8fef3378d..5d534386e 100644 --- a/crates/mq-markdown/src/parser/render_html/obsidian.rs +++ b/crates/mq-markdown/src/parser/render_html/obsidian.rs @@ -59,6 +59,21 @@ fn size(display: &str) -> Option<(&str, Option<&str>)> { } } +/// The alt text and the size from a display like `alt`, `100`, or `alt|100`. +#[cfg(feature = "embed")] +fn alt_and_size(display: &str) -> (Option<&str>, Option<(&str, Option<&str>)>) { + match display.rsplit_once('|') { + Some((alt, last)) => match size(last) { + Some(dimensions) => (Some(alt), Some(dimensions)), + None => (Some(display), None), + }, + None => match size(display) { + Some(dimensions) => (None, Some(dimensions)), + None => (Some(display), None), + }, + } +} + impl Html { /// `<a>` to the target, which is plain text inside another link. #[cfg(any(feature = "wikilink", feature = "embed"))] @@ -90,11 +105,12 @@ impl Html { pub(super) fn embed(&mut self, target: &str, display: Option<&str>) { let kind = media(target); if matches!(kind, Media::Note) { - return self.internal_link("internal-link embed", target, &shown_target(target)); + let shown = display.map_or_else(|| shown_target(target), str::to_string); + return self.internal_link("internal-link embed", target, &shown); } let src = sanitize_with_protocols(target, &SAFE_PROTOCOL_SRC); - let dimensions = display.and_then(size); + let (alt, dimensions) = display.map_or((None, None), alt_and_size); let mut attributes = String::new(); if let Some((width, height)) = dimensions { attributes.push_str(&format!(" width=\"{width}\"")); @@ -106,10 +122,7 @@ impl Html { match kind { Media::Image => { self.out.push_str(&format!("<img src=\"{src}\" alt=\"")); - encode( - &mut self.out, - display.filter(|_| dimensions.is_none()).unwrap_or(target), - ); + encode(&mut self.out, alt.filter(|alt| !alt.is_empty()).unwrap_or(target)); self.out.push_str(&format!("\"{attributes} />")); } Media::Audio => self.out.push_str(&format!("<audio controls src=\"{src}\"></audio>")), diff --git a/crates/mq-markdown/tests/obsidian_tests.rs b/crates/mq-markdown/tests/obsidian_tests.rs index e3f4b97d3..bc3ae7bc1 100644 --- a/crates/mq-markdown/tests/obsidian_tests.rs +++ b/crates/mq-markdown/tests/obsidian_tests.rs @@ -134,6 +134,15 @@ fn writing_and_reading_back_keeps_the_nodes(#[case] input: &str) { assert_eq!(written, input); } +#[rstest] +#[case::after_wikilink("[[note]]\n- - -\n")] +#[case::after_embed("![[img.png]]\n- - -\n")] +fn a_dash_rule_after_inline_nodes_stays_a_rule(#[case] input: &str) { + let written = Markdown::from_markdown_str(input).unwrap().to_string(); + assert_eq!(parse(&written), parse(input), "{written:?}"); + assert!(!parse(&written).contains("heading"), "{written:?}"); +} + /// Title lines, and the title they have to give: markup in a title spreads it over several nodes. const TITLES: &[(&str, &str)] = &[ ("Title", "Title"), @@ -349,6 +358,19 @@ proptest! { "a ![[note#Part]] b\n", "<p>a <a class=\"internal-link embed\" href=\"note#Part\" data-href=\"note#Part\">note &gt; Part</a> b</p>\n" )] +#[case::image_with_alt_and_width( + "![[img.png|photo|200]]\n", + "<p><img src=\"img.png\" alt=\"photo\" width=\"200\" /></p>\n" +)] +#[case::image_with_alt_and_size( + "![[img.png|photo|200x100]]\n", + "<p><img src=\"img.png\" alt=\"photo\" width=\"200\" height=\"100\" /></p>\n" +)] +#[case::image_with_alt_and_pipe_text("![[img.png|a|b]]\n", "<p><img src=\"img.png\" alt=\"a|b\" /></p>\n")] +#[case::note_with_label( + "![[project-notes|Meeting notes]]\n", + "<p><a class=\"internal-link embed\" href=\"project-notes\" data-href=\"project-notes\">Meeting notes</a></p>\n" +)] fn html_renders_wikilinks_and_embeds(#[case] input: &str, #[case] expected: &str) { assert_eq!(mq_markdown::to_html(input), expected); assert_eq!(Markdown::from_markdown_str(input).unwrap().to_html(), expected); From 24e2fd7d44c1681c26eab4f198177eae4b416c33 Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Thu, 8 Oct 2026 13:06:13 +0900 Subject: [PATCH 13/88] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor(markdown):?= =?UTF-8?q?=20split=20block=20constructs=20into=20statically=20dispatched?= =?UTF-8?q?=20rules?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move each construct tried by parse_blocks into a BlockRule type chained in Rules, so the order is the precedence and the dispatch is static. The state handed between constructs is returned as a Step instead of local variables. --- crates/mq-markdown/src/parser/block.rs | 107 ++------ crates/mq-markdown/src/parser/block/rule.rs | 281 ++++++++++++++++++++ 2 files changed, 304 insertions(+), 84 deletions(-) create mode 100644 crates/mq-markdown/src/parser/block/rule.rs diff --git a/crates/mq-markdown/src/parser/block.rs b/crates/mq-markdown/src/parser/block.rs index 81d6882ab..4f7b108c5 100644 --- a/crates/mq-markdown/src/parser/block.rs +++ b/crates/mq-markdown/src/parser/block.rs @@ -4,19 +4,18 @@ //! the remaining lines again, up to [`MAX_DEPTH`] levels. The remaining leaf blocks (HTML, table, //! definition, frontmatter, math) are not handled yet, so their lines currently fall into paragraphs. -use super::code::{Fence, fenced_code, indented_code}; +use super::code::Fence; use super::definition; use super::html_flow::{self, Kind as HtmlKind}; use super::inline; use super::line::{Line, split_lines}; -use super::mdx_flow::{ - FlowOutcome, absorbed_until, blocks_lazy_continuation, looks_like_mdx_flow, mdx_flow, probe_mdx_flow, -}; +use super::mdx_flow::{FlowOutcome, absorbed_until, blocks_lazy_continuation, looks_like_mdx_flow, probe_mdx_flow}; use super::table; +mod rule; + use super::tree::{Block, FootnoteBlock, InlineBlock, InlineKind, InlineSource, Item, ListBlock, QuoteBlock}; -use crate::node::{ - HorizontalRule, HorizontalRuleMarker, Html, ListMarker, MdxJsEsm, Node, Point, Position, Toml, Yaml, -}; +use crate::node::{Html, ListMarker, Node, Point, Position, Toml, Yaml}; +use rule::{BlockRule, Cx, Rules, fallback}; /// Lines indented by this many columns or more are code, not other blocks. pub(super) const CODE_INDENT: usize = 4; @@ -102,83 +101,23 @@ fn parse_blocks(lines: &[Line<'_>], depth: usize, interrupting: Interrupt) -> Ve let separated = std::mem::take(&mut blank_between); let (columns, indent) = line.indent(); - let rest = &line.text[indent..]; - - if columns >= line.code_indent() && !interrupting.code { - index = indented_code(lines, index, &mut blocks); - } else if columns >= line.code_indent() { - // Indented code cannot interrupt a paragraph. - let (next, interrupt) = paragraph(lines, index, &mut blocks); - index = next; - after_paragraph = interrupt; - } else if let Some(fence) = Fence::open(rest, !line.mdx) { - // The end of an unclosed fence without content is quirky right after a container. - let own_end = closed_container - && !(separated - && matches!(blocks.last(), Some(Block::List(l)) if l.items.last().is_some_and(|i| i.children.is_empty()))); - index = fenced_code(lines, index, indent, &fence, own_end, &mut blocks); - } else if let Some(depth) = atx_depth(rest) { - blocks.push(atx_heading(line, indent, depth)); - index += 1; - } else if is_thematic_break(line, indent) { - blocks.push(Block::Node(Node::HorizontalRule(HorizontalRule { - marker: rest.chars().next().and_then(HorizontalRuleMarker::from_char), - position: Some(Position { - start: line.point(0), - end: line.end(), - }), - }))); - index += 1; - } else if containers && blockquote_marker(line).is_some() { - index = blockquote(lines, index, depth, interrupting, &mut blocks); - after_container = true; - } else if let Some(marker) = - ItemMarker::parse(line).filter(|marker| containers && (!interrupting.list || marker.interrupts_paragraph())) - { - index = list(lines, index, &marker, depth, interrupting, &mut blocks); - after_container = true; - } else if let Some(kind) = html_start(line, rest) { - index = html_block(lines, index, kind, &mut blocks); - } else if let Some(marker) = footnote_marker(line).filter(|_| containers) { - index = footnote(lines, index, &marker, depth, interrupting, &mut blocks); - after_container = true; - } else if let Some((items, next)) = table::parse(lines, index, interrupts_paragraph, ends_table) { - blocks.push(Block::Table(items)); - index = next; - } else if line.mdx - && depth == 0 - && indent == 0 - && (line.text.starts_with("import ") || line.text.starts_with("export ")) - { - // ESM goes on to the next blank line. Without a JavaScript parser it is taken as it is. - let end = lines[index..] - .iter() - .position(Line::is_blank) - .map_or(lines.len(), |offset| index + offset); - let parts = lines[index..end] - .iter() - .map(|line| (line.text, line.eol)) - .collect::<Vec<_>>(); - blocks.push(Block::Node(Node::MdxJsEsm(MdxJsEsm { - value: join_lines(&parts).into(), - position: Some(Position { - start: lines[index].point(0), - end: lines[end - 1].end(), - }), - }))); - index = end; - } else if line.mdx - && matches!(rest.as_bytes().first(), Some(b'<' | b'{')) - && let FlowOutcome::Flow(next) = mdx_flow(lines, index, &mut blocks) - { - index = next; - // Flow content that interrupted a paragraph keeps the restrictions that came with it. - after_paragraph = interrupting; - } else { - let (next, interrupt) = paragraph(lines, index, &mut blocks); - index = next; - after_paragraph = interrupt; - } + let cx = Cx { + lines, + index, + line, + columns, + indent, + rest: &line.text[indent..], + depth, + containers, + interrupting, + closed_container, + separated, + }; + let step = Rules::parse(&cx, &mut blocks).unwrap_or_else(|| fallback(&cx, &mut blocks)); + index = step.next; + after_container = step.container; + after_paragraph = step.interrupt; } blocks diff --git a/crates/mq-markdown/src/parser/block/rule.rs b/crates/mq-markdown/src/parser/block/rule.rs new file mode 100644 index 000000000..909c3e7e0 --- /dev/null +++ b/crates/mq-markdown/src/parser/block/rule.rs @@ -0,0 +1,281 @@ +//! Block constructs as rules that [`parse_blocks`](super::parse_blocks) tries in order. +//! +//! A rule starts at the current line, pushes its blocks and tells where parsing resumes. The rules are +//! types chained in [`Rules`], so the dispatch is static and the order is the precedence of the +//! constructs. The paragraph is not a rule: it is what is left when no rule starts. + +use super::{ + Interrupt, ItemMarker, atx_depth, atx_heading, blockquote, blockquote_marker, footnote, footnote_marker, + html_block, html_start, is_thematic_break, join_lines, list, paragraph, +}; +use crate::node::{HorizontalRule, HorizontalRuleMarker, MdxJsEsm, Node, Position}; +use crate::parser::code::{Fence, fenced_code, indented_code}; +use crate::parser::line::Line; +use crate::parser::mdx_flow::{FlowOutcome, mdx_flow}; +use crate::parser::table; +use crate::parser::tree::Block; + +/// The line a rule starts at and what is known about the lines before it. +pub(super) struct Cx<'a, 'l> { + pub(super) lines: &'l [Line<'a>], + pub(super) index: usize, + pub(super) line: &'l Line<'a>, + /// Columns and byte length of the indentation of `line`, and the text after it. + pub(super) columns: usize, + pub(super) indent: usize, + pub(super) rest: &'a str, + pub(super) depth: usize, + /// Whether containers can still be nested. + pub(super) containers: bool, + pub(super) interrupting: Interrupt, + /// Whether the previous block is a container. + pub(super) closed_container: bool, + /// Whether a blank line or an empty list item preceded `line`. + pub(super) separated: bool, +} + +/// Where parsing resumes after a rule and what the block restricts on the next line. +pub(super) struct Step { + pub(super) next: usize, + pub(super) container: bool, + pub(super) interrupt: Interrupt, +} + +impl Step { + fn to(next: usize) -> Self { + Self { + next, + container: false, + interrupt: Interrupt::default(), + } + } + + fn container(next: usize) -> Self { + Self { + container: true, + ..Self::to(next) + } + } + + fn interrupt((next, interrupt): (usize, Interrupt)) -> Self { + Self { + interrupt, + ..Self::to(next) + } + } +} + +/// A block construct. `parse` pushes to `blocks` only when it returns a step. +pub(super) trait BlockRule { + fn parse(cx: &Cx<'_, '_>, blocks: &mut Vec<Block>) -> Option<Step>; +} + +impl<A: BlockRule, B: BlockRule> BlockRule for (A, B) { + fn parse(cx: &Cx<'_, '_>, blocks: &mut Vec<Block>) -> Option<Step> { + A::parse(cx, blocks).or_else(|| B::parse(cx, blocks)) + } +} + +/// Lines a paragraph is made of when no rule starts. +pub(super) fn fallback(cx: &Cx<'_, '_>, blocks: &mut Vec<Block>) -> Step { + Step::interrupt(paragraph(cx.lines, cx.index, blocks)) +} + +macro_rules! chain { + ($rule:ty) => { $rule }; + ($rule:ty, $($rest:ty),+) => { ($rule, chain!($($rest),+)) }; +} + +/// The rules in the order they are tried. +pub(super) type Rules = chain!( + IndentedCode, + FencedCode, + AtxHeading, + ThematicBreak, + Blockquote, + List, + Html, + Footnote, + Table, + Esm, + MdxFlow +); + +/// Indented code, which cannot interrupt a paragraph and is part of it then. +pub(super) struct IndentedCode; + +impl BlockRule for IndentedCode { + fn parse(cx: &Cx<'_, '_>, blocks: &mut Vec<Block>) -> Option<Step> { + if cx.columns < cx.line.code_indent() { + return None; + } + Some(if cx.interrupting.code { + fallback(cx, blocks) + } else { + Step::to(indented_code(cx.lines, cx.index, blocks)) + }) + } +} + +pub(super) struct FencedCode; + +impl BlockRule for FencedCode { + fn parse(cx: &Cx<'_, '_>, blocks: &mut Vec<Block>) -> Option<Step> { + let fence = Fence::open(cx.rest, !cx.line.mdx)?; + // The end of an unclosed fence without content is quirky right after a container. + let own_end = cx.closed_container + && !(cx.separated + && matches!(blocks.last(), Some(Block::List(l)) if l.items.last().is_some_and(|i| i.children.is_empty()))); + Some(Step::to(fenced_code( + cx.lines, cx.index, cx.indent, &fence, own_end, blocks, + ))) + } +} + +pub(super) struct AtxHeading; + +impl BlockRule for AtxHeading { + fn parse(cx: &Cx<'_, '_>, blocks: &mut Vec<Block>) -> Option<Step> { + let depth = atx_depth(cx.rest)?; + blocks.push(atx_heading(cx.line, cx.indent, depth)); + Some(Step::to(cx.index + 1)) + } +} + +pub(super) struct ThematicBreak; + +impl BlockRule for ThematicBreak { + fn parse(cx: &Cx<'_, '_>, blocks: &mut Vec<Block>) -> Option<Step> { + if !is_thematic_break(cx.line, cx.indent) { + return None; + } + blocks.push(Block::Node(Node::HorizontalRule(HorizontalRule { + marker: cx.rest.chars().next().and_then(HorizontalRuleMarker::from_char), + position: Some(Position { + start: cx.line.point(0), + end: cx.line.end(), + }), + }))); + Some(Step::to(cx.index + 1)) + } +} + +pub(super) struct Blockquote; + +impl BlockRule for Blockquote { + fn parse(cx: &Cx<'_, '_>, blocks: &mut Vec<Block>) -> Option<Step> { + if !cx.containers || blockquote_marker(cx.line).is_none() { + return None; + } + Some(Step::container(blockquote( + cx.lines, + cx.index, + cx.depth, + cx.interrupting, + blocks, + ))) + } +} + +pub(super) struct List; + +impl BlockRule for List { + fn parse(cx: &Cx<'_, '_>, blocks: &mut Vec<Block>) -> Option<Step> { + let marker = ItemMarker::parse(cx.line) + .filter(|marker| cx.containers && (!cx.interrupting.list || marker.interrupts_paragraph()))?; + Some(Step::container(list( + cx.lines, + cx.index, + &marker, + cx.depth, + cx.interrupting, + blocks, + ))) + } +} + +pub(super) struct Html; + +impl BlockRule for Html { + fn parse(cx: &Cx<'_, '_>, blocks: &mut Vec<Block>) -> Option<Step> { + let kind = html_start(cx.line, cx.rest)?; + Some(Step::to(html_block(cx.lines, cx.index, kind, blocks))) + } +} + +pub(super) struct Footnote; + +impl BlockRule for Footnote { + fn parse(cx: &Cx<'_, '_>, blocks: &mut Vec<Block>) -> Option<Step> { + let marker = footnote_marker(cx.line).filter(|_| cx.containers)?; + Some(Step::container(footnote( + cx.lines, + cx.index, + &marker, + cx.depth, + cx.interrupting, + blocks, + ))) + } +} + +pub(super) struct Table; + +impl BlockRule for Table { + fn parse(cx: &Cx<'_, '_>, blocks: &mut Vec<Block>) -> Option<Step> { + let (items, next) = table::parse(cx.lines, cx.index, super::interrupts_paragraph, super::ends_table)?; + blocks.push(Block::Table(items)); + Some(Step::to(next)) + } +} + +/// MDX `import` and `export`, which go on to the next blank line. +pub(super) struct Esm; + +impl BlockRule for Esm { + fn parse(cx: &Cx<'_, '_>, blocks: &mut Vec<Block>) -> Option<Step> { + let line = cx.line; + if !(line.mdx + && cx.depth == 0 + && cx.indent == 0 + && (line.text.starts_with("import ") || line.text.starts_with("export "))) + { + return None; + } + // Without a JavaScript parser the module is taken as it is. + let end = cx.lines[cx.index..] + .iter() + .position(Line::is_blank) + .map_or(cx.lines.len(), |offset| cx.index + offset); + let parts = cx.lines[cx.index..end] + .iter() + .map(|line| (line.text, line.eol)) + .collect::<Vec<_>>(); + blocks.push(Block::Node(Node::MdxJsEsm(MdxJsEsm { + value: join_lines(&parts).into(), + position: Some(Position { + start: line.point(0), + end: cx.lines[end - 1].end(), + }), + }))); + Some(Step::to(end)) + } +} + +pub(super) struct MdxFlow; + +impl BlockRule for MdxFlow { + fn parse(cx: &Cx<'_, '_>, blocks: &mut Vec<Block>) -> Option<Step> { + if !(cx.line.mdx && matches!(cx.rest.as_bytes().first(), Some(b'<' | b'{'))) { + return None; + } + let FlowOutcome::Flow(next) = mdx_flow(cx.lines, cx.index, blocks) else { + return None; + }; + // Flow content that interrupted a paragraph keeps the restrictions that came with it. + Some(Step { + interrupt: cx.interrupting, + ..Step::to(next) + }) + } +} From b4c6d18c3d44dcdb8b7e67e304d1b0834ff814a3 Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Thu, 8 Oct 2026 13:06:20 +0900 Subject: [PATCH 14/88] =?UTF-8?q?=E2=9A=A1=20perf(markdown):=20remember=20?= =?UTF-8?q?missing=20terminators=20to=20keep=20inline=20scans=20linear?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Raw HTML comments, instructions, CDATA and declarations, and MDX expressions no longer scan to the end of the input from every opener. --- crates/mq-markdown/src/parser/inline.rs | 15 +++-- crates/mq-markdown/src/parser/inline/html.rs | 38 ++++++++--- crates/mq-markdown/src/parser/mdx.rs | 66 +++++++++++++++++-- crates/mq-markdown/src/parser/mdx_flow.rs | 4 +- .../mq-markdown/tests/parser_limits_tests.rs | 11 ++++ 5 files changed, 113 insertions(+), 21 deletions(-) diff --git a/crates/mq-markdown/src/parser/inline.rs b/crates/mq-markdown/src/parser/inline.rs index 6c4fdaea9..13de5268b 100644 --- a/crates/mq-markdown/src/parser/inline.rs +++ b/crates/mq-markdown/src/parser/inline.rs @@ -144,6 +144,8 @@ pub(super) fn parse(source: &InlineSource, references: &References) -> Result<Ve inactive_below: 0, pos: 0, run: 0, + misses: html::Misses::default(), + braces: mdx::Braces::default(), }; scanner.scan(); scanner.flush(); @@ -300,6 +302,10 @@ struct Scanner<'a> { pos: usize, /// Start of the plain text that has not been pushed as an item yet. run: usize, + /// Raw HTML terminators already known to be missing. + misses: html::Misses, + /// Where the braces of the content close, for MDX. + braces: mdx::Braces, } /// Bytes where `Scanner::scan` may start a construct. Every other byte is plain text. @@ -643,7 +649,8 @@ impl Scanner<'_> { /// `<` starts an autolink or raw inline HTML. fn angle(&mut self) { - let src = self.src(); + let context = self.context; + let src = context.src(); let pos = self.pos; if let Some(autolink) = html::autolink(src, pos) { let position = |start, end| Some(self.context.position(start, end)); @@ -657,7 +664,7 @@ impl Scanner<'_> { position: position(pos, autolink.end), }); self.push_node(node, autolink.end); - } else if let Some(end) = html::inline_html(src, pos) { + } else if let Some(end) = html::inline_html(src, pos, &mut self.misses) { // Lines of a paragraph lose their leading whitespace, also inside a tag. let value = src[pos..end] .split_inclusive('\n') @@ -682,7 +689,7 @@ impl Scanner<'_> { /// A JSX tag in MDX text. fn jsx(&mut self) { - match mdx::tag(self.src(), self.pos) { + match mdx::tag(self.src(), self.pos, Some(&self.braces)) { Parsed::Ok(tag) => { let end = tag.end; let item = Item::Jsx(TextTag { @@ -708,7 +715,7 @@ impl Scanner<'_> { /// An expression in MDX text. fn expression(&mut self) { - match mdx::expression(self.src(), self.pos) { + match mdx::expression(self.src(), self.pos, Some(&self.braces)) { Parsed::Ok((end, value)) => { let node = Node::MdxTextExpression(MdxTextExpression { value, diff --git a/crates/mq-markdown/src/parser/inline/html.rs b/crates/mq-markdown/src/parser/inline/html.rs index c25b1e4e8..0ad4dd4a8 100644 --- a/crates/mq-markdown/src/parser/inline/html.rs +++ b/crates/mq-markdown/src/parser/inline/html.rs @@ -85,28 +85,50 @@ fn email(bytes: &[u8], start: usize) -> Option<usize> { } } +/// Terminators that a search reached the end of the input without finding, so that later searches +/// for the same one fail without scanning again. Each is the first offset a search started from. +#[derive(Default)] +pub(super) struct Misses { + comment: Option<usize>, + instruction: Option<usize>, + cdata: Option<usize>, + declaration: Option<usize>, +} + +/// Finds `terminator` in `src` from `from`, recording a failure in `miss`. +fn find_from(src: &str, from: usize, terminator: &str, miss: &mut Option<usize>) -> Option<usize> { + if miss.is_some_and(|missed| from >= missed) { + return None; + } + let found = src[from..].find(terminator).map(|index| from + index); + if found.is_none() { + *miss = Some(from); + } + found +} + /// Parses raw inline HTML at `pos` (a `<`), returning the offset after it. -pub(super) fn inline_html(src: &str, pos: usize) -> Option<usize> { +pub(super) fn inline_html(src: &str, pos: usize, misses: &mut Misses) -> Option<usize> { let rest = &src[pos..]; let bytes = rest.as_bytes(); - if let Some(after) = rest.strip_prefix("<!--") { + if rest.starts_with("<!--") { if rest.starts_with("<!-->") { return Some(pos + 5); } if rest.starts_with("<!--->") { return Some(pos + 6); } - return after.find("-->").map(|index| pos + 4 + index + 3); + return find_from(src, pos + 4, "-->", &mut misses.comment).map(|index| index + 3); } - if let Some(after) = rest.strip_prefix("<?") { - return after.find("?>").map(|index| pos + 2 + index + 2); + if rest.starts_with("<?") { + return find_from(src, pos + 2, "?>", &mut misses.instruction).map(|index| index + 2); } - if let Some(after) = rest.strip_prefix("<![CDATA[") { - return after.find("]]>").map(|index| pos + 9 + index + 3); + if rest.starts_with("<![CDATA[") { + return find_from(src, pos + 9, "]]>", &mut misses.cdata).map(|index| index + 3); } if rest.starts_with("<!") && bytes.get(2).is_some_and(u8::is_ascii_alphabetic) { - return rest.find('>').map(|index| pos + index + 1); + return find_from(src, pos, ">", &mut misses.declaration).map(|index| index + 1); } if rest.starts_with("</") { return closing_tag(bytes).map(|length| pos + length); diff --git a/crates/mq-markdown/src/parser/mdx.rs b/crates/mq-markdown/src/parser/mdx.rs index 5d358fcbe..cc01bcb9e 100644 --- a/crates/mq-markdown/src/parser/mdx.rs +++ b/crates/mq-markdown/src/parser/mdx.rs @@ -3,6 +3,8 @@ use crate::node::{MdxAttributeContent, MdxAttributeValue, MdxJsxAttribute}; use smol_str::SmolStr; +use std::cell::OnceCell; +use std::collections::HashMap; /// What the end of the input means for a construct it ends inside of. pub(super) enum Fallback { @@ -57,9 +59,44 @@ enum Stop { type Step<T> = Result<T, Stop>; +const UNCLOSED_EXPRESSION: &str = + "Unexpected end of file in expression, expected a corresponding closing brace for `{`"; + +/// Where each `{` of a source closes, found in one pass the first time one is asked for, so that +/// many unclosed braces do not each scan to the end of the input. +#[derive(Default)] +pub(super) struct Braces(OnceCell<HashMap<usize, usize>>); + +impl Braces { + /// The offset of the `}` that closes the `{` at `open`, if there is one. + fn close(&self, src: &str, open: usize) -> Option<usize> { + self.0 + .get_or_init(|| { + let mut closes = HashMap::new(); + let mut opens = Vec::new(); + for (index, byte) in src.bytes().enumerate() { + match byte { + b'{' => opens.push(index), + b'}' => { + if let Some(open) = opens.pop() { + closes.insert(open, index); + } + } + _ => {} + } + } + closes + }) + .get(&open) + .copied() + } +} + struct Cursor<'a> { src: &'a str, index: usize, + /// Set when the same source is searched for many expressions. Without it each one is scanned. + braces: Option<&'a Braces>, } impl Cursor<'_> { @@ -143,15 +180,22 @@ impl Cursor<'_> { /// A `{...}` expression with balanced braces, returning the offsets of its content. fn expression(&mut self) -> Step<(usize, usize)> { + let open = self.index; self.bump(); let start = self.index; + if let Some(braces) = self.braces { + let Some(end) = braces.close(self.src, open) else { + self.index = self.src.len(); + return Err(Stop::More(Fallback::Error(UNCLOSED_EXPRESSION.into()))); + }; + self.index = end + 1; + return Ok((start, end)); + } let mut depth = 0usize; loop { match self.peek() { None => { - return Err(Stop::More(Fallback::Error( - "Unexpected end of file in expression, expected a corresponding closing brace for `{`".into(), - ))); + return Err(Stop::More(Fallback::Error(UNCLOSED_EXPRESSION.into()))); } Some('{') => depth += 1, Some('}') if depth == 0 => { @@ -360,14 +404,22 @@ fn parsed<T>(step: Step<T>) -> Parsed<T> { } /// Parses the JSX tag that starts at `pos`, a `<`. -pub(super) fn tag(src: &str, pos: usize) -> Parsed<Tag> { - let mut cursor = Cursor { src, index: pos + 1 }; +pub(super) fn tag(src: &str, pos: usize, braces: Option<&Braces>) -> Parsed<Tag> { + let mut cursor = Cursor { + src, + index: pos + 1, + braces, + }; parsed(cursor.tag()) } /// Parses the expression that starts at `pos`, a `{`, returning the offset after it and its value. -pub(super) fn expression(src: &str, pos: usize) -> Parsed<(usize, SmolStr)> { - let mut cursor = Cursor { src, index: pos }; +pub(super) fn expression(src: &str, pos: usize, braces: Option<&Braces>) -> Parsed<(usize, SmolStr)> { + let mut cursor = Cursor { + src, + index: pos, + braces, + }; parsed( cursor .expression() diff --git a/crates/mq-markdown/src/parser/mdx_flow.rs b/crates/mq-markdown/src/parser/mdx_flow.rs index 6fc74b457..41acd080d 100644 --- a/crates/mq-markdown/src/parser/mdx_flow.rs +++ b/crates/mq-markdown/src/parser/mdx_flow.rs @@ -29,7 +29,7 @@ fn flow_items(source: &InlineSource) -> Flow { loop { let start = index; let after_expression = match text[index..].chars().next() { - Some('<') => match mdx::tag(text, index) { + Some('<') => match mdx::tag(text, index, None) { Parsed::Ok(tag) => { index = tag.end; items.push(FlowItem::Tag(tag, start)); @@ -39,7 +39,7 @@ fn flow_items(source: &InlineSource) -> Flow { Parsed::More(fallback) => return Flow::More(fallback), Parsed::Error(message) => return Flow::Error(message), }, - Some('{') => match mdx::expression(text, index) { + Some('{') => match mdx::expression(text, index, None) { Parsed::Ok((end, value)) => { index = end; items.push(FlowItem::Expression(value, start, end)); diff --git a/crates/mq-markdown/tests/parser_limits_tests.rs b/crates/mq-markdown/tests/parser_limits_tests.rs index d3e80d2e1..b2ca5e5c3 100644 --- a/crates/mq-markdown/tests/parser_limits_tests.rs +++ b/crates/mq-markdown/tests/parser_limits_tests.rs @@ -73,10 +73,21 @@ fn elements_nested_within_the_limit_parse() { #[case::quote_markers_on_a_line(format!("{}a\n", "> ".repeat(20_000)).repeat(20))] #[case::numbered_markers_on_a_line(format!("{}a\n", "1. ".repeat(20_000)).repeat(20))] #[case::long_labels(format!("[a]: /u\n{}", "[b ".repeat(100_000)))] +#[case::unclosed_comments("a <!--".repeat(300_000))] +#[case::unclosed_comments_after_a_closer(format!("</{}", "<!--".repeat(300_000)))] +#[case::unclosed_instructions("a <?".repeat(300_000))] +#[case::unclosed_cdata("a <![CDATA[".repeat(300_000))] +#[case::unclosed_declarations("a <!A".repeat(300_000))] fn repeated_constructs_parse(#[case] input: String) { exercise(Markdown::from_markdown_str(&input).unwrap()); } +#[test] +fn repeated_unclosed_expressions_are_an_error() { + let error = Markdown::from_mdx_str(&"a {".repeat(300_000)).unwrap_err().to_string(); + assert!(error.contains("closing brace"), "{error}"); +} + #[rstest] #[case::emphasis( "*a **a *a b a* a** a*\n", From 9d585f0283454b98167f5fe3683a75b912591bfd Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Thu, 8 Oct 2026 13:06:26 +0900 Subject: [PATCH 15/88] =?UTF-8?q?=F0=9F=90=9B=20fix(markdown):=20allow=20w?= =?UTF-8?q?hitespace=20after=20=3D=20in=20MDX=20JSX=20attributes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crates/mq-markdown/src/parser.rs | 6 ++++++ crates/mq-markdown/src/parser/mdx.rs | 5 +---- crates/mq-markdown/src/parser/snapshots.txt | 14 +++++++++++++- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/crates/mq-markdown/src/parser.rs b/crates/mq-markdown/src/parser.rs index ee97ba263..3bc611d8f 100644 --- a/crates/mq-markdown/src/parser.rs +++ b/crates/mq-markdown/src/parser.rs @@ -70,6 +70,12 @@ mod tests { ("lt_space", "a < b"), ("lt_digit", "a <3"), ("attr_no_value", "<a b=>"), + ("attr_space_after_eq", "<b e= \"f\"/>"), + ("attr_spaces_around_eq", "a <b e = \"f\"/>."), + ("attr_line_before_eq", "<b c\n= \"x\">c</b>"), + ("attr_namespace_spaces", "a <b xml :\tlang\n= \"de-CH\" foo:bar>c</b>."), + ("attr_spaced_names", "a <b a b : c d : e = \"f\" g/>."), + ("attr_expr_space_after_eq", "<b e= {f}/>"), ("jsx_nested_flow", "<a>\n<b>\n</b>\n</a>"), ("jsx_in_quote", "> <a>\n> x\n> </a>"), ("jsx_in_list", "- <a>\n x\n </a>"), diff --git a/crates/mq-markdown/src/parser/mdx.rs b/crates/mq-markdown/src/parser/mdx.rs index cc01bcb9e..11003cdd1 100644 --- a/crates/mq-markdown/src/parser/mdx.rs +++ b/crates/mq-markdown/src/parser/mdx.rs @@ -273,10 +273,7 @@ impl Cursor<'_> { } self.bump(); - // Unlike before the `=`, no whitespace is allowed after it. - if self.peek().is_some_and(char::is_whitespace) { - return Err(Stop::Nok); - } + self.skip_whitespace()?; let value = match self.peek() { Some(quote @ ('"' | '\'')) => { self.bump(); diff --git a/crates/mq-markdown/src/parser/snapshots.txt b/crates/mq-markdown/src/parser/snapshots.txt index d66b5ba9e..bca07c272 100644 --- a/crates/mq-markdown/src/parser/snapshots.txt +++ b/crates/mq-markdown/src/parser/snapshots.txt @@ -1185,6 +1185,18 @@ nodes: [Text(Text { value: "a < b", @1:1-1:6 })] nodes: error: Unexpected character `3` (U+0033) before name ## mdx attr_no_value "<a b=>" nodes: error: Unexpected character `>` (U+003E) before attribute value +## mdx attr_space_after_eq "<b e= \"f\"/>" +nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [], @1:1-1:12, name: Some("b"), attributes: [Property(MdxJsxAttribute { name: "e", value: Some(Literal("f")) })] })] +## mdx attr_spaces_around_eq "a <b e = \"f\"/>." +nodes: [Text(Text { value: "a ", @1:1-1:3 }), MdxJsxTextElement(MdxJsxTextElement { children: [], @1:3-1:15, name: Some("b"), attributes: [Property(MdxJsxAttribute { name: "e", value: Some(Literal("f")) })] }), Text(Text { value: ".", @1:15-1:16 })] +## mdx attr_line_before_eq "<b c\n= \"x\">c</b>" +nodes: [MdxJsxTextElement(MdxJsxTextElement { children: [Text(Text { value: "c", @2:7-2:8 })], @1:1-2:12, name: Some("b"), attributes: [Property(MdxJsxAttribute { name: "c", value: Some(Literal("x")) })] })] +## mdx attr_namespace_spaces "a <b xml :\tlang\n= \"de-CH\" foo:bar>c</b>." +nodes: [Text(Text { value: "a ", @1:1-1:3 }), MdxJsxTextElement(MdxJsxTextElement { children: [Text(Text { value: "c", @2:19-2:20 })], @1:3-2:24, name: Some("b"), attributes: [Property(MdxJsxAttribute { name: "xml:lang", value: Some(Literal("de-CH")) }), Property(MdxJsxAttribute { name: "foo:bar", value: None })] }), Text(Text { value: ".", @2:24-2:25 })] +## mdx attr_spaced_names "a <b a b : c d : e = \"f\" g/>." +nodes: [Text(Text { value: "a ", @1:1-1:3 }), MdxJsxTextElement(MdxJsxTextElement { children: [], @1:3-1:29, name: Some("b"), attributes: [Property(MdxJsxAttribute { name: "a", value: None }), Property(MdxJsxAttribute { name: "b:c", value: None }), Property(MdxJsxAttribute { name: "d:e", value: Some(Literal("f")) }), Property(MdxJsxAttribute { name: "g", value: None })] }), Text(Text { value: ".", @1:29-1:30 })] +## mdx attr_expr_space_after_eq "<b e= {f}/>" +nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [], @1:1-1:12, name: Some("b"), attributes: [Property(MdxJsxAttribute { name: "e", value: Some(Expression("f")) })] })] ## mdx jsx_nested_flow "<a>\n<b>\n</b>\n</a>" nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [MdxJsxFlowElement(MdxJsxFlowElement { children: [], @2:1-3:5, name: Some("b"), attributes: [] })], @1:1-4:5, name: Some("a"), attributes: [] })] ## mdx jsx_in_quote "> <a>\n> x\n> </a>" @@ -1262,7 +1274,7 @@ nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [List(List { values: [Te ## mdx list_items_unbalanced "- <a>\n- </a>" nodes: error: Expected a closing tag for `<a>` (1:3) before the end of its container ## mdx spaced_equals "<a b = \"c\" />" -nodes: [Text(Text { value: "<a b = \"c\" />", @1:1-1:14 })] +nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [], @1:1-1:14, name: Some("a"), attributes: [Property(MdxJsxAttribute { name: "b", value: Some(Literal("c")) })] })] ## mdx heading "# h\n\ntext *a* [b](c) `d`\n" nodes: [Heading(Heading { depth: 1, values: [Text(Text { value: "h", @1:3-1:4 })], @1:1-1:4 }), Text(Text { value: "text ", @3:1-3:6 }), Emphasis(Emphasis { values: [Text(Text { value: "a", @3:7-3:8 })], @3:6-3:9 }), Text(Text { value: " ", @3:9-3:10 }), Link(Link { url: Url("c"), title: None, values: [Text(Text { value: "b", @3:11-3:12 })], @3:10-3:16 }), Text(Text { value: " ", @3:16-3:17 }), CodeInline(CodeInline { value: "d", @3:17-3:20 })] ## mdx indented_is_text " not code\n" From 10d6812bc559fd96a5685b43dd0c7e6b0758a015 Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Thu, 8 Oct 2026 13:06:32 +0900 Subject: [PATCH 16/88] =?UTF-8?q?=F0=9F=90=9B=20fix(markdown):=20write=20M?= =?UTF-8?q?DX=20flow=20element=20children=20on=20their=20own=20lines?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Children were joined to the tags on one line, which dropped headings and turned flow elements into text elements when read back. --- crates/mq-markdown/src/node.rs | 9 +++-- .../mq-markdown/tests/mdx_roundtrip_tests.rs | 36 +++++++++++++++++++ crates/mq-run/tests/cookbook_tests.rs | 2 +- .../src/cookbook/extract-mdx-components.md | 4 ++- 4 files changed, 46 insertions(+), 5 deletions(-) create mode 100644 crates/mq-markdown/tests/mdx_roundtrip_tests.rs diff --git a/crates/mq-markdown/src/node.rs b/crates/mq-markdown/src/node.rs index eac17d494..7c9f88310 100644 --- a/crates/mq-markdown/src/node.rs +++ b/crates/mq-markdown/src/node.rs @@ -1492,11 +1492,14 @@ impl Node { if mdx_jsx_flow_element.children.is_empty() { format!("<{}{} />", name, attributes,) } else { + // Each tag and the children sit on their own lines, or the children would read as + // inline content of the tag. + let children = render_values_block(&mdx_jsx_flow_element.children, options, theme); format!( - "<{}{}>{}</{}>", + "<{}{}>\n{}\n</{}>", name, attributes, - render_values(&mdx_jsx_flow_element.children, options, theme), + indent_continuation(&format!(" {}", children), 2), name ) } @@ -4581,7 +4584,7 @@ mod tests { "content".to_string().into() ], position: None - }), RenderOptions::default(), "<div className=\"container\">content</div>")] + }), RenderOptions::default(), "<div className=\"container\">\n content\n</div>")] #[case::mdx_jsx_flow_element(Node::MdxJsxFlowElement(MdxJsxFlowElement{ name: Some("div".to_string()), attributes: vec![ diff --git a/crates/mq-markdown/tests/mdx_roundtrip_tests.rs b/crates/mq-markdown/tests/mdx_roundtrip_tests.rs new file mode 100644 index 000000000..3b225c376 --- /dev/null +++ b/crates/mq-markdown/tests/mdx_roundtrip_tests.rs @@ -0,0 +1,36 @@ +//! MDX written out and read back keeps the kinds and values of its nodes. + +use mq_markdown::{Markdown, Node}; +use rstest::rstest; + +fn describe(node: &Node) -> String { + let children = node.children().iter().map(describe).collect::<Vec<_>>().join(", "); + format!("{}({:?})[{children}]", node.name(), node.value()) +} + +fn describe_all(markdown: &Markdown) -> Vec<String> { + markdown.nodes.iter().map(describe).collect() +} + +#[rstest] +#[case::blocks("<A>\n\n# h\n\n- l\n\n</A>")] +#[case::image("<Zoom>\n\n![a](b)\n\n</Zoom>")] +#[case::nested("<A>\n <B>\n x\n </B>\n</A>")] +#[case::nested_blocks("<A>\n\n<B>\n\n# h\n\n</B>\n\n- l\n\n</A>")] +#[case::code("<A>\n\n```js\nlet x = 1;\n\n y();\n```\n\n</A>")] +#[case::list_item("- <A>\n x\n </A>")] +#[case::quote("> <A>\n>\n> # h\n>\n> </A>")] +fn flow_elements_survive_a_round_trip(#[case] input: &str) { + let markdown = Markdown::from_mdx_str(input).unwrap(); + let written = markdown.to_string(); + let reparsed = Markdown::from_mdx_str(&written).unwrap_or_else(|error| panic!("{error}\n{written}")); + assert_eq!(describe_all(&markdown), describe_all(&reparsed), "{written}"); +} + +#[rstest] +#[case::blocks("<A>\n\n# h\n\n- l\n\n</A>", "<A>\n # h\n\n - l\n</A>")] +#[case::image("<Zoom>\n\n![a](b)\n\n</Zoom>", "<Zoom>\n ![a](b)\n</Zoom>")] +#[case::nested("<A>\n <B>\n x\n </B>\n</A>", "<A>\n <B>\n x\n </B>\n</A>")] +fn flow_elements_keep_tags_and_children_on_their_own_lines(#[case] input: &str, #[case] expected: &str) { + assert_eq!(Markdown::from_mdx_str(input).unwrap().to_string().trim_end(), expected); +} diff --git a/crates/mq-run/tests/cookbook_tests.rs b/crates/mq-run/tests/cookbook_tests.rs index 01c8234a0..155839baf 100644 --- a/crates/mq-run/tests/cookbook_tests.rs +++ b/crates/mq-run/tests/cookbook_tests.rs @@ -332,7 +332,7 @@ fn cookbook_extract_mdx_components() { let out = run(&["-I", "mdx", "select(is_mdx())", path.to_str().unwrap()]); assert_eq!( out.trim(), - "<CustomComponent prop=\"value\" />\n<AnotherComponent>Content</AnotherComponent>" + "<CustomComponent prop=\"value\" />\n<AnotherComponent>\n Content\n</AnotherComponent>" ); } diff --git a/docs/books/src/cookbook/extract-mdx-components.md b/docs/books/src/cookbook/extract-mdx-components.md index a88554d3e..639ecce44 100644 --- a/docs/books/src/cookbook/extract-mdx-components.md +++ b/docs/books/src/cookbook/extract-mdx-components.md @@ -32,7 +32,9 @@ Another paragraph. ```markdown <CustomComponent prop="value" /> -<AnotherComponent>Content</AnotherComponent> +<AnotherComponent> + Content +</AnotherComponent> ``` ## Notes From 87e8024b110a2647913a1226382bb732e82f3bf5 Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Thu, 8 Oct 2026 13:06:38 +0900 Subject: [PATCH 17/88] =?UTF-8?q?=F0=9F=90=9B=20fix(markdown):=20serialize?= =?UTF-8?q?=20MDX=20JSX=20attribute=20values=20to=20JSON?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Newtype variants cannot carry an internal tag, so literal and expression attributes failed to serialize. They now use adjacent tagging. --- crates/mq-markdown/src/markdown.rs | 23 +++++++++++++++++++++++ crates/mq-markdown/src/node.rs | 4 ++-- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/crates/mq-markdown/src/markdown.rs b/crates/mq-markdown/src/markdown.rs index c9718fd76..2799c2780 100644 --- a/crates/mq-markdown/src/markdown.rs +++ b/crates/mq-markdown/src/markdown.rs @@ -921,6 +921,29 @@ mod json_tests { assert!(json.contains("\"values\":")); } + #[test] + fn test_to_json_mdx_attributes() { + let md = Markdown::from_mdx_str("<a b=\"c\" d={e} f {...g} />").unwrap(); + let json: serde_json::Value = serde_json::from_str(&md.to_json().unwrap()).unwrap(); + let attributes = &json[0]["attributes"]; + assert_eq!( + attributes[0], + serde_json::json!({"type": "property", "value": {"type": "MdxJsxAttribute", "name": "b", "value": {"type": "literal", "value": "c"}}}) + ); + assert_eq!( + attributes[1], + serde_json::json!({"type": "property", "value": {"type": "MdxJsxAttribute", "name": "d", "value": {"type": "expression", "value": "e"}}}) + ); + assert_eq!( + attributes[2], + serde_json::json!({"type": "property", "value": {"type": "MdxJsxAttribute", "name": "f", "value": null}}) + ); + assert_eq!( + attributes[3], + serde_json::json!({"type": "expression", "value": "...g"}) + ); + } + #[test] fn test_to_json_complex() { let md = "# Header\n\n- Item 1\n- Item 2\n\n*Emphasis* and **Strong**" diff --git a/crates/mq-markdown/src/node.rs b/crates/mq-markdown/src/node.rs index 7c9f88310..061fa9392 100644 --- a/crates/mq-markdown/src/node.rs +++ b/crates/mq-markdown/src/node.rs @@ -746,7 +746,7 @@ pub struct MdxJsxFlowElement { #[cfg_attr( feature = "json", derive(serde::Serialize, serde::Deserialize), - serde(rename_all = "camelCase", tag = "type") + serde(rename_all = "camelCase", tag = "type", content = "value") )] pub enum MdxAttributeContent { Expression(SmolStr), @@ -768,7 +768,7 @@ pub struct MdxJsxAttribute { #[cfg_attr( feature = "json", derive(serde::Serialize, serde::Deserialize), - serde(rename_all = "camelCase", tag = "type") + serde(rename_all = "camelCase", tag = "type", content = "value") )] pub enum MdxAttributeValue { Expression(SmolStr), From 8ff1e6ea1ea06848be971aa5aa6cc9b4809141c1 Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Thu, 8 Oct 2026 13:06:44 +0900 Subject: [PATCH 18/88] =?UTF-8?q?=F0=9F=90=9B=20fix(markdown):=20keep=20th?= =?UTF-8?q?e=20notation=20of=20link=20references=20when=20writing=20them?= =?UTF-8?q?=20back?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Full references write the label as it was, and the check that decides between shortcut and full form uses the label normalization of the parser. --- crates/mq-markdown/src/node.rs | 36 ++++++++++++++----- crates/mq-markdown/src/parser.rs | 2 ++ crates/mq-markdown/src/parser/inline.rs | 2 +- crates/mq-markdown/src/parser/inline/link.rs | 2 +- .../tests/parser_structure_tests.rs | 18 ++++++++++ 5 files changed, 49 insertions(+), 11 deletions(-) diff --git a/crates/mq-markdown/src/node.rs b/crates/mq-markdown/src/node.rs index 061fa9392..dcdb5eea5 100644 --- a/crates/mq-markdown/src/node.rs +++ b/crates/mq-markdown/src/node.rs @@ -1388,11 +1388,17 @@ impl Node { } // Shortcut is safe only if `alt` normalizes back to `ident`; `ident` is raw // (already correctly escaped), so escape_label would double it up. - Self::ImageRef(ImageRef { alt, ident, .. }) => { + Self::ImageRef(ImageRef { alt, ident, label, .. }) => { let (is, ie) = &theme.image; - let mismatched = normalize_reference_identifier(alt) != ident.as_str(); + let mismatched = crate::parser::normalize(alt) != ident.as_str(); if mismatched || needs_broad_escaping(alt) { - format!("{}![{}][{}]{}", is, escape_label(alt), ident, ie) + format!( + "{}![{}][{}]{}", + is, + escape_label(alt), + reference_label(label.as_deref(), ident), + ie + ) } else { format!("{}![{}]{}", is, escape_label(alt), ie) } @@ -1444,16 +1450,24 @@ impl Node { } } // Same reasoning as ImageRef, plus the same broad-escaping fallback. - Self::LinkRef(LinkRef { values, ident, .. }) => { + Self::LinkRef(LinkRef { + values, ident, label, .. + }) => { let (ls, le) = &theme.link; let rendered = render_values(values, options, theme); let plain = values_to_value(values); // The label of a shortcut reference is its text as written, markup included. let written = render_values(values, options, &ColorTheme::PLAIN); - let mismatched = normalize_reference_identifier(&written) != ident.as_str(); + let mismatched = crate::parser::normalize(&written) != ident.as_str(); if mismatched || needs_broad_escaping(&plain) { - format!("{}[{}][{}]{}", ls, rendered, ident, le) + format!( + "{}[{}][{}]{}", + ls, + rendered, + reference_label(label.as_deref(), ident), + le + ) } else { format!("{}[{}]{}", ls, rendered, le) } @@ -3456,9 +3470,13 @@ fn code_fence(value: &str, info: &str) -> String { } } -/// CommonMark reference-label normalization: collapse whitespace, trim, case-fold. -fn normalize_reference_identifier(s: &str) -> String { - s.split_whitespace().collect::<Vec<_>>().join(" ").to_lowercase() +/// The label to write in the second brackets of a full reference. The label as written is kept when it +/// still resolves to `ident`. +fn reference_label(label: Option<&str>, ident: &str) -> String { + match label { + Some(label) if crate::parser::normalize(label) == ident => escape_label(label), + _ => ident.to_string(), + } } /// Picks an inline code span delimiter one backtick longer than the longest diff --git a/crates/mq-markdown/src/parser.rs b/crates/mq-markdown/src/parser.rs index 3bc611d8f..f4629eca6 100644 --- a/crates/mq-markdown/src/parser.rs +++ b/crates/mq-markdown/src/parser.rs @@ -23,6 +23,8 @@ mod tree; use crate::node::Node; +pub(crate) use inline::normalize; + /// Parses `content` into a flat list of nodes. pub(crate) fn parse(content: &str) -> miette::Result<Vec<Node>> { resolve::resolve(block::parse(content, false), false).map_err(|message| miette::miette!(message)) diff --git a/crates/mq-markdown/src/parser/inline.rs b/crates/mq-markdown/src/parser/inline.rs index 13de5268b..c22e79cbe 100644 --- a/crates/mq-markdown/src/parser/inline.rs +++ b/crates/mq-markdown/src/parser/inline.rs @@ -15,7 +15,7 @@ mod punctuation; mod tail; pub(super) use entity::{decode_references, remove_line_indent, unescape}; -pub(super) use link::normalize; +pub(crate) use link::normalize; pub(super) use tail::{destination, title_at}; use super::mdx::{self, Fallback, Parsed, TagKind}; diff --git a/crates/mq-markdown/src/parser/inline/link.rs b/crates/mq-markdown/src/parser/inline/link.rs index c9f8c8650..c4484e197 100644 --- a/crates/mq-markdown/src/parser/inline/link.rs +++ b/crates/mq-markdown/src/parser/inline/link.rs @@ -9,7 +9,7 @@ use crate::node::{RenderOptions, Text}; /// Normalizes a reference label: whitespace runs become one space, the ends are trimmed, and the case /// is folded. `markdown-rs` drops the first gap between words, which this does not. -pub(in crate::parser) fn normalize(label: &str) -> String { +pub(crate) fn normalize(label: &str) -> String { let bytes = label.as_bytes(); let mut result = String::with_capacity(label.len()); let mut in_whitespace = true; diff --git a/crates/mq-markdown/tests/parser_structure_tests.rs b/crates/mq-markdown/tests/parser_structure_tests.rs index ce40d6940..deeda0abf 100644 --- a/crates/mq-markdown/tests/parser_structure_tests.rs +++ b/crates/mq-markdown/tests/parser_structure_tests.rs @@ -311,6 +311,24 @@ fn rendering_reads_back_as_the_same_html(#[case] input: &str) { ); } +/// References keep the notation they were written in, as long as it resolves to the same definition. +#[rstest] +#[case::full_keeps_case_and_spaces("[x][Foo Bar]\n\n[foo bar]: /u\n", "[x][Foo Bar]")] +#[case::full_image("![x][Foo Bar]\n\n[foo bar]: /u\n", "![x][Foo Bar]")] +#[case::full_escaped_label("[x][a\\]b]\n\n[a\\]b]: /u\n", "[x][a\\]b]")] +#[case::shortcut_sharp_s("[Straße]\n\n[strasse]: /u\n", "[Straße]")] +#[case::shortcut_capital_sharp_s("[ẞ]\n\n[ss]: /u\n", "[ẞ]")] +#[case::collapsed("[Straße][]\n\n[strasse]: /u\n", "[Straße]")] +fn references_are_written_as_they_were(#[case] input: &str, #[case] expected: &str) { + let rendered = Markdown::from_markdown_str(input).unwrap().to_string(); + assert!(rendered.starts_with(expected), "{rendered:?}"); + assert_eq!( + mq_markdown::to_html(&rendered), + mq_markdown::to_html(input), + "{rendered:?}" + ); +} + /// What an MDX document starts its blocks with: `import` and `export` lines are one block up to the /// next blank line. #[rstest] From de9e1687523bc6a5a28e42f26e2d3e914ac2b320 Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Thu, 8 Oct 2026 13:06:50 +0900 Subject: [PATCH 19/88] =?UTF-8?q?=F0=9F=90=9B=20fix(markdown):=20keep=20th?= =?UTF-8?q?e=20meaning=20of=20text=20when=20writing=20it=20back?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Escape character references, a bang before a link and email-like text, write line endings at the ends of a heading as references, decode autolink literals in link text, and drop trailing whitespace-only lines from indented code. The three GFM spec examples that changed meaning on a round trip no longer do. --- crates/mq-markdown/src/markdown.rs | 4 +- crates/mq-markdown/src/node.rs | 105 +++++++++++++++--- crates/mq-markdown/src/parser/code.rs | 8 +- crates/mq-markdown/src/parser/inline/link.rs | 20 +++- .../tests/gfm_roundtrip_fidelity.rs | 6 +- .../tests/parser_structure_tests.rs | 31 ++++++ 6 files changed, 146 insertions(+), 28 deletions(-) diff --git a/crates/mq-markdown/src/markdown.rs b/crates/mq-markdown/src/markdown.rs index 2799c2780..a1437a66d 100644 --- a/crates/mq-markdown/src/markdown.rs +++ b/crates/mq-markdown/src/markdown.rs @@ -6,7 +6,7 @@ use crate::html_to_markdown::ConversionOptions; use crate::node::ListStyle; use crate::node::{ Code, ColorTheme, ListMarker, Node, Position, RenderOptions, TableAlign, TableCell, indent_lines, - list_own_prefix_width, reindent_all_lines, render_cell_values, render_values_block, + list_own_prefix_width, reindent_all_lines, render_before, render_cell_values, render_values_block, }; #[cfg(any(feature = "json", feature = "html-to-markdown"))] use miette::miette; @@ -184,7 +184,7 @@ impl Markdown { .render_with_theme(&self.options, theme) } else { list_indent_stack.clear(); - node.render_with_theme(&self.options, theme) + render_before(node, nodes.get(i + 1), &self.options, theme) }; if value.is_empty() || value == "\n" { diff --git a/crates/mq-markdown/src/node.rs b/crates/mq-markdown/src/node.rs index dcdb5eea5..5e1d1c42b 100644 --- a/crates/mq-markdown/src/node.rs +++ b/crates/mq-markdown/src/node.rs @@ -1345,6 +1345,18 @@ impl Node { Self::Heading(Heading { depth, values, .. }) => { let (hs, he) = &theme.heading; let text = render_values(values, options, theme); + // A line ending at either end is not part of the heading text, so it is written as a + // character reference. + let text = { + let inner = text.trim_matches('\n'); + let leading = text.len() - text.trim_start_matches('\n').len(); + let trailing = text.len() - text.trim_end_matches('\n').len(); + if inner.is_empty() || (leading == 0 && trailing == 0) { + text.clone() + } else { + format!("{}{}{}", "&#10;".repeat(leading), inner, "&#10;".repeat(trailing)) + } + }; // Multi-line content must stay setext for depths 1-2; ATX has no setext form. if text.contains('\n') && matches!(depth, 1 | 2) { let underline = if *depth == 1 { "===" } else { "---" }; @@ -3245,11 +3257,33 @@ pub(crate) fn render_cell_values(values: &[Node], options: &RenderOptions, theme result } +/// Renders `value`, escaping a `!` that ends its text when `next` starts a link, which would make an image. +pub(crate) fn render_before( + value: &Node, + next: Option<&Node>, + options: &RenderOptions, + theme: &ColorTheme<'_>, +) -> String { + let mut rendered = value.render_with_theme(options, theme); + let starts_link = match next { + Some(Node::Link(_) | Node::LinkRef(_)) => true, + #[cfg(feature = "wikilink")] + Some(Node::WikiLink(_)) => true, + _ => false, + }; + if starts_link && matches!(value, Node::Text(Text { position: Some(_), .. })) && rendered.ends_with('!') { + rendered.insert(rendered.len() - 1, '\\'); + } + rendered +} + pub(crate) fn render_values(values: &[Node], options: &RenderOptions, theme: &ColorTheme<'_>) -> String { let mut pre_position: Option<Position> = None; values .iter() - .map(|value| { + .enumerate() + .map(|(index, value)| { + let rendered = || render_before(value, values.get(index + 1), options, theme); if let Some(pos) = value.position() { let new_line_count = pre_position .as_ref() @@ -3270,25 +3304,17 @@ pub(crate) fn render_values(values: &[Node], options: &RenderOptions, theme: &Co pre_position = Some(pos); if space.is_empty() { - format!( - "{}{}", - "\n".repeat(new_line_count), - value.render_with_theme(options, theme) - ) + format!("{}{}", "\n".repeat(new_line_count), rendered()) } else { format!( "{}{}", "\n".repeat(new_line_count), - value - .render_with_theme(options, theme) - .lines() - .map(|line| format!("{}{}", space, line)) - .join("\n") + rendered().lines().map(|line| format!("{}{}", space, line)).join("\n") ) } } else { pre_position = None; - value.render_with_theme(options, theme) + rendered() } }) .collect::<String>() @@ -3331,7 +3357,11 @@ pub(crate) fn render_values_block(values: &[Node], options: &RenderOptions, them ) } else { let value = &values[index]; - (value.render_with_theme(options, theme), value.position(), index + 1) + ( + render_before(value, values.get(index + 1), options, theme), + value.position(), + index + 1, + ) }; index = next; @@ -3536,6 +3566,40 @@ fn autolink_target(url: &str, values: &[Node]) -> Option<String> { (plain && (scheme || email) && escape_text(target.to_string()) != target).then(|| target.to_string()) } +/// Whether `rest`, the text after an `&`, reads as the rest of a character reference. +fn starts_reference(rest: &str) -> bool { + let name = match rest.strip_prefix('#') { + Some(number) => match number.strip_prefix(['x', 'X']) { + Some(hex) => hex + .split_once(';') + .map(|(digits, _)| (digits, char::is_ascii_hexdigit as fn(&char) -> bool)), + None => number + .split_once(';') + .map(|(digits, _)| (digits, char::is_ascii_digit as fn(&char) -> bool)), + }, + None => rest + .split_once(';') + .map(|(name, _)| (name, char::is_ascii_alphanumeric as fn(&char) -> bool)), + }; + name.is_some_and(|(name, is_valid)| !name.is_empty() && name.chars().all(|c| is_valid(&c))) +} + +fn is_email_local(c: char) -> bool { + c.is_ascii_alphanumeric() || matches!(c, '.' | '-' | '_' | '+') +} + +/// Whether `rest`, the text after an `@`, starts with a domain that makes an email address of the text +/// before it. +fn is_email_domain(rest: &str) -> bool { + let domain = rest + .split(|c: char| !(c.is_ascii_alphanumeric() || matches!(c, '.' | '-' | '_'))) + .next() + .unwrap_or_default(); + domain + .split_once('.') + .is_some_and(|(label, after)| !label.is_empty() && after.starts_with(|c: char| c.is_ascii_alphanumeric())) +} + /// Re-escapes markdown-significant characters so plain text can't be reinterpreted as /// syntax on re-parse. `# - + > =` only unsafe at line start, `. )` only after leading digits; /// a leading `\t` and runs of 2+ `\n` (only possible via decoded entities) use `&#N;` instead. @@ -3563,6 +3627,8 @@ fn escape_text(value: String) -> String { | b')' | b'\n' | b'\t' + | b'&' + | b'@' ) }); if !needs_escaping { @@ -3573,11 +3639,15 @@ fn escape_text(value: String) -> String { let mut at_line_start = true; // True while in an unbroken digit run since line start (ordered-list marker number). let mut leading_digits = false; - let mut chars = value.chars().peekable(); + let mut chars = value + .char_indices() + .map(|(index, c)| (c, &value[index + c.len_utf8()..])) + .peekable(); + let mut previous = None; - while let Some(c) = chars.next() { + while let Some((c, rest)) = chars.next() { match c { - '\n' if chars.peek() == Some(&'\n') => { + '\n' if chars.peek().is_some_and(|(next, _)| *next == '\n') => { result.push_str("&#10;"); at_line_start = true; leading_digits = false; @@ -3592,9 +3662,12 @@ fn escape_text(value: String) -> String { '\\' | '`' | '*' | '_' | '[' | ']' | '|' | '~' | '$' | '<' => result.push('\\'), '#' | '-' | '+' | '>' | '=' if at_line_start => result.push('\\'), '.' | ')' if leading_digits => result.push('\\'), + '&' if starts_reference(rest) => result.push('\\'), + '@' if previous.is_some_and(is_email_local) && is_email_domain(rest) => result.push('\\'), _ => {} } result.push(c); + previous = Some(c); let is_digit = c.is_ascii_digit(); leading_digits = if at_line_start { diff --git a/crates/mq-markdown/src/parser/code.rs b/crates/mq-markdown/src/parser/code.rs index 8103cf01a..2d1071856 100644 --- a/crates/mq-markdown/src/parser/code.rs +++ b/crates/mq-markdown/src/parser/code.rs @@ -135,8 +135,12 @@ pub(super) fn indented_code(lines: &[Line<'_>], start: usize, blocks: &mut Vec<B (content.pad, content.text, line.eol) }) .collect::<Vec<_>>(); - // Lines that are empty once the indent is removed only extend the position, not the value. - while parts.len() > 1 && parts.last().is_some_and(|(pad, text, _)| *pad == 0 && text.is_empty()) { + // Trailing lines of whitespace only extend the position, not the value. + while parts.len() > 1 + && parts + .last() + .is_some_and(|(_, text, _)| text.trim_matches([' ', '\t']).is_empty()) + { parts.pop(); } // What is left of a tab that the indent consumes in part is code, as spaces. diff --git a/crates/mq-markdown/src/parser/inline/link.rs b/crates/mq-markdown/src/parser/inline/link.rs index c4484e197..2cd213af7 100644 --- a/crates/mq-markdown/src/parser/inline/link.rs +++ b/crates/mq-markdown/src/parser/inline/link.rs @@ -3,9 +3,9 @@ use super::entity::{remove_line_indent, unescape}; use super::tail::{Tail, inline_tail, reference}; use super::{Context, Item, MAX_NESTING, Scanner, Value, emphasis, item_depth, to_nodes}; -use crate::node::{FootnoteRef, Image, ImageRef, Link, LinkRef, Node, Position, Title, Url}; #[cfg(feature = "wikilink")] -use crate::node::{RenderOptions, Text}; +use crate::node::RenderOptions; +use crate::node::{FootnoteRef, Image, ImageRef, Link, LinkRef, Node, Position, Text, Title, Url}; /// Normalizes a reference label: whitespace runs become one space, the ends are trimmed, and the case /// is folded. `markdown-rs` drops the first gap between words, which this does not. @@ -203,7 +203,21 @@ fn unwrap_links(values: Vec<Node>) -> Vec<Node> { values .into_iter() .flat_map(|child| match child { - Node::Link(Link { values, .. }) => values, + Node::Link(Link { values, position, .. }) => values + .into_iter() + .map(|value| match value { + // An autolink literal spans exactly its text, which is not decoded as it was linked. In + // link text it is ordinary text, so escapes and references in it apply. + Node::Text(Text { + value, + position: text_position, + }) if text_position == position => Node::Text(Text { + value: unescape(&value), + position: text_position, + }), + other => other, + }) + .collect(), #[cfg(feature = "wikilink")] Node::WikiLink(link) => vec![Node::Text(Text { value: Node::WikiLink(link).to_string_with(&RenderOptions::default()), diff --git a/crates/mq-markdown/tests/gfm_roundtrip_fidelity.rs b/crates/mq-markdown/tests/gfm_roundtrip_fidelity.rs index 7e8f68831..f47d290cc 100644 --- a/crates/mq-markdown/tests/gfm_roundtrip_fidelity.rs +++ b/crates/mq-markdown/tests/gfm_roundtrip_fidelity.rs @@ -69,11 +69,7 @@ fn parse_spec_examples(text: &str) -> Vec<SpecExample> { /// Examples where the round trip currently changes rendered meaning, by /// number in the spec revision pinned above. Remove an entry once it's /// fixed; add one (with its section) if a new gap turns up. -const KNOWN_FAILURES: &[usize] = &[ - 300, // Backslash escapes - 589, // Images - 602, // Autolinks -]; +const KNOWN_FAILURES: &[usize] = &[]; #[test] #[ignore = "fetches spec.txt over the network; run via `just test-gfm-spec`"] diff --git a/crates/mq-markdown/tests/parser_structure_tests.rs b/crates/mq-markdown/tests/parser_structure_tests.rs index deeda0abf..fd7832f0e 100644 --- a/crates/mq-markdown/tests/parser_structure_tests.rs +++ b/crates/mq-markdown/tests/parser_structure_tests.rs @@ -302,6 +302,20 @@ fn rendering_dashes_at_start_keeps_the_rules(#[case] input: &str) { #[case::code_after_nested_list("- a\n - s\n ```\n code\n ```\n")] #[case::nested_lists_around_paragraph("- a\n - s\n\n b\n - t\n")] #[case::ordered_item_with_nested_list_and_paragraph("1. x\n - s\n\n para\n2. y\n")] +#[case::escaped_reference("\\&ouml; x\n")] +#[case::escaped_numeric_reference("\\&#35; x \\&#x23; y\n")] +#[case::ampersand_without_a_reference("a & b &c d &amp\n")] +#[case::bang_before_a_link("\\![foo]\n\n[foo]: /url\n")] +#[case::bang_before_an_inline_link("wow\\![a](b)\n")] +#[case::bang_before_a_full_reference("\\![a][foo]\n\n[foo]: /url\n")] +#[case::escaped_email("<foo\\+@bar.example.com>\n")] +#[case::escaped_email_domain("<a\\_b@c.de> x\n")] +#[case::heading_with_a_trailing_line_ending("# a&#10;\n")] +#[case::heading_with_a_leading_line_ending("# &#10;a\n")] +#[case::heading_with_a_line_ending_inside("# a&#10;b\n")] +#[case::link_text_that_looks_like_a_url("[https://a.com/x\\_y](u)\n")] +#[case::link_text_with_a_reference_in_a_url("[https://a.com/x&amp;y](u)\n")] +#[case::code_ending_in_whitespace(" brew install x\n \n# h\n")] fn rendering_reads_back_as_the_same_html(#[case] input: &str) { let rendered = Markdown::from_markdown_str(input).unwrap().to_string(); assert_eq!( @@ -311,6 +325,23 @@ fn rendering_reads_back_as_the_same_html(#[case] input: &str) { ); } +#[rstest] +#[case::escape("[https://a.com/x\\_y](u)\n", "https://a.com/x_y")] +#[case::reference("[https://a.com/x&amp;y](u)\n", "https://a.com/x&y")] +#[case::www("[www.a.com/x\\_y](u)\n", "www.a.com/x_y")] +fn link_text_that_looks_like_a_url_is_decoded(#[case] input: &str, #[case] expected: &str) { + let nodes = Markdown::from_markdown_str(input).unwrap().nodes; + assert_eq!(nodes[0].children()[0].value(), expected, "{nodes:?}"); +} + +#[test] +fn indented_code_drops_trailing_lines_of_whitespace() { + let nodes = Markdown::from_markdown_str(" brew install x\n \n\t\n# h\n") + .unwrap() + .nodes; + assert_eq!(nodes[0].value(), "brew install x"); +} + /// References keep the notation they were written in, as long as it resolves to the same definition. #[rstest] #[case::full_keeps_case_and_spaces("[x][Foo Bar]\n\n[foo bar]: /u\n", "[x][Foo Bar]")] From 3b1533e0b995c90665db97646332c9f20c858cb9 Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Thu, 8 Oct 2026 13:06:56 +0900 Subject: [PATCH 20/88] =?UTF-8?q?=E2=9A=A1=20perf(markdown):=20find=20code?= =?UTF-8?q?=20and=20math=20span=20closers=20from=20a=20table=20of=20run=20?= =?UTF-8?q?lengths?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Spans whose opener has no closer no longer scan to the end of the input, so runs of backticks or dollar signs of every length parse in linear time. --- crates/mq-markdown/src/parser/inline.rs | 56 +++++++++++++------ .../mq-markdown/tests/parser_limits_tests.rs | 2 + 2 files changed, 41 insertions(+), 17 deletions(-) diff --git a/crates/mq-markdown/src/parser/inline.rs b/crates/mq-markdown/src/parser/inline.rs index c22e79cbe..c64b5a191 100644 --- a/crates/mq-markdown/src/parser/inline.rs +++ b/crates/mq-markdown/src/parser/inline.rs @@ -26,7 +26,8 @@ use crate::node::{ }; use smol_str::SmolStr; use std::borrow::Cow; -use std::cell::RefCell; +use std::cell::{OnceCell, RefCell}; +use std::collections::HashMap; /// The value of a text item: a slice of the source, or a decoded string. pub(super) enum Value { @@ -145,6 +146,7 @@ pub(super) fn parse(source: &InlineSource, references: &References) -> Result<Ve pos: 0, run: 0, misses: html::Misses::default(), + runs: SpanRuns::default(), braces: mdx::Braces::default(), }; scanner.scan(); @@ -290,6 +292,39 @@ fn text_element(context: &Context<'_>, tag: TextTag, children: Vec<Node>, end: u }) } +/// Where the runs of backticks and of dollar signs start, by length, found in one pass the first time a +/// span asks, so that openers without a closer do not each scan to the end. +#[derive(Default)] +struct SpanRuns { + backticks: OnceCell<HashMap<usize, Vec<usize>>>, + dollars: OnceCell<HashMap<usize, Vec<usize>>>, +} + +impl SpanRuns { + /// The start of the first run of exactly `size` bytes `ch` that starts at or after `from`. + fn next_of_length(&self, src: &str, ch: u8, size: usize, from: usize) -> Option<usize> { + let runs = if ch == b'`' { &self.backticks } else { &self.dollars }; + let starts = runs + .get_or_init(|| { + let bytes = src.as_bytes(); + let mut runs: HashMap<usize, Vec<usize>> = HashMap::new(); + let mut index = 0; + while index < bytes.len() { + if bytes[index] == ch { + let length = bytes[index..].iter().take_while(|&&b| b == ch).count(); + runs.entry(length).or_default().push(index); + index += length; + } else { + index += 1; + } + } + runs + }) + .get(&size)?; + starts.get(starts.partition_point(|&start| start < from)).copied() + } +} + struct Scanner<'a> { context: &'a Context<'a>, items: Vec<Item>, @@ -304,6 +339,7 @@ struct Scanner<'a> { run: usize, /// Raw HTML terminators already known to be missing. misses: html::Misses, + runs: SpanRuns, /// Where the braces of the content close, for MDX. braces: mdx::Braces, } @@ -545,26 +581,12 @@ impl Scanner<'_> { /// A code span (`` ` ``) or an inline math span (`$`). fn span(&mut self, ch: u8) { - let src = self.src(); + let src = self.context.src(); let bytes = src.as_bytes(); let start = self.pos; let size = bytes[start..].iter().take_while(|&&b| b == ch).count(); - // Find the next run of exactly the same length. - let mut index = start + size; - let mut close = None; - while index < bytes.len() { - if bytes[index] == ch { - let run = bytes[index..].iter().take_while(|&&b| b == ch).count(); - if run == size { - close = Some(index); - break; - } - index += run; - } else { - index += 1; - } - } + let close = self.runs.next_of_length(src, ch, size, start + size); let Some(close) = close else { self.pos = start + size; diff --git a/crates/mq-markdown/tests/parser_limits_tests.rs b/crates/mq-markdown/tests/parser_limits_tests.rs index b2ca5e5c3..8fa1d7c7e 100644 --- a/crates/mq-markdown/tests/parser_limits_tests.rs +++ b/crates/mq-markdown/tests/parser_limits_tests.rs @@ -73,6 +73,8 @@ fn elements_nested_within_the_limit_parse() { #[case::quote_markers_on_a_line(format!("{}a\n", "> ".repeat(20_000)).repeat(20))] #[case::numbered_markers_on_a_line(format!("{}a\n", "1. ".repeat(20_000)).repeat(20))] #[case::long_labels(format!("[a]: /u\n{}", "[b ".repeat(100_000)))] +#[case::backtick_runs_of_every_length((1..3_000).map(|length| format!("e{}", "`".repeat(length))).collect::<String>())] +#[case::dollar_runs_of_every_length((1..3_000).map(|length| format!("e{}", "$".repeat(length))).collect::<String>())] #[case::unclosed_comments("a <!--".repeat(300_000))] #[case::unclosed_comments_after_a_closer(format!("</{}", "<!--".repeat(300_000)))] #[case::unclosed_instructions("a <?".repeat(300_000))] From 6c1246fd3c5c607c2c6b231d2a9ebc7c1f8c401e Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Thu, 8 Oct 2026 13:07:02 +0900 Subject: [PATCH 21/88] =?UTF-8?q?=F0=9F=90=9B=20fix(markdown):=20skip=20st?= =?UTF-8?q?rings,=20template=20literals=20and=20comments=20when=20matching?= =?UTF-8?q?=20MDX=20braces?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A brace inside a JavaScript string or comment ended the expression early, so valid MDX such as {'}'} was read as an expression followed by text. --- crates/mq-markdown/src/parser.rs | 2 +- crates/mq-markdown/src/parser/mdx.rs | 132 ++++++++++++------ .../tests/parser_structure_tests.rs | 23 +++ 3 files changed, 114 insertions(+), 43 deletions(-) diff --git a/crates/mq-markdown/src/parser.rs b/crates/mq-markdown/src/parser.rs index f4629eca6..8a503f37a 100644 --- a/crates/mq-markdown/src/parser.rs +++ b/crates/mq-markdown/src/parser.rs @@ -5,7 +5,7 @@ //! text of paragraphs, headings and table cells along the way. //! //! It reads `CommonMark`, GFM, frontmatter, math and MDX (without a JavaScript parser, so expressions -//! only need balanced braces). HTML is rendered by [`render_html`]. +//! only need balanced braces outside of strings and comments). HTML is rendered by [`render_html`]. mod block; #[cfg(feature = "callout")] mod callout; diff --git a/crates/mq-markdown/src/parser/mdx.rs b/crates/mq-markdown/src/parser/mdx.rs index 11003cdd1..ace22781d 100644 --- a/crates/mq-markdown/src/parser/mdx.rs +++ b/crates/mq-markdown/src/parser/mdx.rs @@ -1,5 +1,5 @@ //! MDX: JSX tags and expressions, ported from the rules of `markdown-rs` without a JavaScript parser, -//! so expressions are only checked for balanced braces. +//! so expressions are only checked for balanced braces, outside of strings, template literals and comments. use crate::node::{MdxAttributeContent, MdxAttributeValue, MdxJsxAttribute}; use smol_str::SmolStr; @@ -70,26 +70,89 @@ pub(super) struct Braces(OnceCell<HashMap<usize, usize>>); impl Braces { /// The offset of the `}` that closes the `{` at `open`, if there is one. fn close(&self, src: &str, open: usize) -> Option<usize> { - self.0 - .get_or_init(|| { - let mut closes = HashMap::new(); - let mut opens = Vec::new(); - for (index, byte) in src.bytes().enumerate() { - match byte { - b'{' => opens.push(index), - b'}' => { - if let Some(open) = opens.pop() { - closes.insert(open, index); - } + self.0.get_or_init(|| match_braces(src, 0, false)).get(&open).copied() + } +} + +enum Frame { + /// A `{`, or a `${` of a template literal when it has no offset. + Brace(Option<usize>), + Template, +} + +/// Matches the braces of the expressions in `src` from `from`, which are not checked as JavaScript but +/// skip over strings, template literals and comments. Text outside braces is not looked at. With +/// `single`, `from` is a `{` and the search ends once it is closed. +fn match_braces(src: &str, from: usize, single: bool) -> HashMap<usize, usize> { + let bytes = src.as_bytes(); + let mut closes = HashMap::new(); + let mut stack = Vec::new(); + let mut index = from; + + while index < bytes.len() { + let byte = bytes[index]; + index += 1; + match stack.last() { + None => { + if byte == b'{' { + stack.push(Frame::Brace(Some(index - 1))); + } + } + Some(Frame::Template) => match byte { + b'\\' => index += 1, + b'`' => { + stack.pop(); + } + b'$' if bytes.get(index) == Some(&b'{') => { + stack.push(Frame::Brace(None)); + index += 1; + } + _ => {} + }, + Some(Frame::Brace(_)) => match byte { + b'{' => stack.push(Frame::Brace(Some(index - 1))), + b'}' => { + if let Some(Frame::Brace(Some(open))) = stack.pop() { + closes.insert(open, index - 1); + if single && open == from { + break; } - _ => {} } } - closes - }) - .get(&open) - .copied() + b'`' => stack.push(Frame::Template), + b'\'' | b'"' => index = skip_string(bytes, index, byte).unwrap_or(index), + b'/' => match bytes.get(index) { + Some(b'/') => { + index += bytes[index..].iter().take_while(|&&b| b != b'\n').count(); + } + Some(b'*') => { + if let Some(end) = src[index + 1..].find("*/") { + index += 1 + end + 2; + } + } + _ => {} + }, + _ => {} + }, + } } + closes +} + +/// The offset after the string that starts before `from` with the `quote`. A string that does not end on +/// its line is not one. +fn skip_string(bytes: &[u8], from: usize, quote: u8) -> Option<usize> { + let mut index = from; + while let Some(&byte) = bytes.get(index) { + match byte { + b'\\' => index += 1, + b'\n' | b'\r' => return None, + _ if byte == quote => return Some(index + 1), + _ => {} + } + index += 1; + } + None } struct Cursor<'a> { @@ -183,31 +246,16 @@ impl Cursor<'_> { let open = self.index; self.bump(); let start = self.index; - if let Some(braces) = self.braces { - let Some(end) = braces.close(self.src, open) else { - self.index = self.src.len(); - return Err(Stop::More(Fallback::Error(UNCLOSED_EXPRESSION.into()))); - }; - self.index = end + 1; - return Ok((start, end)); - } - let mut depth = 0usize; - loop { - match self.peek() { - None => { - return Err(Stop::More(Fallback::Error(UNCLOSED_EXPRESSION.into()))); - } - Some('{') => depth += 1, - Some('}') if depth == 0 => { - let end = self.index; - self.bump(); - return Ok((start, end)); - } - Some('}') => depth -= 1, - Some(_) => {} - } - self.bump(); - } + let end = match self.braces { + Some(braces) => braces.close(self.src, open), + None => match_braces(self.src, open, true).get(&open).copied(), + }; + let Some(end) = end else { + self.index = self.src.len(); + return Err(Stop::More(Fallback::Error(UNCLOSED_EXPRESSION.into()))); + }; + self.index = end + 1; + Ok((start, end)) } } diff --git a/crates/mq-markdown/tests/parser_structure_tests.rs b/crates/mq-markdown/tests/parser_structure_tests.rs index fd7832f0e..23729391f 100644 --- a/crates/mq-markdown/tests/parser_structure_tests.rs +++ b/crates/mq-markdown/tests/parser_structure_tests.rs @@ -360,6 +360,29 @@ fn references_are_written_as_they_were(#[case] input: &str, #[case] expected: &s ); } +/// Braces in strings, template literals and comments do not end an expression. +#[rstest] +#[case::string("{'}'}", "'}'")] +#[case::double_quoted("{\"}\" + '{'}", "\"}\" + '{'")] +#[case::escaped_quote("{'\\'}'}", "'\\'}'")] +#[case::block_comment("{a /* } */}", "a /* } */")] +#[case::line_comment("{a // }\n}", "a // }\n")] +#[case::template("{`a${`}`}c`}", "`a${`}`}c`")] +#[case::division("{a / b}", "a / b")] +fn mdx_expression_braces(#[case] input: &str, #[case] expected: &str) { + let nodes = Markdown::from_mdx_str(input).unwrap().nodes; + assert!( + matches!(&nodes[..], [Node::MdxFlowExpression(expression)] if expression.value == expected), + "{nodes:?}" + ); +} + +#[test] +fn mdx_attribute_expression_braces() { + let nodes = Markdown::from_mdx_str("<a b={'}'} />").unwrap().nodes; + assert!(matches!(&nodes[..], [Node::MdxJsxFlowElement(_)]), "{nodes:?}"); +} + /// What an MDX document starts its blocks with: `import` and `export` lines are one block up to the /// next blank line. #[rstest] From 056eb68065c573feaccee2ba79461421f8976333 Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Thu, 8 Oct 2026 13:07:09 +0900 Subject: [PATCH 22/88] =?UTF-8?q?=E2=9C=85=20test(markdown):=20compare=20M?= =?UTF-8?q?DX=20nodes=20with=20the=20official=20MDX=20packages?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit just test-mdx-official parses a corpus with mdast-util-from-markdown and micromark-extension-mdxjs and fails on differences that are not listed in known.json. just test-spec runs the CommonMark, GFM and extensions spec examples. --- crates/mq-markdown/tests/mdx_official.rs | 75 ++ .../tests/mdx_official/compare.mjs | 95 ++ .../tests/mdx_official/corpus.json | 198 +++ .../mq-markdown/tests/mdx_official/known.json | 23 + .../tests/mdx_official/package-lock.json | 1175 +++++++++++++++++ .../tests/mdx_official/package.json | 10 + justfile | 10 + 7 files changed, 1586 insertions(+) create mode 100644 crates/mq-markdown/tests/mdx_official.rs create mode 100644 crates/mq-markdown/tests/mdx_official/compare.mjs create mode 100644 crates/mq-markdown/tests/mdx_official/corpus.json create mode 100644 crates/mq-markdown/tests/mdx_official/known.json create mode 100644 crates/mq-markdown/tests/mdx_official/package-lock.json create mode 100644 crates/mq-markdown/tests/mdx_official/package.json diff --git a/crates/mq-markdown/tests/mdx_official.rs b/crates/mq-markdown/tests/mdx_official.rs new file mode 100644 index 000000000..753985aab --- /dev/null +++ b/crates/mq-markdown/tests/mdx_official.rs @@ -0,0 +1,75 @@ +//! Writes what mq makes of each input of `mdx_official/corpus.json`, for `mdx_official/compare.mjs` to +//! compare with the official MDX packages. Run it with `just test-mdx-official`. +#![cfg(feature = "json")] + +use mq_markdown::{Markdown, Node}; +use serde_json::{Value, json}; + +fn attribute(attribute: &Value) -> String { + let value = &attribute["value"]; + if attribute["type"] == "expression" { + return format!("...{{{}}}", value.as_str().unwrap()); + } + let name = value["name"].as_str().unwrap(); + match &value["value"] { + Value::Null => name.to_string(), + literal if literal["type"] == "literal" => { + format!("{name}=lit:{}", json!(literal["value"].as_str().unwrap())) + } + expression => format!("{name}=expr:{{{}}}", expression["value"].as_str().unwrap()), + } +} + +fn attributes(attributes: &impl serde::Serialize) -> String { + let attributes = serde_json::to_value(attributes).unwrap(); + attributes + .as_array() + .unwrap() + .iter() + .map(|a| format!(" {}", attribute(a))) + .collect() +} + +fn element(kind: &str, name: Option<&str>, attributes: String, children: &[Node], out: &mut Vec<String>) { + out.push(format!("{kind} <{}{attributes}>(", name.unwrap_or_default())); + children.iter().for_each(|child| describe(child, out)); + out.push(")".into()); +} + +/// The flat description `compare.mjs` makes of the official nodes: MDX nodes with their children, and +/// the text, code and HTML around them. Other nodes are looked through. +fn describe(node: &Node, out: &mut Vec<String>) { + match node { + Node::MdxJsxFlowElement(e) => element("flow", e.name.as_deref(), attributes(&e.attributes), &e.children, out), + Node::MdxJsxTextElement(e) => element("text", e.name.as_deref(), attributes(&e.attributes), &e.children, out), + Node::MdxFlowExpression(e) => out.push(format!("flowexpr {{{}}}", e.value)), + Node::MdxTextExpression(e) => out.push(format!("textexpr {{{}}}", e.value)), + Node::MdxJsEsm(e) => out.push(format!("esm {}", json!(e.value))), + Node::Text(t) => out.push(format!("T{}", json!(t.value))), + Node::CodeInline(t) => out.push(format!("T{}", json!(t.value))), + Node::Code(t) => out.push(format!("T{}", json!(t.value))), + Node::Html(t) => out.push(format!("html {}", json!(t.value))), + Node::Yaml(_) => out.push("yaml".into()), + Node::HorizontalRule(_) => out.push("hr".into()), + other => other.children().iter().for_each(|child| describe(child, out)), + } +} + +#[test] +#[ignore = "needs the official MDX packages; run via `just test-mdx-official`"] +fn write_nodes_of_the_corpus() { + let corpus = include_str!("mdx_official/corpus.json"); + let corpus: Vec<String> = serde_json::from_str(corpus).unwrap(); + let results: Vec<Value> = corpus + .iter() + .map(|input| match Markdown::from_mdx_str(input) { + Ok(markdown) => { + let mut items = Vec::new(); + markdown.nodes.iter().for_each(|node| describe(node, &mut items)); + json!({"error": null, "items": items}) + } + Err(error) => json!({"error": error.to_string(), "items": []}), + }) + .collect(); + std::fs::write(std::env::var("MDX_OFFICIAL_OUT").unwrap(), json!(results).to_string()).unwrap(); +} diff --git a/crates/mq-markdown/tests/mdx_official/compare.mjs b/crates/mq-markdown/tests/mdx_official/compare.mjs new file mode 100644 index 000000000..0519dfaef --- /dev/null +++ b/crates/mq-markdown/tests/mdx_official/compare.mjs @@ -0,0 +1,95 @@ +// Parses each input of corpus.json with the official MDX packages and compares the result with the +// nodes mq wrote to the file in the first argument. Differences must be listed in known.json. +import fs from 'node:fs' +import {fromMarkdown} from 'mdast-util-from-markdown' +import {mdxFromMarkdown} from 'mdast-util-mdx' +import {mdxjs} from 'micromark-extension-mdxjs' + +const dir = new URL('.', import.meta.url) +const read = (file) => JSON.parse(fs.readFileSync(file, 'utf8')) +const corpus = read(new URL('corpus.json', dir)) +const known = read(new URL('known.json', dir)) +const mq = read(process.argv[2]) + +const attribute = (a) => + a.type === 'mdxJsxExpressionAttribute' + ? `...{${a.value}}` + : a.value == null + ? a.name + : typeof a.value === 'string' + ? `${a.name}=lit:${JSON.stringify(a.value)}` + : `${a.name}=expr:{${a.value.value}}` + +const children = (node) => (node.children ?? []).flatMap(normalize) + +// The same flat description as `tests/mdx_official.rs` writes: MDX nodes with their children, and the +// text, code and HTML around them. Other nodes are looked through. +function normalize(node) { + switch (node.type) { + case 'mdxJsxFlowElement': + case 'mdxJsxTextElement': { + const kind = node.type === 'mdxJsxFlowElement' ? 'flow' : 'text' + const attributes = node.attributes.map((a) => ' ' + attribute(a)).join('') + return [`${kind} <${node.name ?? ''}${attributes}>(`, ...children(node), ')'] + } + case 'mdxFlowExpression': return [`flowexpr {${node.value}}`] + case 'mdxTextExpression': return [`textexpr {${node.value}}`] + case 'mdxjsEsm': return [`esm ${JSON.stringify(node.value)}`] + case 'text': case 'inlineCode': case 'code': return [`T${JSON.stringify(node.value)}`] + case 'html': return [`html ${JSON.stringify(node.value)}`] + case 'yaml': return ['yaml'] + case 'thematicBreak': return ['hr'] + default: return children(node) + } +} + +// Adjacent text is joined and whitespace collapsed, as the node structure of paragraphs differs. +function merge(items) { + const out = [] + for (const item of items) { + if (!item.startsWith('T')) { + out.push(item) + continue + } + const text = JSON.parse(item.slice(1)).replace(/\s+/g, ' ').trim() + if (!text) continue + if (out.length && out.at(-1).startsWith('T')) { + out[out.length - 1] = 'T' + JSON.stringify(JSON.parse(out.at(-1).slice(1)) + text) + } else { + out.push('T' + JSON.stringify(text)) + } + } + return out +} + +const official = (input) => { + try { + const tree = fromMarkdown(input, {extensions: [mdxjs()], mdastExtensions: [mdxFromMarkdown()]}) + return {error: null, items: merge(children(tree))} + } catch (error) { + return {error: String(error.reason ?? error.message).split('\n')[0], items: []} + } +} + +const unexpected = [] +const fixed = [] +corpus.forEach((input, index) => { + const expected = official(input) + const actual = {error: mq[index].error, items: merge(mq[index].items)} + const same = + (expected.error === null) === (actual.error === null) && + (expected.error !== null || JSON.stringify(expected.items) === JSON.stringify(actual.items)) + if (!same && !(input in known)) { + unexpected.push(`${JSON.stringify(input)}\n official: ${expected.error ?? JSON.stringify(expected.items)}\n mq: ${actual.error ?? JSON.stringify(actual.items)}`) + } else if (same && input in known) { + fixed.push(JSON.stringify(input)) + } +}) + +for (const input of Object.keys(known)) { + if (!corpus.includes(input)) unexpected.push(`known.json has an input that is not in corpus.json: ${JSON.stringify(input)}`) +} +if (unexpected.length) console.error(`differs from the official MDX:\n${unexpected.join('\n')}`) +if (fixed.length) console.error(`now the same, remove from known.json:\n${fixed.join('\n')}`) +console.log(`${corpus.length} inputs, ${Object.keys(known).length} known differences`) +process.exit(unexpected.length || fixed.length ? 1 : 0) diff --git a/crates/mq-markdown/tests/mdx_official/corpus.json b/crates/mq-markdown/tests/mdx_official/corpus.json new file mode 100644 index 000000000..1e138b595 --- /dev/null +++ b/crates/mq-markdown/tests/mdx_official/corpus.json @@ -0,0 +1,198 @@ +[ +"<a />", +"<a b=\"c&amp;d\" e='f' g={h} {...i} j />", +"<a.b.c />", +"<a:b c:d=\"e\" />", +"<></>", +"<a>x</a>", +"<a>\n\nx\n\n</a>", +"<a>\n x\n</a>", +"a <b>c</b> d", +"a <b/> d", +"{a}", +"{a {b} c}", +"{a\nb}", +"{a\n b\n c}", +"x {a} y", +"{a} <b/>", +"<b/> {a}", +"<b/> x", +"{a}x", +"<a>{b}</a>", +"<a>b", +"<a></b>", +"a < b", +"a <3", +"<a b=>", +"<a>\n<b>\n</b>\n</a>", +"> <a>\n> x\n> </a>", +"- <a>\n x\n </a>", +"<a b='c\"d' />", +"<a/><b/>", +"<a>x</a><b>y</b>", +"a\n<b/>", +"a\n{b}", +"<b e= \"f\"/>", +"a <b e = \"f\"/>.", +"<b c\n= \"x\">c</b>", +"a <b xml :\tlang\n= \"de-CH\" foo:bar>c</b>.", +"a <b a b : c d : e = \"f\" g/>.", +"<b e= {f}/>", +"<a>\n\n# h\n\n- l\n\n</a>", +"<Zoom>\n\n![a](b)\n\n</Zoom>", +"<A>\n <B>\n x\n </B>\n</A>", +"import a from \"b\"\nexport const x = 1\n\nx\n", +"export default d\n", +" import a from \"b\"\n", +"import{a} from \"b\"\n", +"importx a\n", +"- import a from \"b\"\n", +"text\nimport a from \"b\"\n", +"a {", +"{", +"<a", +"<a b", +"<a b=", +"<a b=\"", +"<a b={", +"</a>", +"<a></a></a>", +"<a>", +"<a><b></a></b>", +"a <b c=\"d\" /> e", +"a <b>c</b>", +"a <b>c", +"<a b=\"c\nd\" />", +"<a b='&lt;' />", +"<a b=\"&#x41;\" />", +"<a {...b} {...c} />", +"<a {b} />", +"<a b={c} d={e} />", +"<a b={{c: 1}} />", +"{/* comment */}", +"{}", +"a {} b", +"a {b} {c} d", +"<a>{b}{c}</a>", +"# <a/>", +"# a {b} c", +"*<a>b</a>*", +"**a {b}**", +"[<a/>](b)", +"`<a/>`", +"`{a}`", +"<a/>\n<b/>", +"<a/> <b/>", +"<a/>x\n", +"x<a/>", +"<a>\n\nb\n\n</a>\n\n<c/>", +"<a>\nb\n</a>", +"<a>b\n</a>", +"<a>\nb</a>", +"<a>\n <b/>\n</a>", +"- a\n <b/>\n", +"> a\n> <b/>\n", +"<a>\n- b\n</a>", +"<a>\n\n- b\n\n</a>", +"<a b=\"c\" d=\"e\">\n\nf\n\n</a>", +"<a-b />", +"<a.b-c />", +"<a:b.c />", +"<a.b:c />", +"<_a />", +"<$a />", +"<1 />", +"< a />", +"<a / >", +"<a/ >", +"<a b-c=\"d\" />", +"<a b:c-d=\"e\" />", +"<a b:c:d />", +"<a ={b} />", +"<a b=c />", +"<a b='c' d />", +"<a\nb\n/>", +"<a\n b=\"c\"\n/>", +"{a}\n{b}", +"{a}\n\n{b}", +"{a} {b}", +"{a}{b}", +"x\n{a}\n", +"{\n}", +"{a}\n<b/>", +"<b/>\n{a}", +"a <b> c </b> d", +"<b>\n</b>", +"<b></b>", +"<b> </b>", +"</>", +"<>a</>", +"<>\n\na\n\n</>", +"<></a>", +"<a>{`}`}</a>", +"{'}'}", +"{\"}\"}", +"<a b=\"{c}\" />", +"<a b={\"c\"} />", +"<a b={`c`} />", +"a\\<b/>", +"a\\{b}", +"&lt;a/>", +"a &#123;b}", +"<!-- a -->", +"<a>\n<!-- b -->\n</a>", +"<https://a.b>", +"<a@b.c>", +"https://a.b", +"www.a.b", +"---\ntitle: x\n---\n\na", +"a\n---\n", +"***\n", +"```\n<a/>\n```\n", +" <a/>\n", +"<a/>\n x\n", +"<a>\n\n x\n\n</a>", +"1. <a/>\n", +"* <a>\n b\n </a>\n", +"x <a", +"x <a b", +"x <a b=", +"<a\n\nb", +"<a\nb", +"a\n<a", +"<a b=\"c", +"x <a b=\"c", +"x <", +"<", +"x <a\n\nb", +"x <a b\n\nc", +"- <a", +"> <a", +"<a>\n<b", +"<a>\n\n<b", +"<a>x</a", +"<a>x</", +"<a>x<", +"a {b", +"a {b\n\nc}", +"{a\n\nb}", +"x {a}\n\ny {", +"{a /* } */}", +"{`${'}'}`}", +"<a b={'}'} />", +"{a // }\n}", +"{\"a}\" + '{'}", +"<a b={\"}\"} c='}' />", +"{a /* { */}", +"{`a${b}c`}", +"{`a${`}`}c`}", +"{'\\'}'}", +"{a // {\n}", +"x {'}'} y", +"x {`}`} y", +"{/* } */}\n", +"<a>{'}'}</a>", +"{'a\nb'}", +"{a / b}", +"{a /* x */ + 1}" +] \ No newline at end of file diff --git a/crates/mq-markdown/tests/mdx_official/known.json b/crates/mq-markdown/tests/mdx_official/known.json new file mode 100644 index 000000000..394651453 --- /dev/null +++ b/crates/mq-markdown/tests/mdx_official/known.json @@ -0,0 +1,23 @@ +{ + "{a {b} c}": "Expressions are not parsed as JavaScript, so invalid JavaScript is accepted.", + "{a\nb}": "Expressions are not parsed as JavaScript, so invalid JavaScript is accepted.", + "{a\n b\n c}": "Expressions are not parsed as JavaScript, so invalid JavaScript is accepted.", + "<a {b} />": "Expressions are not parsed as JavaScript, so invalid JavaScript is accepted.", + "{a\n\nb}": "Expressions are not parsed as JavaScript, so invalid JavaScript is accepted.", + "{'a\nb'}": "Expressions are not parsed as JavaScript, so invalid JavaScript is accepted.", + "<a": "A tag that is not closed before the end of its text is text here, and an error in the official packages.", + "<a b": "A tag that is not closed before the end of its text is text here, and an error in the official packages.", + "<a b=": "A tag that is not closed before the end of its text is text here, and an error in the official packages.", + "x <a": "A tag that is not closed before the end of its text is text here, and an error in the official packages.", + "x <a b": "A tag that is not closed before the end of its text is text here, and an error in the official packages.", + "x <a b=": "A tag that is not closed before the end of its text is text here, and an error in the official packages.", + "<a\n\nb": "A tag that is not closed before the end of its text is text here, and an error in the official packages.", + "<a\nb": "A tag that is not closed before the end of its text is text here, and an error in the official packages.", + "a\n<a": "A tag that is not closed before the end of its text is text here, and an error in the official packages.", + "x <": "A tag that is not closed before the end of its text is text here, and an error in the official packages.", + "<": "A tag that is not closed before the end of its text is text here, and an error in the official packages.", + "x <a\n\nb": "A tag that is not closed before the end of its text is text here, and an error in the official packages.", + "x <a b\n\nc": "A tag that is not closed before the end of its text is text here, and an error in the official packages.", + "- <a": "A tag that is not closed before the end of its text is text here, and an error in the official packages.", + "> <a": "A tag that is not closed before the end of its text is text here, and an error in the official packages." +} \ No newline at end of file diff --git a/crates/mq-markdown/tests/mdx_official/package-lock.json b/crates/mq-markdown/tests/mdx_official/package-lock.json new file mode 100644 index 000000000..b558da4ea --- /dev/null +++ b/crates/mq-markdown/tests/mdx_official/package-lock.json @@ -0,0 +1,1175 @@ +{ + "name": "mdx_official", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "dependencies": { + "mdast-util-from-markdown": "2.1.0", + "mdast-util-mdx": "3.0.0", + "micromark-extension-mdxjs": "3.0.0" + } + }, + "node_modules/@types/debug": { + "version": "4.1.13", + "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.13.tgz", + "integrity": "sha512-KSVgmQmzMwPlmtljOomayoR89W4FynCAi3E8PPs7vmDVPe84hT+vGPKkJfThkmXs0x0jAaa9U8uW8bbfyS2fWw==", + "license": "MIT", + "dependencies": { + "@types/ms": "*" + } + }, + "node_modules/@types/estree": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz", + "integrity": "sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==", + "license": "MIT" + }, + "node_modules/@types/estree-jsx": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/@types/estree-jsx/-/estree-jsx-1.0.5.tgz", + "integrity": "sha512-52CcUVNFyfb1A2ALocQw/Dd1BQFNmSdkuC3BkZ6iqhdMfQz7JWOFRuJFloOzjk+6WijU56m9oKXFAXc7o3Towg==", + "license": "MIT", + "dependencies": { + "@types/estree": "*" + } + }, + "node_modules/@types/hast": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/@types/hast/-/hast-3.0.5.tgz", + "integrity": "sha512-rp/ezSWaD1m44dPKICGhiskI13nVr7qTloFwDa/IYkhhf5nzwP+zIQcIJh3WIFSBOy/H1PzB40jPjMDksN4F+g==", + "license": "MIT", + "dependencies": { + "@types/unist": "*" + } + }, + "node_modules/@types/mdast": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/@types/mdast/-/mdast-4.0.4.tgz", + "integrity": "sha512-kGaNbPh1k7AFzgpud/gMdvIm5xuECykRR+JnWKQno9TAXVa6WIVCGTPvYGekIDL4uwCZQSYbUxNBSb1aUo79oA==", + "license": "MIT", + "dependencies": { + "@types/unist": "*" + } + }, + "node_modules/@types/ms": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@types/ms/-/ms-2.1.0.tgz", + "integrity": "sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA==", + "license": "MIT" + }, + "node_modules/@types/unist": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.3.tgz", + "integrity": "sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q==", + "license": "MIT" + }, + "node_modules/acorn": { + "version": "8.19.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.19.0.tgz", + "integrity": "sha512-oJlA3XiRm7Cyk6qFD2Jn8ak9B7jSy0qy00ADO3+8dpT0LSjFihQYv4C02LFCSYJV3Q37xYwwRy5m+IpIiUzqWw==", + "license": "MIT", + "peer": true, + "bin": { + "acorn": "bin/acorn" + }, + "engines": { + "node": ">=0.4.0" + } + }, + "node_modules/acorn-jsx": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", + "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", + "license": "MIT", + "peerDependencies": { + "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" + } + }, + "node_modules/ccount": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/ccount/-/ccount-2.0.1.tgz", + "integrity": "sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/character-entities": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/character-entities/-/character-entities-2.0.2.tgz", + "integrity": "sha512-shx7oQ0Awen/BRIdkjkvz54PnEEI/EjwXDSIZp86/KKdbafHh1Df/RYGBhn4hbe2+uKC9FnT5UCEdyPz3ai9hQ==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/character-entities-html4": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/character-entities-html4/-/character-entities-html4-2.1.0.tgz", + "integrity": "sha512-1v7fgQRj6hnSwFpq1Eu0ynr/CDEw0rXo2B61qXrLNdHZmPKgb7fqS1a2JwF0rISo9q77jDI8VMEHoApn8qDoZA==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/character-entities-legacy": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/character-entities-legacy/-/character-entities-legacy-3.0.0.tgz", + "integrity": "sha512-RpPp0asT/6ufRm//AJVwpViZbGM/MkjQFxJccQRHmISF/22NBtsHqAWmL+/pmkPWoIUJdWyeVleTl1wydHATVQ==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/character-reference-invalid": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/character-reference-invalid/-/character-reference-invalid-2.0.1.tgz", + "integrity": "sha512-iBZ4F4wRbyORVsu0jPV7gXkOsGYjGHPmAyv+HiHG8gi5PtC9KI2j1+v8/tlibRvjoWX027ypmG/n0HtO5t7unw==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/decode-named-character-reference": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/decode-named-character-reference/-/decode-named-character-reference-1.3.0.tgz", + "integrity": "sha512-GtpQYB283KrPp6nRw50q3U9/VfOutZOe103qlN7BPP6Ad27xYnOIWv4lPzo8HCAL+mMZofJ9KEy30fq6MfaK6Q==", + "license": "MIT", + "dependencies": { + "character-entities": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/dequal": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", + "integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/devlop": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/devlop/-/devlop-1.1.0.tgz", + "integrity": "sha512-RWmIqhcFf1lRYBvNmr7qTNuyCt/7/ns2jbpp1+PalgE/rDQcBT0fioSMUpJ93irlUhC5hrg4cYqe6U+0ImW0rA==", + "license": "MIT", + "dependencies": { + "dequal": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/estree-util-is-identifier-name": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/estree-util-is-identifier-name/-/estree-util-is-identifier-name-3.0.0.tgz", + "integrity": "sha512-hFtqIDZTIUZ9BXLb8y4pYGyk6+wekIivNVTcmvk8NoOh+VeRn5y6cEHzbURrWbfp1fIqdVipilzj+lfaadNZmg==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/estree-util-visit": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/estree-util-visit/-/estree-util-visit-2.0.0.tgz", + "integrity": "sha512-m5KgiH85xAhhW8Wta0vShLcUvOsh3LLPI2YVwcbio1l7E09NTLL1EyMZFM1OyWowoH0skScNbhOPl4kcBgzTww==", + "license": "MIT", + "dependencies": { + "@types/estree-jsx": "^1.0.0", + "@types/unist": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/is-alphabetical": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-alphabetical/-/is-alphabetical-2.0.1.tgz", + "integrity": "sha512-FWyyY60MeTNyeSRpkM2Iry0G9hpr7/9kD40mD/cGQEuilcZYS4okz8SN2Q6rLCJ8gbCt6fN+rC+6tMGS99LaxQ==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/is-alphanumerical": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-alphanumerical/-/is-alphanumerical-2.0.1.tgz", + "integrity": "sha512-hmbYhX/9MUMF5uh7tOXyK/n0ZvWpad5caBA17GsC6vyuCqaWliRG5K1qS9inmUhEMaOBIW7/whAnSwveW/LtZw==", + "license": "MIT", + "dependencies": { + "is-alphabetical": "^2.0.0", + "is-decimal": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/is-decimal": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-decimal/-/is-decimal-2.0.1.tgz", + "integrity": "sha512-AAB9hiomQs5DXWcRB1rqsxGUstbRroFOPPVAomNk/3XHR5JyEZChOyTWe2oayKnsSsr/kcGqF+z6yuH6HHpN0A==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/is-hexadecimal": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-hexadecimal/-/is-hexadecimal-2.0.1.tgz", + "integrity": "sha512-DgZQp241c8oO6cA1SbTEWiXeoxV42vlcJxgH+B3hi1AiqqKruZR3ZGF8In3fj4+/y/7rHvlOZLZtgJ/4ttYGZg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/longest-streak": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/longest-streak/-/longest-streak-3.1.0.tgz", + "integrity": "sha512-9Ri+o0JYgehTaVBBDoMqIl8GXtbWg711O3srftcHhZ0dqnETqLaoIK0x17fUw9rFSlK/0NlsKe0Ahhyl5pXE2g==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/mdast-util-from-markdown": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mdast-util-from-markdown/-/mdast-util-from-markdown-2.1.0.tgz", + "integrity": "sha512-KRLgmPaP2H1ul2bpoJkOS+9pAcDAAj9Nu1mqLnh8NLeJG5tlvj42TH80PIGBpROWlRXLk0lDrWNuZCFy/tsB4A==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "@types/unist": "^3.0.0", + "decode-named-character-reference": "^1.0.0", + "devlop": "^1.0.0", + "mdast-util-to-string": "^4.0.0", + "micromark": "^4.0.0", + "micromark-util-decode-numeric-character-reference": "^2.0.0", + "micromark-util-decode-string": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0", + "unist-util-stringify-position": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-mdx": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-mdx/-/mdast-util-mdx-3.0.0.tgz", + "integrity": "sha512-JfbYLAW7XnYTTbUsmpu0kdBUVe+yKVJZBItEjwyYJiDJuZ9w4eeaqks4HQO+R7objWgS2ymV60GYpI14Ug554w==", + "license": "MIT", + "dependencies": { + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-mdx-expression": "^2.0.0", + "mdast-util-mdx-jsx": "^3.0.0", + "mdast-util-mdxjs-esm": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-mdx-expression": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/mdast-util-mdx-expression/-/mdast-util-mdx-expression-2.0.1.tgz", + "integrity": "sha512-J6f+9hUp+ldTZqKRSg7Vw5V6MqjATc+3E4gf3CFNcuZNWD8XdyI6zQ8GqH7f8169MM6P7hMBRDVGnn7oHB9kXQ==", + "license": "MIT", + "dependencies": { + "@types/estree-jsx": "^1.0.0", + "@types/hast": "^3.0.0", + "@types/mdast": "^4.0.0", + "devlop": "^1.0.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-mdx-jsx": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/mdast-util-mdx-jsx/-/mdast-util-mdx-jsx-3.2.0.tgz", + "integrity": "sha512-lj/z8v0r6ZtsN/cGNNtemmmfoLAFZnjMbNyLzBafjzikOM+glrjNHPlf6lQDOTccj9n5b0PPihEBbhneMyGs1Q==", + "license": "MIT", + "dependencies": { + "@types/estree-jsx": "^1.0.0", + "@types/hast": "^3.0.0", + "@types/mdast": "^4.0.0", + "@types/unist": "^3.0.0", + "ccount": "^2.0.0", + "devlop": "^1.1.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0", + "parse-entities": "^4.0.0", + "stringify-entities": "^4.0.0", + "unist-util-stringify-position": "^4.0.0", + "vfile-message": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-mdxjs-esm": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/mdast-util-mdxjs-esm/-/mdast-util-mdxjs-esm-2.0.1.tgz", + "integrity": "sha512-EcmOpxsZ96CvlP03NghtH1EsLtr0n9Tm4lPUJUBccV9RwUOneqSycg19n5HGzCf+10LozMRSObtVr3ee1WoHtg==", + "license": "MIT", + "dependencies": { + "@types/estree-jsx": "^1.0.0", + "@types/hast": "^3.0.0", + "@types/mdast": "^4.0.0", + "devlop": "^1.0.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-phrasing": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/mdast-util-phrasing/-/mdast-util-phrasing-4.1.0.tgz", + "integrity": "sha512-TqICwyvJJpBwvGAMZjj4J2n0X8QWp21b9l0o7eXyVJ25YNWYbJDVIyD1bZXE6WtV6RmKJVYmQAKWa0zWOABz2w==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "unist-util-is": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-to-markdown": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/mdast-util-to-markdown/-/mdast-util-to-markdown-2.2.0.tgz", + "integrity": "sha512-2Jn/ADrHFFyyniBMi0bncTbJqU78K3/TEYvzvwrWWzD9MYPUa/Gm4r3nOeKFQg/PDM5BsbDYdc8ackW4NTY6NA==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "@types/unist": "^3.0.0", + "longest-streak": "^3.0.0", + "mdast-util-phrasing": "^4.0.0", + "mdast-util-to-string": "^4.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-classify-character": "^2.0.0", + "micromark-util-decode-string": "^2.0.0", + "micromark-util-html-tag-name": "^2.0.0", + "unist-util-visit": "^5.0.0", + "zwitch": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-to-string": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-to-string/-/mdast-util-to-string-4.0.0.tgz", + "integrity": "sha512-0H44vDimn51F0YwvxSJSm0eCDOJTRlmN0R1yBh4HLj9wiV1Dn0QoXGbvFAWj2hSItVTlCmBF1hqKlIyUBVFLPg==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/micromark/-/micromark-4.0.3.tgz", + "integrity": "sha512-oGYfQzHSG5dOMovQcJ3fyTmZlWAWpi0XA0sJwJs+i6OT88o1+Jtw/8z0CmdowSLxGhhFK89rQ/oXph/wN02PNw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "@types/debug": "^4.0.0", + "debug": "^4.0.0", + "decode-named-character-reference": "^1.0.0", + "devlop": "^1.0.0", + "micromark-core-commonmark": "^2.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-combine-extensions": "^2.0.0", + "micromark-util-decode-numeric-character-reference": "^2.0.0", + "micromark-util-edit-map": "^1.0.0", + "micromark-util-encode": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-resolve-all": "^2.0.0", + "micromark-util-sanitize-uri": "^2.0.0", + "micromark-util-subtokenize": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-core-commonmark": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/micromark-core-commonmark/-/micromark-core-commonmark-2.0.4.tgz", + "integrity": "sha512-wxEeE8v8XVvOrxn1TZj74qYhAtTQsSqufHVS3uNVyT1MupXxhyaHk8/9xDNPh9BmL77QNDgWTkZ3RsYHB2ry7w==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "decode-named-character-reference": "^1.0.0", + "devlop": "^1.0.0", + "micromark-factory-destination": "^2.0.0", + "micromark-factory-label": "^2.0.0", + "micromark-factory-space": "^2.1.0", + "micromark-factory-title": "^2.0.0", + "micromark-factory-whitespace": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-classify-character": "^2.0.0", + "micromark-util-edit-map": "^1.0.0", + "micromark-util-html-tag-name": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-resolve-all": "^2.0.0", + "micromark-util-subtokenize": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.3" + } + }, + "node_modules/micromark-extension-mdx-expression": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/micromark-extension-mdx-expression/-/micromark-extension-mdx-expression-3.0.1.tgz", + "integrity": "sha512-dD/ADLJ1AeMvSAKBwO22zG22N4ybhe7kFIZ3LsDI0GlsNr2A3KYxb0LdC1u5rj4Nw+CHKY0RVdnHX8vj8ejm4Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0", + "devlop": "^1.0.0", + "micromark-factory-mdx-expression": "^2.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-events-to-acorn": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-extension-mdx-jsx": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/micromark-extension-mdx-jsx/-/micromark-extension-mdx-jsx-3.0.2.tgz", + "integrity": "sha512-e5+q1DjMh62LZAJOnDraSSbDMvGJ8x3cbjygy2qFEi7HCeUT4BDKCvMozPozcD6WmOt6sVvYDNBKhFSz3kjOVQ==", + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0", + "devlop": "^1.0.0", + "estree-util-is-identifier-name": "^3.0.0", + "micromark-factory-mdx-expression": "^2.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-events-to-acorn": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0", + "vfile-message": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-mdx-md": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-extension-mdx-md/-/micromark-extension-mdx-md-2.0.0.tgz", + "integrity": "sha512-EpAiszsB3blw4Rpba7xTOUptcFeBFi+6PY8VnJ2hhimH+vCQDirWgsMpz7w1XcZE7LVrSAUGb9VJpG9ghlYvYQ==", + "license": "MIT", + "dependencies": { + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-mdxjs": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/micromark-extension-mdxjs/-/micromark-extension-mdxjs-3.0.0.tgz", + "integrity": "sha512-A873fJfhnJ2siZyUrJ31l34Uqwy4xIFmvPY1oj+Ean5PHcPBYzEsvqvWGaWcfEIr11O5Dlw3p2y0tZWpKHDejQ==", + "license": "MIT", + "dependencies": { + "acorn": "^8.0.0", + "acorn-jsx": "^5.0.0", + "micromark-extension-mdx-expression": "^3.0.0", + "micromark-extension-mdx-jsx": "^3.0.0", + "micromark-extension-mdx-md": "^2.0.0", + "micromark-extension-mdxjs-esm": "^3.0.0", + "micromark-util-combine-extensions": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-mdxjs-esm": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/micromark-extension-mdxjs-esm/-/micromark-extension-mdxjs-esm-3.0.0.tgz", + "integrity": "sha512-DJFl4ZqkErRpq/dAPyeWp15tGrcrrJho1hKK5uBS70BCtfrIFg81sqcTVu3Ta+KD1Tk5vAtBNElWxtAa+m8K9A==", + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0", + "devlop": "^1.0.0", + "micromark-core-commonmark": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-events-to-acorn": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0", + "unist-util-position-from-estree": "^2.0.0", + "vfile-message": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-factory-destination": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-destination/-/micromark-factory-destination-2.0.1.tgz", + "integrity": "sha512-Xe6rDdJlkmbFRExpTOmRj9N3MaWmbAgdpSrBQvCFqhezUn4AHqJHbaEnfbVYYiexVSs//tqOdY/DxhjdCiJnIA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-label": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-label/-/micromark-factory-label-2.0.1.tgz", + "integrity": "sha512-VFMekyQExqIW7xIChcXn4ok29YE3rnuyveW3wZQWWqF4Nv9Wk5rgJ99KzPvHjkmPXF93FXIbBp6YdW3t71/7Vg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-mdx-expression": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/micromark-factory-mdx-expression/-/micromark-factory-mdx-expression-2.0.3.tgz", + "integrity": "sha512-kQnEtA3vzucU2BkrIa8/VaSAsP+EJ3CKOvhMuJgOEGg9KDC6OAY6nSnNDVRiVNRqj7Y4SlSzcStaH/5jge8JdQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0", + "devlop": "^1.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-events-to-acorn": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0", + "unist-util-position-from-estree": "^2.0.0", + "vfile-message": "^4.0.0" + } + }, + "node_modules/micromark-factory-space": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-factory-space/-/micromark-factory-space-2.1.0.tgz", + "integrity": "sha512-fS8hnLIjnjvdQIj39Geug8wWsR0HrYZ43KShKxNfwT7t2LOHo/LbWZEzEaSOjwtjdCsjoE6syHhonEzW0zv0+Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-title": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-title/-/micromark-factory-title-2.0.1.tgz", + "integrity": "sha512-5bZ+3CjhAd9eChYTHsjy6TGxpOFSKgKKJPJxr293jTbfry2KDoWkhBb6TcPVB4NmzaPhMs1Frm9AZH7OD4Cjzw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-whitespace": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-whitespace/-/micromark-factory-whitespace-2.0.1.tgz", + "integrity": "sha512-Ob0nuZ3PKt/n0hORHyvoD9uZhr+Za8sFoP+OnMcnWK5lngSzALgQYKMr9RJVOWLqQYuyn6ulqGWSXdwf6F80lQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-character": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/micromark-util-character/-/micromark-util-character-2.1.1.tgz", + "integrity": "sha512-wv8tdUTJ3thSFFFJKtpYKOYiGP2+v96Hvk4Tu8KpCAsTMs6yi+nVmGh1syvSCsaxz45J6Jbw+9DD6g97+NV67Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-chunked": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-chunked/-/micromark-util-chunked-2.0.1.tgz", + "integrity": "sha512-QUNFEOPELfmvv+4xiNg2sRYeS/P84pTW0TCgP5zc9FpXetHY0ab7SxKyAQCNCc1eK0459uoLI1y5oO5Vc1dbhA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-classify-character": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-classify-character/-/micromark-util-classify-character-2.0.1.tgz", + "integrity": "sha512-K0kHzM6afW/MbeWYWLjoHQv1sgg2Q9EccHEDzSkxiP/EaagNzCm7T/WMKZ3rjMbvIpvBiZgwR3dKMygtA4mG1Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-combine-extensions": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-combine-extensions/-/micromark-util-combine-extensions-2.0.1.tgz", + "integrity": "sha512-OnAnH8Ujmy59JcyZw8JSbK9cGpdVY44NKgSM7E9Eh7DiLS2E9RNQf0dONaGDzEG9yjEl5hcqeIsj4hfRkLH/Bg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-chunked": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-decode-numeric-character-reference": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/micromark-util-decode-numeric-character-reference/-/micromark-util-decode-numeric-character-reference-2.0.2.tgz", + "integrity": "sha512-ccUbYk6CwVdkmCQMyr64dXz42EfHGkPQlBj5p7YVGzq8I7CtjXZJrubAYezf7Rp+bjPseiROqe7G6foFd+lEuw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-decode-string": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-decode-string/-/micromark-util-decode-string-2.0.1.tgz", + "integrity": "sha512-nDV/77Fj6eH1ynwscYTOsbK7rR//Uj0bZXBwJZRfaLEJ1iGBR6kIfNmlNqaqJf649EP0F3NWNdeJi03elllNUQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "decode-named-character-reference": "^1.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-decode-numeric-character-reference": "^2.0.0", + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-edit-map": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/micromark-util-edit-map/-/micromark-util-edit-map-1.0.0.tgz", + "integrity": "sha512-Pa2ljlsEL6sVwFaYeyrOLSYbQt73JvGbPOYFq+9AElXiSnfI5Q4465RRZ1sHv7lDjDOnGRDDaqPXqClqWK/q1Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-encode": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-encode/-/micromark-util-encode-2.0.1.tgz", + "integrity": "sha512-c3cVx2y4KqUnwopcO9b/SCdo2O67LwJJ/UyqGfbigahfegL9myoEFoDYZgkT7f36T0bLrM9hZTAaAyH+PCAXjw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-events-to-acorn": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/micromark-util-events-to-acorn/-/micromark-util-events-to-acorn-2.0.3.tgz", + "integrity": "sha512-jmsiEIiZ1n7X1Rr5k8wVExBQCg5jy4UXVADItHmNk1zkwEVhBuIUKRu3fqv+hs4nxLISi2DQGlqIOGiFxgbfHg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0", + "@types/unist": "^3.0.0", + "devlop": "^1.0.0", + "estree-util-visit": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0", + "vfile-message": "^4.0.0" + } + }, + "node_modules/micromark-util-html-tag-name": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-html-tag-name/-/micromark-util-html-tag-name-2.0.1.tgz", + "integrity": "sha512-2cNEiYDhCWKI+Gs9T0Tiysk136SnR13hhO8yW6BGNyhOC4qYFnwF1nKfD3HFAIXA5c45RrIG1ub11GiXeYd1xA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-normalize-identifier": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-normalize-identifier/-/micromark-util-normalize-identifier-2.0.1.tgz", + "integrity": "sha512-sxPqmo70LyARJs0w2UclACPUUEqltCkJ6PhKdMIDuJ3gSf/Q+/GIe3WKl0Ijb/GyH9lOpUkRAO2wp0GVkLvS9Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-resolve-all": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-resolve-all/-/micromark-util-resolve-all-2.0.1.tgz", + "integrity": "sha512-VdQyxFWFT2/FGJgwQnJYbe1jjQoNTS4RjglmSjTUlpUMa95Htx9NHeYW4rGDJzbjvCsl9eLjMQwGeElsqmzcHg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-sanitize-uri": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-sanitize-uri/-/micromark-util-sanitize-uri-2.0.1.tgz", + "integrity": "sha512-9N9IomZ/YuGGZZmQec1MbgxtlgougxTodVwDzzEouPKo3qFWvymFHWcnDi2vzV1ff6kas9ucW+o3yzJK9YB1AQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-encode": "^2.0.0", + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-subtokenize": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-util-subtokenize/-/micromark-util-subtokenize-2.1.0.tgz", + "integrity": "sha512-XQLu552iSctvnEcgXw6+Sx75GflAPNED1qx7eBJ+wydBb2KCbRZe+NwvIEEMM83uml1+2WSXpBAcp9IUCgCYWA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-symbol": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-symbol/-/micromark-util-symbol-2.0.1.tgz", + "integrity": "sha512-vs5t8Apaud9N28kgCrRUdEed4UJ+wWNvicHLPxCa9ENlYuAY31M0ETy5y1vA33YoNPDFTghEbnh6efaE8h4x0Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-types": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/micromark-util-types/-/micromark-util-types-2.0.3.tgz", + "integrity": "sha512-oxB2Ik03hI0gv+VNn9tnh1t1YEe9MDPptViAEgfdf3YQHsn0pzGTgCdlSCJXcwhqm8phaEuM7zeEu3QQzVBrPg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/parse-entities": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/parse-entities/-/parse-entities-4.0.2.tgz", + "integrity": "sha512-GG2AQYWoLgL877gQIKeRPGO1xF9+eG1ujIb5soS5gPvLQ1y2o8FL90w2QWNdf9I361Mpp7726c+lj3U0qK1uGw==", + "license": "MIT", + "dependencies": { + "@types/unist": "^2.0.0", + "character-entities-legacy": "^3.0.0", + "character-reference-invalid": "^2.0.0", + "decode-named-character-reference": "^1.0.0", + "is-alphanumerical": "^2.0.0", + "is-decimal": "^2.0.0", + "is-hexadecimal": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/parse-entities/node_modules/@types/unist": { + "version": "2.0.11", + "resolved": "https://registry.npmjs.org/@types/unist/-/unist-2.0.11.tgz", + "integrity": "sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA==", + "license": "MIT" + }, + "node_modules/stringify-entities": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/stringify-entities/-/stringify-entities-4.0.4.tgz", + "integrity": "sha512-IwfBptatlO+QCJUo19AqvrPNqlVMpW9YEL2LIVY+Rpv2qsjCGxaDLNRgeGsQWJhfItebuJhsGSLjaBbNSQ+ieg==", + "license": "MIT", + "dependencies": { + "character-entities-html4": "^2.0.0", + "character-entities-legacy": "^3.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/unist-util-is": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/unist-util-is/-/unist-util-is-6.0.1.tgz", + "integrity": "sha512-LsiILbtBETkDz8I9p1dQ0uyRUWuaQzd/cuEeS1hoRSyW5E5XGmTzlwY1OrNzzakGowI9Dr/I8HVaw4hTtnxy8g==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-position-from-estree": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/unist-util-position-from-estree/-/unist-util-position-from-estree-2.0.0.tgz", + "integrity": "sha512-KaFVRjoqLyF6YXCbVLNad/eS4+OfPQQn2yOd7zF/h5T/CSL2v8NpN6a5TPvtbXthAGw5nG+PuTtq+DdIZr+cRQ==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-stringify-position": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/unist-util-stringify-position/-/unist-util-stringify-position-4.0.0.tgz", + "integrity": "sha512-0ASV06AAoKCDkS2+xw5RXJywruurpbC4JZSm7nr7MOt1ojAzvyyaO+UxZf18j8FCF6kmzCZKcAgN/yu2gm2XgQ==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-visit": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/unist-util-visit/-/unist-util-visit-5.1.0.tgz", + "integrity": "sha512-m+vIdyeCOpdr/QeQCu2EzxX/ohgS8KbnPDgFni4dQsfSCtpz8UqDyY5GjRru8PDKuYn7Fq19j1CQ+nJSsGKOzg==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "unist-util-is": "^6.0.0", + "unist-util-visit-parents": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-visit-parents": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/unist-util-visit-parents/-/unist-util-visit-parents-6.0.2.tgz", + "integrity": "sha512-goh1s1TBrqSqukSc8wrjwWhL0hiJxgA8m4kFxGlQ+8FYQ3C/m11FcTs4YYem7V664AhHVvgoQLk890Ssdsr2IQ==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "unist-util-is": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/vfile-message": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/vfile-message/-/vfile-message-4.0.3.tgz", + "integrity": "sha512-QTHzsGd1EhbZs4AsQ20JX1rC3cOlt/IWJruk893DfLRr57lcnOeMaWG4K0JrRta4mIJZKth2Au3mM3u03/JWKw==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "unist-util-stringify-position": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/zwitch": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/zwitch/-/zwitch-2.0.4.tgz", + "integrity": "sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + } + } +} diff --git a/crates/mq-markdown/tests/mdx_official/package.json b/crates/mq-markdown/tests/mdx_official/package.json new file mode 100644 index 000000000..fe3e4f245 --- /dev/null +++ b/crates/mq-markdown/tests/mdx_official/package.json @@ -0,0 +1,10 @@ +{ + "private": true, + "type": "module", + "description": "Reference parser for tests/mdx_official.rs: the mdast of the official MDX packages.", + "dependencies": { + "mdast-util-from-markdown": "2.1.0", + "mdast-util-mdx": "3.0.0", + "micromark-extension-mdxjs": "3.0.0" + } +} diff --git a/justfile b/justfile index 68fb12130..58ff9be9c 100644 --- a/justfile +++ b/justfile @@ -124,6 +124,16 @@ bench-check-phases: test-gfm-spec: cargo test -p mq-markdown --test gfm_roundtrip_fidelity -- --ignored --nocapture +# Render every example of the CommonMark, GFM and GFM extensions specs (fetches them over the network) +test-spec: + cargo test -p mq-markdown --lib -- --ignored spec_examples + +# Compare the MDX nodes with those of the official MDX packages (installs them with npm) +test-mdx-official: + cd crates/mq-markdown/tests/mdx_official && npm install --silent + MDX_OFFICIAL_OUT="$PWD/target/mdx_official.json" cargo test -p mq-markdown --features json --test mdx_official -- --ignored + node crates/mq-markdown/tests/mdx_official/compare.mjs target/mdx_official.json + test-doc: cargo test --doc --workspace From b63986fd34180a593b26c49b4d4de78cb0f389db Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Thu, 8 Oct 2026 13:07:15 +0900 Subject: [PATCH 23/88] =?UTF-8?q?=F0=9F=90=9B=20fix(markdown):=20strip=20a?= =?UTF-8?q?=20line=20ending=20at=20the=20ends=20of=20a=20code=20span?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A code span that starts and ends with a line ending kept both in its value, where one space or line ending is removed from each end. --- crates/mq-markdown/src/parser/inline.rs | 7 +++--- .../{mdx_official.rs => mdast_official.rs} | 0 .../compare.mjs | 0 .../corpus-mdx.json} | 0 .../package-lock.json | 0 .../package.json | 0 .../mq-markdown/tests/mdx_official/known.json | 23 ------------------- .../tests/parser_structure_tests.rs | 10 ++++++++ 8 files changed, 14 insertions(+), 26 deletions(-) rename crates/mq-markdown/tests/{mdx_official.rs => mdast_official.rs} (100%) rename crates/mq-markdown/tests/{mdx_official => mdast_official}/compare.mjs (100%) rename crates/mq-markdown/tests/{mdx_official/corpus.json => mdast_official/corpus-mdx.json} (100%) rename crates/mq-markdown/tests/{mdx_official => mdast_official}/package-lock.json (100%) rename crates/mq-markdown/tests/{mdx_official => mdast_official}/package.json (100%) delete mode 100644 crates/mq-markdown/tests/mdx_official/known.json diff --git a/crates/mq-markdown/src/parser/inline.rs b/crates/mq-markdown/src/parser/inline.rs index c64b5a191..1d5e07f06 100644 --- a/crates/mq-markdown/src/parser/inline.rs +++ b/crates/mq-markdown/src/parser/inline.rs @@ -771,11 +771,12 @@ fn eol_len(bytes: &[u8]) -> usize { } } -/// The value of a code or math span: the content as it is, except that one space is removed from -/// both ends when both have one and the content is not only spaces. +/// The value of a code or math span: the content as it is, except that one space or line ending is +/// removed from both ends when both have one and the content is not only spaces and line endings. fn span_value(content: &str) -> SmolStr { + let is_space = |b: &u8| matches!(b, b' ' | b'\n'); let bytes = content.as_bytes(); - if bytes.len() > 2 && bytes[0] == b' ' && bytes[bytes.len() - 1] == b' ' && bytes.iter().any(|&b| b != b' ') { + if bytes.len() > 2 && is_space(&bytes[0]) && is_space(&bytes[bytes.len() - 1]) && !bytes.iter().all(is_space) { SmolStr::new(&content[1..content.len() - 1]) } else { SmolStr::new(content) diff --git a/crates/mq-markdown/tests/mdx_official.rs b/crates/mq-markdown/tests/mdast_official.rs similarity index 100% rename from crates/mq-markdown/tests/mdx_official.rs rename to crates/mq-markdown/tests/mdast_official.rs diff --git a/crates/mq-markdown/tests/mdx_official/compare.mjs b/crates/mq-markdown/tests/mdast_official/compare.mjs similarity index 100% rename from crates/mq-markdown/tests/mdx_official/compare.mjs rename to crates/mq-markdown/tests/mdast_official/compare.mjs diff --git a/crates/mq-markdown/tests/mdx_official/corpus.json b/crates/mq-markdown/tests/mdast_official/corpus-mdx.json similarity index 100% rename from crates/mq-markdown/tests/mdx_official/corpus.json rename to crates/mq-markdown/tests/mdast_official/corpus-mdx.json diff --git a/crates/mq-markdown/tests/mdx_official/package-lock.json b/crates/mq-markdown/tests/mdast_official/package-lock.json similarity index 100% rename from crates/mq-markdown/tests/mdx_official/package-lock.json rename to crates/mq-markdown/tests/mdast_official/package-lock.json diff --git a/crates/mq-markdown/tests/mdx_official/package.json b/crates/mq-markdown/tests/mdast_official/package.json similarity index 100% rename from crates/mq-markdown/tests/mdx_official/package.json rename to crates/mq-markdown/tests/mdast_official/package.json diff --git a/crates/mq-markdown/tests/mdx_official/known.json b/crates/mq-markdown/tests/mdx_official/known.json deleted file mode 100644 index 394651453..000000000 --- a/crates/mq-markdown/tests/mdx_official/known.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "{a {b} c}": "Expressions are not parsed as JavaScript, so invalid JavaScript is accepted.", - "{a\nb}": "Expressions are not parsed as JavaScript, so invalid JavaScript is accepted.", - "{a\n b\n c}": "Expressions are not parsed as JavaScript, so invalid JavaScript is accepted.", - "<a {b} />": "Expressions are not parsed as JavaScript, so invalid JavaScript is accepted.", - "{a\n\nb}": "Expressions are not parsed as JavaScript, so invalid JavaScript is accepted.", - "{'a\nb'}": "Expressions are not parsed as JavaScript, so invalid JavaScript is accepted.", - "<a": "A tag that is not closed before the end of its text is text here, and an error in the official packages.", - "<a b": "A tag that is not closed before the end of its text is text here, and an error in the official packages.", - "<a b=": "A tag that is not closed before the end of its text is text here, and an error in the official packages.", - "x <a": "A tag that is not closed before the end of its text is text here, and an error in the official packages.", - "x <a b": "A tag that is not closed before the end of its text is text here, and an error in the official packages.", - "x <a b=": "A tag that is not closed before the end of its text is text here, and an error in the official packages.", - "<a\n\nb": "A tag that is not closed before the end of its text is text here, and an error in the official packages.", - "<a\nb": "A tag that is not closed before the end of its text is text here, and an error in the official packages.", - "a\n<a": "A tag that is not closed before the end of its text is text here, and an error in the official packages.", - "x <": "A tag that is not closed before the end of its text is text here, and an error in the official packages.", - "<": "A tag that is not closed before the end of its text is text here, and an error in the official packages.", - "x <a\n\nb": "A tag that is not closed before the end of its text is text here, and an error in the official packages.", - "x <a b\n\nc": "A tag that is not closed before the end of its text is text here, and an error in the official packages.", - "- <a": "A tag that is not closed before the end of its text is text here, and an error in the official packages.", - "> <a": "A tag that is not closed before the end of its text is text here, and an error in the official packages." -} \ No newline at end of file diff --git a/crates/mq-markdown/tests/parser_structure_tests.rs b/crates/mq-markdown/tests/parser_structure_tests.rs index 23729391f..9b3e071c4 100644 --- a/crates/mq-markdown/tests/parser_structure_tests.rs +++ b/crates/mq-markdown/tests/parser_structure_tests.rs @@ -342,6 +342,16 @@ fn indented_code_drops_trailing_lines_of_whitespace() { assert_eq!(nodes[0].value(), "brew install x"); } +#[rstest] +#[case::line_endings_at_the_ends("``\nfoo\n``\n", "foo")] +#[case::line_ending_and_space("`\nfoo `\n", "foo")] +#[case::inner_line_endings_stay("``\nfoo\nbar \nbaz\n``\n", "foo\nbar \nbaz")] +#[case::only_spaces_and_line_endings("` \n`\n", " \n")] +fn code_span_strips_one_space_or_line_ending(#[case] input: &str, #[case] expected: &str) { + let nodes = Markdown::from_markdown_str(input).unwrap().nodes; + assert_eq!(nodes[0].value(), expected, "{nodes:?}"); +} + /// References keep the notation they were written in, as long as it resolves to the same definition. #[rstest] #[case::full_keeps_case_and_spaces("[x][Foo Bar]\n\n[foo bar]: /u\n", "[x][Foo Bar]")] From 4fdac53141a59d1b51fa380078329f30d1c0ea6c Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Thu, 8 Oct 2026 13:07:21 +0900 Subject: [PATCH 24/88] =?UTF-8?q?=E2=9C=85=20test(markdown):=20compare=20M?= =?UTF-8?q?arkdown=20nodes=20with=20the=20official=20mdast?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The corpus gains the examples of the CommonMark, GFM and extensions specs, and the whole node tree is compared, not only the MDX nodes. just test-mdast-official replaces test-mdx-official. --- crates/mq-markdown/tests/mdast_official.rs | 79 +--- .../tests/mdast_official/compare.mjs | 227 +++++++--- .../tests/mdast_official/corpus-markdown.json | 12 + .../tests/mdast_official/corpus.mjs | 41 ++ .../tests/mdast_official/known.json | 27 ++ .../tests/mdast_official/package-lock.json | 406 +++++++++++++++++- .../tests/mdast_official/package.json | 6 + justfile | 12 +- 8 files changed, 684 insertions(+), 126 deletions(-) create mode 100644 crates/mq-markdown/tests/mdast_official/corpus-markdown.json create mode 100644 crates/mq-markdown/tests/mdast_official/corpus.mjs create mode 100644 crates/mq-markdown/tests/mdast_official/known.json diff --git a/crates/mq-markdown/tests/mdast_official.rs b/crates/mq-markdown/tests/mdast_official.rs index 753985aab..b5fe485a3 100644 --- a/crates/mq-markdown/tests/mdast_official.rs +++ b/crates/mq-markdown/tests/mdast_official.rs @@ -1,75 +1,28 @@ -//! Writes what mq makes of each input of `mdx_official/corpus.json`, for `mdx_official/compare.mjs` to -//! compare with the official MDX packages. Run it with `just test-mdx-official`. +//! Writes the nodes mq makes of each input of the corpus as JSON, for `mdast_official/compare.mjs` to +//! compare with the mdast of the official packages. Run it with `just test-mdast-official`. #![cfg(feature = "json")] -use mq_markdown::{Markdown, Node}; +use mq_markdown::Markdown; use serde_json::{Value, json}; -fn attribute(attribute: &Value) -> String { - let value = &attribute["value"]; - if attribute["type"] == "expression" { - return format!("...{{{}}}", value.as_str().unwrap()); - } - let name = value["name"].as_str().unwrap(); - match &value["value"] { - Value::Null => name.to_string(), - literal if literal["type"] == "literal" => { - format!("{name}=lit:{}", json!(literal["value"].as_str().unwrap())) - } - expression => format!("{name}=expr:{{{}}}", expression["value"].as_str().unwrap()), - } -} - -fn attributes(attributes: &impl serde::Serialize) -> String { - let attributes = serde_json::to_value(attributes).unwrap(); - attributes - .as_array() - .unwrap() - .iter() - .map(|a| format!(" {}", attribute(a))) - .collect() -} - -fn element(kind: &str, name: Option<&str>, attributes: String, children: &[Node], out: &mut Vec<String>) { - out.push(format!("{kind} <{}{attributes}>(", name.unwrap_or_default())); - children.iter().for_each(|child| describe(child, out)); - out.push(")".into()); -} - -/// The flat description `compare.mjs` makes of the official nodes: MDX nodes with their children, and -/// the text, code and HTML around them. Other nodes are looked through. -fn describe(node: &Node, out: &mut Vec<String>) { - match node { - Node::MdxJsxFlowElement(e) => element("flow", e.name.as_deref(), attributes(&e.attributes), &e.children, out), - Node::MdxJsxTextElement(e) => element("text", e.name.as_deref(), attributes(&e.attributes), &e.children, out), - Node::MdxFlowExpression(e) => out.push(format!("flowexpr {{{}}}", e.value)), - Node::MdxTextExpression(e) => out.push(format!("textexpr {{{}}}", e.value)), - Node::MdxJsEsm(e) => out.push(format!("esm {}", json!(e.value))), - Node::Text(t) => out.push(format!("T{}", json!(t.value))), - Node::CodeInline(t) => out.push(format!("T{}", json!(t.value))), - Node::Code(t) => out.push(format!("T{}", json!(t.value))), - Node::Html(t) => out.push(format!("html {}", json!(t.value))), - Node::Yaml(_) => out.push("yaml".into()), - Node::HorizontalRule(_) => out.push("hr".into()), - other => other.children().iter().for_each(|child| describe(child, out)), - } -} - #[test] -#[ignore = "needs the official MDX packages; run via `just test-mdx-official`"] +#[ignore = "needs the official packages and the specs; run via `just test-mdast-official`"] fn write_nodes_of_the_corpus() { - let corpus = include_str!("mdx_official/corpus.json"); - let corpus: Vec<String> = serde_json::from_str(corpus).unwrap(); + let corpus = std::fs::read_to_string(std::env::var("MDAST_CORPUS").unwrap()).unwrap(); + let corpus: Vec<Value> = serde_json::from_str(&corpus).unwrap(); let results: Vec<Value> = corpus .iter() - .map(|input| match Markdown::from_mdx_str(input) { - Ok(markdown) => { - let mut items = Vec::new(); - markdown.nodes.iter().for_each(|node| describe(node, &mut items)); - json!({"error": null, "items": items}) + .map(|entry| { + let input = entry["input"].as_str().unwrap(); + let parsed = match entry["mode"].as_str().unwrap() { + "mdx" => Markdown::from_mdx_str(input), + _ => Markdown::from_markdown_str(input), + }; + match parsed.and_then(|markdown| markdown.to_json()) { + Ok(nodes) => json!({"error": null, "nodes": serde_json::from_str::<Value>(&nodes).unwrap()}), + Err(error) => json!({"error": error.to_string(), "nodes": []}), } - Err(error) => json!({"error": error.to_string(), "items": []}), }) .collect(); - std::fs::write(std::env::var("MDX_OFFICIAL_OUT").unwrap(), json!(results).to_string()).unwrap(); + std::fs::write(std::env::var("MDAST_OUT").unwrap(), json!(results).to_string()).unwrap(); } diff --git a/crates/mq-markdown/tests/mdast_official/compare.mjs b/crates/mq-markdown/tests/mdast_official/compare.mjs index 0519dfaef..c79b2e3c0 100644 --- a/crates/mq-markdown/tests/mdast_official/compare.mjs +++ b/crates/mq-markdown/tests/mdast_official/compare.mjs @@ -1,95 +1,208 @@ -// Parses each input of corpus.json with the official MDX packages and compares the result with the -// nodes mq wrote to the file in the first argument. Differences must be listed in known.json. +// Parses each input of the corpus (first argument) with the official packages and compares the nodes with +// those mq wrote (second argument). Both are put in the same flat form first: the kinds and values of the +// nodes in document order, with containers opened by `kind(` and closed by `)`. Paragraphs are not +// marked, since mq has none, and the items of nested lists follow their parent item, as in mq. +// Differences must be listed in known.json by the id of the input. import fs from 'node:fs' import {fromMarkdown} from 'mdast-util-from-markdown' +import {frontmatterFromMarkdown} from 'mdast-util-frontmatter' +import {gfmFromMarkdown} from 'mdast-util-gfm' +import {mathFromMarkdown} from 'mdast-util-math' import {mdxFromMarkdown} from 'mdast-util-mdx' +import {frontmatter} from 'micromark-extension-frontmatter' +import {gfm} from 'micromark-extension-gfm' +import {math} from 'micromark-extension-math' import {mdxjs} from 'micromark-extension-mdxjs' -const dir = new URL('.', import.meta.url) const read = (file) => JSON.parse(fs.readFileSync(file, 'utf8')) -const corpus = read(new URL('corpus.json', dir)) -const known = read(new URL('known.json', dir)) -const mq = read(process.argv[2]) +const corpus = read(process.argv[2]) +const actual = read(process.argv[3]) +const known = read(new URL('known.json', import.meta.url)) -const attribute = (a) => - a.type === 'mdxJsxExpressionAttribute' - ? `...{${a.value}}` - : a.value == null - ? a.name - : typeof a.value === 'string' - ? `${a.name}=lit:${JSON.stringify(a.value)}` - : `${a.name}=expr:{${a.value.value}}` +const json = JSON.stringify +const text = (value) => `T${json(value)}` -const children = (node) => (node.children ?? []).flatMap(normalize) +/** Joins adjacent text tokens. */ +function merge(tokens) { + const out = [] + for (const token of tokens) { + if (token.startsWith('T') && out.length && out.at(-1).startsWith('T')) { + out[out.length - 1] = text(JSON.parse(out.at(-1).slice(1)) + JSON.parse(token.slice(1))) + } else { + out.push(token) + } + } + return out +} + +const container = (head, inner) => [`${head}(`, ...inner, ')'] +const mdxAttribute = (attribute) => + attribute.type === 'mdxJsxExpressionAttribute' + ? `...{${attribute.value}}` + : attribute.value == null + ? attribute.name + : typeof attribute.value === 'string' + ? `${attribute.name}=lit:${json(attribute.value)}` + : `${attribute.name}=expr:{${attribute.value.value}}` +const item = (level, ordered, checked, spread) => + `item ${level} ${ordered === null ? 'u' : `o${ordered}`} ${checked === true ? 'x' : checked === false ? '_' : '-'} ${spread ? 's' : 't'}` +const alignment = (align) => `align ${json(align.map((a) => a ?? 'none'))}` -// The same flat description as `tests/mdx_official.rs` writes: MDX nodes with their children, and the -// text, code and HTML around them. Other nodes are looked through. -function normalize(node) { +function fromMdast(node, level = 0) { + const kids = (inner = true) => { + const tokens = (node.children ?? []).flatMap((child) => fromMdast(child, level)) + return inner ? merge(tokens) : tokens + } switch (node.type) { + case 'root': return kids(false) + case 'paragraph': return kids() + case 'heading': return container(`heading ${node.depth}`, kids()) + case 'thematicBreak': return ['hr'] + case 'code': return [`code ${node.lang ?? ''} ${node.meta ?? ''} ${json(node.value)}`] + case 'html': return [`html ${json(node.value)}`] + case 'blockquote': return container('blockquote', kids(false)) + case 'list': + return node.children.flatMap((listItem, index) => { + const nested = [] + const body = listItem.children.flatMap((child) => + child.type === 'list' ? (nested.push(...fromMdast(child, level + 1)), []) : fromMdast(child, level), + ) + const number = node.ordered ? (node.start ?? 1) + index : null + return [...container(item(level, number, listItem.checked, node.spread), body), ...nested] + }) + case 'table': + return node.children.flatMap((row, r) => [ + ...row.children.flatMap((cell, c) => container(`cell ${r} ${c}`, merge(cell.children.flatMap((n) => fromMdast(n))))), + ...(r === 0 ? [alignment(node.align)] : []), + ]) + case 'definition': return [`definition ${node.identifier} ${node.url} ${node.title ?? ''}`] + case 'footnoteDefinition': return container(`footnote ${node.identifier}`, kids(false)) + case 'footnoteReference': return [`footnoteref ${node.identifier}`] + case 'math': return [`math ${json(node.value)}`] + case 'inlineMath': return [`imath ${json(node.value)}`] + case 'yaml': return [`yaml ${json(node.value)}`] + case 'toml': return [`toml ${json(node.value)}`] + case 'text': return [text(node.value)] + case 'emphasis': return container('em', kids()) + case 'strong': return container('strong', kids()) + case 'delete': return container('del', kids()) + case 'inlineCode': return [`code_inline ${json(node.value)}`] + case 'break': return ['br'] + case 'link': return container(`link ${node.url} ${node.title ?? ''}`, kids()) + case 'image': return [`image ${node.url} ${node.title ?? ''} ${json(node.alt)}`] + case 'linkReference': return container(`linkref ${node.identifier}`, kids()) + case 'imageReference': return [`imageref ${node.identifier} ${json(node.alt)}`] case 'mdxJsxFlowElement': case 'mdxJsxTextElement': { const kind = node.type === 'mdxJsxFlowElement' ? 'flow' : 'text' - const attributes = node.attributes.map((a) => ' ' + attribute(a)).join('') - return [`${kind} <${node.name ?? ''}${attributes}>(`, ...children(node), ')'] + const attributes = node.attributes.map((a) => ' ' + mdxAttribute(a)).join('') + return container(`${kind} <${node.name ?? ''}${attributes}>`, kids(node.type === 'mdxJsxTextElement')) } case 'mdxFlowExpression': return [`flowexpr {${node.value}}`] case 'mdxTextExpression': return [`textexpr {${node.value}}`] - case 'mdxjsEsm': return [`esm ${JSON.stringify(node.value)}`] - case 'text': case 'inlineCode': case 'code': return [`T${JSON.stringify(node.value)}`] - case 'html': return [`html ${JSON.stringify(node.value)}`] - case 'yaml': return ['yaml'] - case 'thematicBreak': return ['hr'] - default: return children(node) + case 'mdxjsEsm': return [`esm ${json(node.value)}`] + default: throw new Error(`unknown mdast node ${node.type}`) } } -// Adjacent text is joined and whitespace collapsed, as the node structure of paragraphs differs. -function merge(items) { - const out = [] - for (const item of items) { - if (!item.startsWith('T')) { - out.push(item) - continue +function mqAttribute(attribute) { + if (attribute.type === 'expression') return `...{${attribute.value}}` + const {name, value} = attribute.value + if (value === null) return name + return value.type === 'literal' ? `${name}=lit:${json(value.value)}` : `${name}=expr:{${value.value}}` +} + +function fromMq(node) { + const kids = (inner = true) => { + const tokens = (node.values ?? node.children ?? []).flatMap(fromMq) + return inner ? merge(tokens) : tokens + } + switch (node.type) { + case 'Heading': return container(`heading ${node.depth}`, kids()) + case 'HorizontalRule': return ['hr'] + case 'Code': return [`code ${node.lang ?? ''} ${node.meta ?? ''} ${json(node.value)}`] + case 'Html': return [`html ${json(node.value)}`] + case 'Blockquote': return container('blockquote', kids(false)) + case 'List': { + const number = node.ordered ? (node.start ?? 1) + node.index : null + return container(item(node.level, number, node.checked, node.spread), kids(false)) } - const text = JSON.parse(item.slice(1)).replace(/\s+/g, ' ').trim() - if (!text) continue - if (out.length && out.at(-1).startsWith('T')) { - out[out.length - 1] = 'T' + JSON.stringify(JSON.parse(out.at(-1).slice(1)) + text) - } else { - out.push('T' + JSON.stringify(text)) + case 'TableCell': return container(`cell ${node.row} ${node.column}`, kids()) + case 'TableAlign': return [alignment(node.align)] + case 'Definition': return [`definition ${node.ident} ${node.url} ${node.title ?? ''}`] + case 'Footnote': return container(`footnote ${node.ident}`, kids(false)) + case 'FootnoteRef': return [`footnoteref ${node.ident}`] + case 'Math': return [`math ${json(node.value)}`] + case 'MathInline': return [`imath ${json(node.value)}`] + case 'Yaml': return [`yaml ${json(node.value)}`] + case 'Toml': return [`toml ${json(node.value)}`] + case 'Text': return [text(node.value)] + case 'Emphasis': return container('em', kids()) + case 'Strong': return container('strong', kids()) + case 'Delete': return container('del', kids()) + case 'CodeInline': return [`code_inline ${json(node.value)}`] + case 'Break': return ['br'] + case 'Link': return container(`link ${node.url} ${node.title ?? ''}`, kids()) + case 'Image': return [`image ${node.url} ${node.title ?? ''} ${json(node.alt)}`] + case 'LinkRef': return container(`linkref ${node.ident}`, kids()) + case 'ImageRef': return [`imageref ${node.ident} ${json(node.alt)}`] + case 'MdxJsxFlowElement': + case 'MdxJsxTextElement': { + const kind = node.type === 'MdxJsxFlowElement' ? 'flow' : 'text' + const attributes = node.attributes.map((a) => ' ' + mqAttribute(a)).join('') + return container(`${kind} <${node.name ?? ''}${attributes}>`, kids(node.type === 'MdxJsxTextElement')) } + case 'MdxFlowExpression': return [`flowexpr {${node.value}}`] + case 'MdxTextExpression': return [`textexpr {${node.value}}`] + case 'MdxJsEsm': return [`esm ${json(node.value)}`] + case 'Empty': return [] + default: throw new Error(`unknown mq node ${node.type}`) } - return out } -const official = (input) => { +const OPTIONS = { + markdown: { + extensions: [gfm(), frontmatter(['yaml', 'toml']), math()], + mdastExtensions: [gfmFromMarkdown(), frontmatterFromMarkdown(['yaml', 'toml']), mathFromMarkdown()], + }, + mdx: {extensions: [mdxjs()], mdastExtensions: [mdxFromMarkdown()]}, +} + +function official({mode, input}) { try { - const tree = fromMarkdown(input, {extensions: [mdxjs()], mdastExtensions: [mdxFromMarkdown()]}) - return {error: null, items: merge(children(tree))} + return {error: null, tokens: fromMdast(fromMarkdown(input, OPTIONS[mode]))} } catch (error) { - return {error: String(error.reason ?? error.message).split('\n')[0], items: []} + if (error.message.startsWith('unknown mdast node')) throw error + return {error: String(error.reason ?? error.message).split('\n')[0], tokens: []} } } const unexpected = [] const fixed = [] -corpus.forEach((input, index) => { - const expected = official(input) - const actual = {error: mq[index].error, items: merge(mq[index].items)} +let differing = 0 +corpus.forEach((entry, index) => { + const expected = official(entry) + const mq = actual[index] + const got = mq.error === null ? {error: null, tokens: mq.nodes.flatMap(fromMq)} : {error: mq.error, tokens: []} const same = - (expected.error === null) === (actual.error === null) && - (expected.error !== null || JSON.stringify(expected.items) === JSON.stringify(actual.items)) - if (!same && !(input in known)) { - unexpected.push(`${JSON.stringify(input)}\n official: ${expected.error ?? JSON.stringify(expected.items)}\n mq: ${actual.error ?? JSON.stringify(actual.items)}`) - } else if (same && input in known) { - fixed.push(JSON.stringify(input)) + (expected.error === null) === (got.error === null) && + (expected.error !== null || json(expected.tokens) === json(got.tokens)) + if (!same) differing++ + if (!same && !(entry.id in known)) { + const at = expected.tokens.findIndex((token, i) => token !== got.tokens[i]) + unexpected.push( + `${entry.id} ${json(entry.input)}\n official: ${expected.error ?? json(expected.tokens.slice(Math.max(at, 0)))}\n mq: ${got.error ?? json(got.tokens.slice(Math.max(at, 0)))}`, + ) + } else if (same && entry.id in known) { + fixed.push(entry.id) } }) -for (const input of Object.keys(known)) { - if (!corpus.includes(input)) unexpected.push(`known.json has an input that is not in corpus.json: ${JSON.stringify(input)}`) +const ids = new Set(corpus.map((entry) => entry.id)) +for (const id of Object.keys(known)) { + if (!ids.has(id)) unexpected.push(`known.json has an id that is not in the corpus: ${id}`) } -if (unexpected.length) console.error(`differs from the official MDX:\n${unexpected.join('\n')}`) +if (unexpected.length) console.error(`differs from the official packages:\n${unexpected.join('\n')}`) if (fixed.length) console.error(`now the same, remove from known.json:\n${fixed.join('\n')}`) -console.log(`${corpus.length} inputs, ${Object.keys(known).length} known differences`) +console.log(`${corpus.length} inputs, ${differing} differ, ${Object.keys(known).length} known`) process.exit(unexpected.length || fixed.length ? 1 : 0) diff --git a/crates/mq-markdown/tests/mdast_official/corpus-markdown.json b/crates/mq-markdown/tests/mdast_official/corpus-markdown.json new file mode 100644 index 000000000..c9e817abf --- /dev/null +++ b/crates/mq-markdown/tests/mdast_official/corpus-markdown.json @@ -0,0 +1,12 @@ +[ +"# a\n\n- x\n - y\n- [ ] z\n\n1. q\n", +"| a | b |\n|:-|-:|\n| 1 | 2 |\n", +"> t *e* **s** ~~d~~ `c` [l](u \"t\") ![i](u) [r][d] <b> a \nb\n\n[d]: /u\n", +"[^1]\n\n[^1]: note\n", +"---\ntitle: x\n---\n\n# a\n", +"+++\ntitle = 1\n+++\n\na\n", +"$$\nm\n$$\n\n$i$\n", +"```rs x\nc\n```\n", +"a\n\nb\n", +"*a*\n\n*b*\n" +] diff --git a/crates/mq-markdown/tests/mdast_official/corpus.mjs b/crates/mq-markdown/tests/mdast_official/corpus.mjs new file mode 100644 index 000000000..84069555e --- /dev/null +++ b/crates/mq-markdown/tests/mdast_official/corpus.mjs @@ -0,0 +1,41 @@ +// Writes the inputs to compare to the file in the first argument: the hand-written corpora and the +// examples of the specs, which are fetched over the network. +import fs from 'node:fs' + +const dir = new URL('.', import.meta.url) +const read = (file) => JSON.parse(fs.readFileSync(new URL(file, dir), 'utf8')) + +const SPECS = [ + ['commonmark', 'https://raw.githubusercontent.com/commonmark/commonmark-spec/0.31.2/spec.txt'], + ['gfm', 'https://raw.githubusercontent.com/github/cmark-gfm/828322d1ee4facdab56f0d3edccb13e9af90dcd2/test/spec.txt'], + ['gfm-extensions', 'https://raw.githubusercontent.com/github/cmark-gfm/828322d1ee4facdab56f0d3edccb13e9af90dcd2/test/extensions.txt'], +] + +const fence = '`'.repeat(32) + +/** The markdown of each example of a spec, numbered from 1 as in the specs. */ +function examples(text) { + const lines = text.split('\n') + const out = [] + for (let i = 0; i < lines.length; i++) { + if (lines[i] !== `${fence} example`) continue + const markdown = [] + for (i++; lines[i] !== '.'; i++) markdown.push(lines[i]) + while (lines[i] !== fence) i++ + out.push(markdown.join('\n').replaceAll('→', '\t') + '\n') + } + return out +} + +const corpus = [] +for (const input of read('corpus-mdx.json')) corpus.push({id: `mdx:${input}`, mode: 'mdx', input}) +for (const input of read('corpus-markdown.json')) corpus.push({id: `markdown:${input}`, mode: 'markdown', input}) +for (const [name, url] of SPECS) { + const response = await fetch(url) + if (!response.ok) throw new Error(`${url}: ${response.status}`) + examples(await response.text()).forEach((input, index) => { + corpus.push({id: `${name}#${index + 1}`, mode: 'markdown', input}) + }) +} +fs.writeFileSync(process.argv[2], JSON.stringify(corpus)) +console.log(`${corpus.length} inputs`) diff --git a/crates/mq-markdown/tests/mdast_official/known.json b/crates/mq-markdown/tests/mdast_official/known.json new file mode 100644 index 000000000..d32688eb8 --- /dev/null +++ b/crates/mq-markdown/tests/mdast_official/known.json @@ -0,0 +1,27 @@ +{ + "mdx:{a {b} c}": "Expressions are not parsed as JavaScript, so invalid JavaScript is accepted.", + "mdx:{a\nb}": "Expressions are not parsed as JavaScript, so invalid JavaScript is accepted.", + "mdx:{a\n b\n c}": "Expressions are not parsed as JavaScript, so invalid JavaScript is accepted.", + "mdx:<a {b} />": "Expressions are not parsed as JavaScript, so invalid JavaScript is accepted.", + "mdx:{a\n\nb}": "Expressions are not parsed as JavaScript, so invalid JavaScript is accepted.", + "mdx:{'a\nb'}": "Expressions are not parsed as JavaScript, so invalid JavaScript is accepted.", + "mdx:<a": "A tag that is not closed before the end of its text is text here, and an error in the official packages.", + "mdx:<a b": "A tag that is not closed before the end of its text is text here, and an error in the official packages.", + "mdx:<a b=": "A tag that is not closed before the end of its text is text here, and an error in the official packages.", + "mdx:x <a": "A tag that is not closed before the end of its text is text here, and an error in the official packages.", + "mdx:x <a b": "A tag that is not closed before the end of its text is text here, and an error in the official packages.", + "mdx:x <a b=": "A tag that is not closed before the end of its text is text here, and an error in the official packages.", + "mdx:<a\n\nb": "A tag that is not closed before the end of its text is text here, and an error in the official packages.", + "mdx:<a\nb": "A tag that is not closed before the end of its text is text here, and an error in the official packages.", + "mdx:a\n<a": "A tag that is not closed before the end of its text is text here, and an error in the official packages.", + "mdx:x <": "A tag that is not closed before the end of its text is text here, and an error in the official packages.", + "mdx:<": "A tag that is not closed before the end of its text is text here, and an error in the official packages.", + "mdx:x <a\n\nb": "A tag that is not closed before the end of its text is text here, and an error in the official packages.", + "mdx:x <a b\n\nc": "A tag that is not closed before the end of its text is text here, and an error in the official packages.", + "mdx:- <a": "A tag that is not closed before the end of its text is text here, and an error in the official packages.", + "mdx:> <a": "A tag that is not closed before the end of its text is text here, and an error in the official packages.", + "commonmark#606": "GFM autolink literals: an email after a backslash escape is text here, as in the CommonMark spec.", + "gfm#602": "GFM autolink literals: an email after a backslash escape is text here, as in the CommonMark spec.", + "gfm-extensions#19": "GFM autolink literals: the text of a `mailto:` link includes the scheme here, as in the GFM spec.", + "gfm-extensions#20": "GFM autolink literals: the spec only requires that this input does not crash." +} \ No newline at end of file diff --git a/crates/mq-markdown/tests/mdast_official/package-lock.json b/crates/mq-markdown/tests/mdast_official/package-lock.json index b558da4ea..45f0b43d3 100644 --- a/crates/mq-markdown/tests/mdast_official/package-lock.json +++ b/crates/mq-markdown/tests/mdast_official/package-lock.json @@ -1,12 +1,18 @@ { - "name": "mdx_official", + "name": "mdast_official", "lockfileVersion": 3, "requires": true, "packages": { "": { "dependencies": { "mdast-util-from-markdown": "2.1.0", + "mdast-util-frontmatter": "^2.0.1", + "mdast-util-gfm": "^3.1.0", + "mdast-util-math": "^3.0.0", "mdast-util-mdx": "3.0.0", + "micromark-extension-frontmatter": "^2.0.0", + "micromark-extension-gfm": "^3.0.0", + "micromark-extension-math": "^3.1.0", "micromark-extension-mdxjs": "3.0.0" } }, @@ -43,6 +49,12 @@ "@types/unist": "*" } }, + "node_modules/@types/katex": { + "version": "0.16.8", + "resolved": "https://registry.npmjs.org/@types/katex/-/katex-0.16.8.tgz", + "integrity": "sha512-trgaNyfU+Xh2Tc+ABIb44a5AYUpicB3uwirOioeOkNPPbmgRNtcWyDeeFRzjPZENO9Vq8gvVqfhaaXWLlevVwg==", + "license": "MIT" + }, "node_modules/@types/mdast": { "version": "4.0.4", "resolved": "https://registry.npmjs.org/@types/mdast/-/mdast-4.0.4.tgz", @@ -136,6 +148,15 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/commander": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-8.3.0.tgz", + "integrity": "sha512-OkTL9umf+He2DZkUq8f8J9of7yL6RJKI24dVITBmNfZBmri9zYZQrKkuXiKhyfPSu8tUhnVBB1iKXevvnlR4Ww==", + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, "node_modules/debug": { "version": "4.4.3", "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", @@ -188,6 +209,18 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/escape-string-regexp": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-5.0.0.tgz", + "integrity": "sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/estree-util-is-identifier-name": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/estree-util-is-identifier-name/-/estree-util-is-identifier-name-3.0.0.tgz", @@ -212,6 +245,27 @@ "url": "https://opencollective.com/unified" } }, + "node_modules/fault": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/fault/-/fault-2.0.1.tgz", + "integrity": "sha512-WtySTkS4OKev5JtpHXnib4Gxiurzh5NCGvWrFaZ34m6JehfTUhKZvn9njTfw48t6JumVQOmrKqpmGcdwxnhqBQ==", + "license": "MIT", + "dependencies": { + "format": "^0.2.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/format": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/format/-/format-0.2.2.tgz", + "integrity": "sha512-wzsgA6WOq+09wrU1tsJ09udeR/YZRaeArL9e1wPbFg3GG2yDnC2ldKpxs4xunpFF9DgqCqOIra3bc1HWrJ37Ww==", + "engines": { + "node": ">=0.4.x" + } + }, "node_modules/is-alphabetical": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/is-alphabetical/-/is-alphabetical-2.0.1.tgz", @@ -256,6 +310,22 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/katex": { + "version": "0.16.47", + "resolved": "https://registry.npmjs.org/katex/-/katex-0.16.47.tgz", + "integrity": "sha512-Eeo8Ys1doU1z+x8AZsPpQu+p/QcZBI5PeOo7QGQdy2x2m0MU/hYagBbGOmXwr5KVbEfVuWv9LpnQWeehogurjg==", + "funding": [ + "https://opencollective.com/katex", + "https://github.com/sponsors/katex" + ], + "license": "MIT", + "dependencies": { + "commander": "^8.3.0" + }, + "bin": { + "katex": "cli.js" + } + }, "node_modules/longest-streak": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/longest-streak/-/longest-streak-3.1.0.tgz", @@ -266,6 +336,32 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/markdown-table": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/markdown-table/-/markdown-table-3.0.4.tgz", + "integrity": "sha512-wiYz4+JrLyb/DqW2hkFJxP7Vd7JuTDm77fvbM8VfEQdmSMqcImWeeRbHwZjBjIFki/VaMK2BhFi7oUUZeM5bqw==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/mdast-util-find-and-replace": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/mdast-util-find-and-replace/-/mdast-util-find-and-replace-3.0.3.tgz", + "integrity": "sha512-xPgpDNl0/OXHsI7TlaIs22lWnH3KlpvZdXG1ET/m/YT2Hhdkx8lJV4kLphE6l9simyM4KPDFBwas08BgEob6Jw==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "escape-string-regexp": "^5.0.0", + "unist-util-is": "^6.0.0", + "unist-util-visit-parents": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/mdast-util-from-markdown": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/mdast-util-from-markdown/-/mdast-util-from-markdown-2.1.0.tgz", @@ -290,6 +386,144 @@ "url": "https://opencollective.com/unified" } }, + "node_modules/mdast-util-frontmatter": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/mdast-util-frontmatter/-/mdast-util-frontmatter-2.0.1.tgz", + "integrity": "sha512-LRqI9+wdgC25P0URIJY9vwocIzCcksduHQ9OF2joxQoyTNVduwLAFUzjoopuRJbJAReaKrNQKAZKL3uCMugWJA==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "devlop": "^1.0.0", + "escape-string-regexp": "^5.0.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0", + "micromark-extension-frontmatter": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/mdast-util-gfm/-/mdast-util-gfm-3.1.0.tgz", + "integrity": "sha512-0ulfdQOM3ysHhCJ1p06l0b0VKlhU0wuQs3thxZQagjcjPrlFRqY215uZGHHJan9GEAXd9MbfPjFJz+qMkVR6zQ==", + "license": "MIT", + "dependencies": { + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-gfm-autolink-literal": "^2.0.0", + "mdast-util-gfm-footnote": "^2.0.0", + "mdast-util-gfm-strikethrough": "^2.0.0", + "mdast-util-gfm-table": "^2.0.0", + "mdast-util-gfm-task-list-item": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm-autolink-literal": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-autolink-literal/-/mdast-util-gfm-autolink-literal-2.0.1.tgz", + "integrity": "sha512-5HVP2MKaP6L+G6YaxPNjuL0BPrq9orG3TsrZ9YXbA3vDw/ACI4MEsnoDpn6ZNm7GnZgtAcONJyPhOP8tNJQavQ==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "ccount": "^2.0.0", + "devlop": "^1.0.0", + "mdast-util-find-and-replace": "^3.0.0", + "micromark-util-character": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm-footnote": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-footnote/-/mdast-util-gfm-footnote-2.1.0.tgz", + "integrity": "sha512-sqpDWlsHn7Ac9GNZQMeUzPQSMzR6Wv0WKRNvQRg0KqHh02fpTz69Qc1QSseNX29bhz1ROIyNyxExfawVKTm1GQ==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "devlop": "^1.1.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm-strikethrough": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-strikethrough/-/mdast-util-gfm-strikethrough-2.0.1.tgz", + "integrity": "sha512-OuJHqvr455pwu2OaOrir7dbzqs4jlWKOtlu9L6GjcIjjQwNyw02VntQsr/Ek6/A13vgnXfUhBekWRUP1OkNivw==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm-table": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-table/-/mdast-util-gfm-table-2.0.0.tgz", + "integrity": "sha512-78UEvebzz/rJIxLvE7ZtDd/vIQ0RHv+3Mh5DR96p7cS7HsBhYIICDBCu8csTNWNO6tBWfqXPWekRuj2FNOGOZg==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "devlop": "^1.0.0", + "markdown-table": "^3.0.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-gfm-task-list-item": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-gfm-task-list-item/-/mdast-util-gfm-task-list-item-2.0.0.tgz", + "integrity": "sha512-IrtvNvjxC1o06taBAVJznEnkiHxLFTzgonUdy8hzFVeDun0uTjxxrRGVaNFqkU1wJR3RBPEfsxmU6jDWPofrTQ==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "devlop": "^1.0.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-math": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-math/-/mdast-util-math-3.0.0.tgz", + "integrity": "sha512-Tl9GBNeG/AhJnQM221bJR2HPvLOSnLE/T9cJI9tlc6zwQk2nPk/4f0cHkOdEixQPC/j8UtKDdITswvLAy1OZ1w==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "@types/mdast": "^4.0.0", + "devlop": "^1.0.0", + "longest-streak": "^3.0.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.1.0", + "unist-util-remove-position": "^5.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/mdast-util-mdx": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/mdast-util-mdx/-/mdast-util-mdx-3.0.0.tgz", @@ -488,6 +722,162 @@ "micromark-util-types": "^2.0.3" } }, + "node_modules/micromark-extension-frontmatter": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-extension-frontmatter/-/micromark-extension-frontmatter-2.0.0.tgz", + "integrity": "sha512-C4AkuM3dA58cgZha7zVnuVxBhDsbttIMiytjgsM2XbHAB2faRVaHRle40558FBN+DJcrLNCoqG5mlrpdU4cRtg==", + "license": "MIT", + "dependencies": { + "fault": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm/-/micromark-extension-gfm-3.0.0.tgz", + "integrity": "sha512-vsKArQsicm7t0z2GugkCKtZehqUm31oeGBV/KVSorWSy8ZlNAv7ytjFhvaryUiCUJYqs+NoE6AFhpQvBTM6Q4w==", + "license": "MIT", + "dependencies": { + "micromark-extension-gfm-autolink-literal": "^2.0.0", + "micromark-extension-gfm-footnote": "^2.0.0", + "micromark-extension-gfm-strikethrough": "^2.0.0", + "micromark-extension-gfm-table": "^2.0.0", + "micromark-extension-gfm-tagfilter": "^2.0.0", + "micromark-extension-gfm-task-list-item": "^2.0.0", + "micromark-util-combine-extensions": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-autolink-literal": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-autolink-literal/-/micromark-extension-gfm-autolink-literal-2.1.0.tgz", + "integrity": "sha512-oOg7knzhicgQ3t4QCjCWgTmfNhvQbDDnJeVu9v81r7NltNCVmhPy1fJRX27pISafdjL+SVc4d3l48Gb6pbRypw==", + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-sanitize-uri": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-footnote": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-footnote/-/micromark-extension-gfm-footnote-2.1.0.tgz", + "integrity": "sha512-/yPhxI1ntnDNsiHtzLKYnE3vf9JZ6cAisqVDauhp4CEHxlb4uoOTxOCJ+9s51bIB8U1N1FJ1RXOKTIlD5B/gqw==", + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-core-commonmark": "^2.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-sanitize-uri": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-strikethrough": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-strikethrough/-/micromark-extension-gfm-strikethrough-2.1.0.tgz", + "integrity": "sha512-ADVjpOOkjz1hhkZLlBiYA9cR2Anf8F4HqZUO6e5eDcPQd0Txw5fxLzzxnEkSkfnD0wziSGiv7sYhk/ktvbf1uw==", + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-classify-character": "^2.0.0", + "micromark-util-resolve-all": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-table": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-table/-/micromark-extension-gfm-table-2.1.2.tgz", + "integrity": "sha512-pRzm4kDTu0MjlmBkxmS9yYhw60nncfcEwu9NNdPFSQEFXS95ZKyIIyTSHu/o3ReBUrLKYEq+7YaXCRn/bPB4MA==", + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-tagfilter": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-tagfilter/-/micromark-extension-gfm-tagfilter-2.0.0.tgz", + "integrity": "sha512-xHlTOmuCSotIA8TW1mDIM6X2O1SiX5P9IuDtqGonFhEK0qgRI4yeC6vMxEV2dgyr2TiD+2PQ10o+cOhdVAcwfg==", + "license": "MIT", + "dependencies": { + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-gfm-task-list-item": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/micromark-extension-gfm-task-list-item/-/micromark-extension-gfm-task-list-item-2.1.0.tgz", + "integrity": "sha512-qIBZhqxqI6fjLDYFTBIa4eivDMnP+OZqsNwmQ3xNLE4Cxwc+zfQEfbs6tzAo2Hjq+bh6q5F+Z8/cksrLFYWQQw==", + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark-extension-math": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/micromark-extension-math/-/micromark-extension-math-3.1.0.tgz", + "integrity": "sha512-lvEqd+fHjATVs+2v/8kg9i5Q0AP2k85H0WUOwpIVvUML8BapsMvh1XAogmQjOCsLpoKRCVQqEkQBB3NhVBcsOg==", + "license": "MIT", + "dependencies": { + "@types/katex": "^0.16.0", + "devlop": "^1.0.0", + "katex": "^0.16.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/micromark-extension-mdx-expression": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/micromark-extension-mdx-expression/-/micromark-extension-mdx-expression-3.0.1.tgz", @@ -1105,6 +1495,20 @@ "url": "https://opencollective.com/unified" } }, + "node_modules/unist-util-remove-position": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/unist-util-remove-position/-/unist-util-remove-position-5.0.0.tgz", + "integrity": "sha512-Hp5Kh3wLxv0PHj9m2yZhhLt58KzPtEYKQQ4yxfYFEO7EvHwzyDYnduhHnY1mDxoqr7VUwVuHXk9RXKIiYS1N8Q==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "unist-util-visit": "^5.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/unist-util-stringify-position": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/unist-util-stringify-position/-/unist-util-stringify-position-4.0.0.tgz", diff --git a/crates/mq-markdown/tests/mdast_official/package.json b/crates/mq-markdown/tests/mdast_official/package.json index fe3e4f245..983c55c0f 100644 --- a/crates/mq-markdown/tests/mdast_official/package.json +++ b/crates/mq-markdown/tests/mdast_official/package.json @@ -4,7 +4,13 @@ "description": "Reference parser for tests/mdx_official.rs: the mdast of the official MDX packages.", "dependencies": { "mdast-util-from-markdown": "2.1.0", + "mdast-util-frontmatter": "^2.0.1", + "mdast-util-gfm": "^3.1.0", + "mdast-util-math": "^3.0.0", "mdast-util-mdx": "3.0.0", + "micromark-extension-frontmatter": "^2.0.0", + "micromark-extension-gfm": "^3.0.0", + "micromark-extension-math": "^3.1.0", "micromark-extension-mdxjs": "3.0.0" } } diff --git a/justfile b/justfile index 58ff9be9c..399f7e7c8 100644 --- a/justfile +++ b/justfile @@ -128,11 +128,13 @@ test-gfm-spec: test-spec: cargo test -p mq-markdown --lib -- --ignored spec_examples -# Compare the MDX nodes with those of the official MDX packages (installs them with npm) -test-mdx-official: - cd crates/mq-markdown/tests/mdx_official && npm install --silent - MDX_OFFICIAL_OUT="$PWD/target/mdx_official.json" cargo test -p mq-markdown --features json --test mdx_official -- --ignored - node crates/mq-markdown/tests/mdx_official/compare.mjs target/mdx_official.json +# Compare the nodes of Markdown and MDX with the mdast of the official packages (installs them with npm and +# fetches the specs) +test-mdast-official: + cd crates/mq-markdown/tests/mdast_official && npm install --silent + node crates/mq-markdown/tests/mdast_official/corpus.mjs target/mdast_corpus.json + MDAST_CORPUS="$PWD/target/mdast_corpus.json" MDAST_OUT="$PWD/target/mdast_nodes.json" cargo test -p mq-markdown --features json --test mdast_official -- --ignored + node crates/mq-markdown/tests/mdast_official/compare.mjs target/mdast_corpus.json target/mdast_nodes.json test-doc: cargo test --doc --workspace From 41bfb7e4fd0910d0da41415ab9dbefa64f3361ee Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Thu, 8 Oct 2026 13:07:27 +0900 Subject: [PATCH 25/88] =?UTF-8?q?=F0=9F=90=9B=20fix(markdown):=20match=20t?= =?UTF-8?q?he=20official=20packages=20on=20titles,=20fences=20and=20JSX=20?= =?UTF-8?q?text?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Titles across lines lose the indent of their lines. - Fenced code keeps what is left of a tab and of whitespace-only lines after the indent, also in list items. - Line endings inside the content of a text element are kept. --- crates/mq-markdown/src/parser/block.rs | 2 +- crates/mq-markdown/src/parser/code.rs | 18 +++++-- crates/mq-markdown/src/parser/inline.rs | 7 --- crates/mq-markdown/src/parser/inline/tail.rs | 6 ++- .../tests/parser_structure_tests.rs | 52 +++++++++++++++++++ 5 files changed, 71 insertions(+), 14 deletions(-) diff --git a/crates/mq-markdown/src/parser/block.rs b/crates/mq-markdown/src/parser/block.rs index 4f7b108c5..1e0c87966 100644 --- a/crates/mq-markdown/src/parser/block.rs +++ b/crates/mq-markdown/src/parser/block.rs @@ -586,7 +586,7 @@ fn list_item( break; } last_blank_has_whitespace = !line.text.is_empty(); - blanks.push(line.skip(line.text.len())); + blanks.push(line.skip_columns(marker.width)); } else if line.lazy { state.feed(line); inner.push(*line); diff --git a/crates/mq-markdown/src/parser/code.rs b/crates/mq-markdown/src/parser/code.rs index 2d1071856..65fb21776 100644 --- a/crates/mq-markdown/src/parser/code.rs +++ b/crates/mq-markdown/src/parser/code.rs @@ -1,6 +1,6 @@ //! Fenced and indented code, and math fences. -use super::block::{CODE_INDENT, join_lines}; +use super::block::CODE_INDENT; use super::inline; use super::line::Line; use super::tree::{Block, FencedBlock}; @@ -61,9 +61,10 @@ pub(super) fn fenced_code( index += 1; break; } - // Remove up to the opening fence's indentation from each content line. - let strip = line.text.bytes().take(indent).take_while(|&b| b == b' ').count(); - body.push((&line.text[strip..], line.eol)); + // Remove up to the opening fence's indentation from each content line. What is left of a tab + // that is only consumed in part stays as spaces. + let content = line.skip_columns(lines[start].pad + indent); + body.push((content.pad, content.text, line.eol)); index += 1; } @@ -84,7 +85,14 @@ pub(super) fn fenced_code( start: lines[start].content_point(indent), end, }); - let value = join_lines(&body); + let mut value = String::with_capacity(body.iter().map(|(pad, text, eol)| pad + text.len() + eol.len()).sum()); + for (index, (pad, text, eol)) in body.iter().enumerate() { + value.extend(std::iter::repeat_n(' ', *pad)); + value.push_str(text); + if index + 1 < body.len() { + value.push_str(eol); + } + } let node = if fence.marker == b'$' { Node::Math(Math { value, position }) } else { diff --git a/crates/mq-markdown/src/parser/inline.rs b/crates/mq-markdown/src/parser/inline.rs index 1d5e07f06..7c28a26d5 100644 --- a/crates/mq-markdown/src/parser/inline.rs +++ b/crates/mq-markdown/src/parser/inline.rs @@ -186,13 +186,6 @@ pub(super) fn to_nodes(items: Vec<Item>, context: &Context<'_>) -> Vec<Node> { for item in items { let (start, end, text): (usize, usize, Cow<'_, str>) = match item { - // A line ending directly inside a JSX element is not part of its content. - Item::Text { - value: Value::Slice(from, to), - .. - } if !open.is_empty() && from < to && src[from..to].bytes().all(|b| matches!(b, b'\n' | b'\r')) => { - continue; - } Item::Text { start, end, value } => match value { // The backslash of an escape is not part of the position of the text it starts. Value::Slice(from, to) => (from, end, Cow::Borrowed(&src[from..to])), diff --git a/crates/mq-markdown/src/parser/inline/tail.rs b/crates/mq-markdown/src/parser/inline/tail.rs index af4bc86e6..20197b581 100644 --- a/crates/mq-markdown/src/parser/inline/tail.rs +++ b/crates/mq-markdown/src/parser/inline/tail.rs @@ -170,7 +170,11 @@ pub(in crate::parser) fn title_at(src: &str, pos: usize) -> Option<(String, usiz let mut index = pos + 1; while let Some(&byte) = bytes.get(index) { match byte { - byte if byte == close => return Some((entity::unescape(&src[pos + 1..index]), index + 1)), + byte if byte == close => { + // The lines of a paragraph lose their leading whitespace, also inside a title. + let title = entity::remove_line_indent(&src[pos + 1..index]); + return Some((entity::unescape(&title), index + 1)); + } b'(' if close == b')' => return None, b'\\' => index += 1, b'\n' | b'\r' => { diff --git a/crates/mq-markdown/tests/parser_structure_tests.rs b/crates/mq-markdown/tests/parser_structure_tests.rs index 9b3e071c4..36cc3b58a 100644 --- a/crates/mq-markdown/tests/parser_structure_tests.rs +++ b/crates/mq-markdown/tests/parser_structure_tests.rs @@ -352,6 +352,58 @@ fn code_span_strips_one_space_or_line_ending(#[case] input: &str, #[case] expect assert_eq!(nodes[0].value(), expected, "{nodes:?}"); } +#[rstest] +#[case::title_lines_lose_their_indent("[a](b \"\n c\")\n", "\nc")] +#[case::definition_title("[a]\n\n[a]: b \"\n c\"\n", "\nc")] +fn titles_across_lines_lose_the_indent_of_the_lines(#[case] input: &str, #[case] expected: &str) { + let nodes = Markdown::from_markdown_str(input).unwrap().nodes; + let title = nodes.iter().find_map(|node| match node { + Node::Link(link) => link.title.as_ref().map(|title| title.to_string()), + Node::Definition(definition) => definition.title.as_ref().map(|title| title.to_string()), + _ => None, + }); + assert_eq!(title.as_deref(), Some(expected), "{nodes:?}"); +} + +#[rstest] +#[case::tab_after_an_indented_fence(" ```\n\tx", " x")] +#[case::tab_after_a_deeper_fence(" ```\n\tx", " x")] +#[case::spaces_after_an_item_marker("- ```\n \n ```", " ")] +#[case::tab_in_an_item("- ```\n\t\n ```", " ")] +fn fenced_code_keeps_the_whitespace_left_after_the_indent(#[case] input: &str, #[case] expected: &str) { + let nodes = Markdown::from_markdown_str(input).unwrap().nodes; + let code = match &nodes[0] { + Node::List(list) => &list.values[0], + node => node, + }; + assert_eq!(code.value(), expected, "{nodes:?}"); +} + +#[rstest] +#[case::in_a_paragraph("a <b>x\nc</b>", "x\nc")] +#[case::after_the_tag("a <b>\nc</b> d", "\nc")] +#[case::before_the_closing_tag("a <b>c\n</b> d", "c\n")] +#[case::in_a_quote("> a <b>\n> c </b> d.", "\nc ")] +fn line_endings_in_text_elements_are_kept(#[case] input: &str, #[case] expected: &str) { + let nodes = Markdown::from_mdx_str(input).unwrap().nodes; + let children = nodes + .iter() + .flat_map(|node| match node { + Node::Blockquote(quote) => quote.values.clone(), + node => vec![node.clone()], + }) + .find_map(|node| match node { + Node::MdxJsxTextElement(element) => Some(element.children), + _ => None, + }) + .unwrap(); + assert_eq!( + children.iter().map(Node::value).collect::<String>(), + expected, + "{nodes:?}" + ); +} + /// References keep the notation they were written in, as long as it resolves to the same definition. #[rstest] #[case::full_keeps_case_and_spaces("[x][Foo Bar]\n\n[foo bar]: /u\n", "[x][Foo Bar]")] From e3decbe0dfea3aaa6b3769a894d7a6e435820a68 Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Thu, 8 Oct 2026 13:07:33 +0900 Subject: [PATCH 26/88] =?UTF-8?q?=E2=9C=85=20test(markdown):=20add=20the?= =?UTF-8?q?=20tests=20of=20markdown-rs=20and=20micromark=20to=20the=20mdas?= =?UTF-8?q?t=20comparison?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/mdast_official/corpus.mjs | 94 ++++++++++++++++++- .../tests/mdast_official/known.json | 52 +++++++++- 2 files changed, 142 insertions(+), 4 deletions(-) diff --git a/crates/mq-markdown/tests/mdast_official/corpus.mjs b/crates/mq-markdown/tests/mdast_official/corpus.mjs index 84069555e..418695ca3 100644 --- a/crates/mq-markdown/tests/mdast_official/corpus.mjs +++ b/crates/mq-markdown/tests/mdast_official/corpus.mjs @@ -11,8 +11,79 @@ const SPECS = [ ['gfm-extensions', 'https://raw.githubusercontent.com/github/cmark-gfm/828322d1ee4facdab56f0d3edccb13e9af90dcd2/test/extensions.txt'], ] +const MARKDOWN_RS = 'https://raw.githubusercontent.com/wooorm/markdown-rs/1.0.0/tests' +// Test files whose inputs are read as MDX; the others are read as Markdown. `mdx_swc` needs a JavaScript parser. +const MDX_TESTS = ['mdx_esm', 'mdx_expression_flow', 'mdx_expression_text', 'mdx_jsx_flow', 'mdx_jsx_text'] +const MARKDOWN_RS_TESTS = [ + ...MDX_TESTS, + 'attention', 'autolink', 'block_quote', 'character_escape', 'character_reference', 'code_fenced', + 'code_indented', 'code_text', 'commonmark', 'definition', 'frontmatter', 'gfm_autolink_literal', + 'gfm_footnote', 'gfm_strikethrough', 'gfm_table', 'gfm_tagfilter', 'gfm_task_list_item', + 'hard_break_escape', 'hard_break_trailing', 'heading_atx', 'heading_setext', 'html_flow', 'html_text', + 'image', 'link_reference', 'link_resource', 'list', 'math_flow', 'math_text', 'misc_bom', + 'misc_dangerous_html', 'misc_dangerous_protocol', 'misc_line_ending', 'misc_soft_break', 'misc_tabs', + 'misc_url', 'misc_zero', 'text', 'thematic_break', +] +const MICROMARK_TESTS = [ + 'https://raw.githubusercontent.com/micromark/micromark-extension-mdx-jsx/3.0.2/test/index.js', + 'https://raw.githubusercontent.com/micromark/micromark-extension-mdx-expression/micromark-extension-mdx-expression%403.0.1/test/index.js', + 'https://raw.githubusercontent.com/micromark/micromark-extension-mdxjs-esm/3.0.0/test/index.js', +] + const fence = '`'.repeat(32) +const ESCAPES = {n: '\n', r: '\r', t: '\t', 0: '\0', '\\': '\\', '"': '"', "'": "'", '`': '`', $: '$'} + +/** The string literal that starts at `index` in Rust or JavaScript source, or null. */ +function literal(source, index) { + const quote = source[index] + if (!['"', "'", '`'].includes(quote)) return null + let value = '' + for (let i = index + 1; i < source.length; i++) { + const char = source[i] + if (char === quote) return value + if (char !== '\\') { + value += char + continue + } + const next = source[++i] + if (next === '\n') { + // A Rust line continuation drops the whitespace that follows. + while (/\s/.test(source[i + 1] ?? '')) i++ + } else if (next === 'u' && source[i + 1] === '{') { + const end = source.indexOf('}', i) + value += String.fromCodePoint(parseInt(source.slice(i + 2, end), 16)) + i = end + } else if (next === 'u' || next === 'x') { + const length = next === 'u' ? 4 : 2 + value += String.fromCodePoint(parseInt(source.slice(i + 1, i + 1 + length), 16)) + i += length + } else if (next in ESCAPES) { + value += ESCAPES[next] + } else { + return null + } + } + return null +} + +/** The first argument, when it is a string literal, of each call to one of `names` in `source`. */ +function firstArguments(source, names) { + const out = [] + const pattern = new RegExp(`\\b(?:${names.join('|')})\\(\\s*(?=["'\`])`, 'g') + for (const match of source.matchAll(pattern)) { + const value = literal(source, match.index + match[0].length) + if (value !== null) out.push(value) + } + return out +} + +async function fetchText(url) { + const response = await fetch(url) + if (!response.ok) throw new Error(`${url}: ${response.status}`) + return response.text() +} + /** The markdown of each example of a spec, numbered from 1 as in the specs. */ function examples(text) { const lines = text.split('\n') @@ -31,11 +102,28 @@ const corpus = [] for (const input of read('corpus-mdx.json')) corpus.push({id: `mdx:${input}`, mode: 'mdx', input}) for (const input of read('corpus-markdown.json')) corpus.push({id: `markdown:${input}`, mode: 'markdown', input}) for (const [name, url] of SPECS) { - const response = await fetch(url) - if (!response.ok) throw new Error(`${url}: ${response.status}`) - examples(await response.text()).forEach((input, index) => { + examples(await fetchText(url)).forEach((input, index) => { corpus.push({id: `${name}#${index + 1}`, mode: 'markdown', input}) }) } + +const seen = new Set(corpus.map((entry) => `${entry.mode}\0${entry.input}`)) +function addTests(name, mode, inputs) { + let count = 0 + for (const input of inputs) { + if (input.length > 5000 || !input.isWellFormed() || seen.has(`${mode}\0${input}`)) continue + seen.add(`${mode}\0${input}`) + corpus.push({id: `${name}#${++count}`, mode, input}) + } +} +for (const name of MARKDOWN_RS_TESTS) { + const source = await fetchText(`${MARKDOWN_RS}/${name}.rs`) + const mode = MDX_TESTS.includes(name) ? 'mdx' : 'markdown' + addTests(`markdown-rs/${name}`, mode, firstArguments(source, ['to_html', 'to_html_with_options', 'to_mdast'])) +} +for (const url of MICROMARK_TESTS) { + const name = url.split('/').slice(-4, -2).join('/').replace('%40', '@') + addTests(name, 'mdx', firstArguments(await fetchText(url), ['micromark'])) +} fs.writeFileSync(process.argv[2], JSON.stringify(corpus)) console.log(`${corpus.length} inputs`) diff --git a/crates/mq-markdown/tests/mdast_official/known.json b/crates/mq-markdown/tests/mdast_official/known.json index d32688eb8..8c3f2eac7 100644 --- a/crates/mq-markdown/tests/mdast_official/known.json +++ b/crates/mq-markdown/tests/mdast_official/known.json @@ -23,5 +23,55 @@ "commonmark#606": "GFM autolink literals: an email after a backslash escape is text here, as in the CommonMark spec.", "gfm#602": "GFM autolink literals: an email after a backslash escape is text here, as in the CommonMark spec.", "gfm-extensions#19": "GFM autolink literals: the text of a `mailto:` link includes the scheme here, as in the GFM spec.", - "gfm-extensions#20": "GFM autolink literals: the spec only requires that this input does not crash." + "gfm-extensions#20": "GFM autolink literals: the spec only requires that this input does not crash.", + "markdown-rs/mdx_esm#13": "Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted.", + "markdown-rs/mdx_esm#14": "Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted.", + "markdown-rs/mdx_esm#17": "Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted.", + "markdown-rs/mdx_esm#19": "Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted.", + "markdown-rs/mdx_esm#20": "Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted.", + "markdown-rs/mdx_esm#25": "Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted.", + "markdown-rs/mdx_expression_flow#23": "Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted.", + "markdown-rs/mdx_expression_flow#26": "Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted.", + "markdown-rs/mdx_expression_flow#27": "Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted.", + "markdown-rs/mdx_expression_flow#28": "Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted.", + "markdown-rs/mdx_expression_text#16": "Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted.", + "markdown-rs/mdx_expression_text#18": "Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted.", + "markdown-rs/mdx_expression_text#19": "Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted.", + "markdown-rs/mdx_expression_text#27": "Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted.", + "markdown-rs/mdx_jsx_flow#4": "Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted.", + "markdown-rs/mdx_jsx_text#30": "Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted.", + "markdown-rs/mdx_jsx_text#35": "Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted.", + "markdown-rs/mdx_jsx_text#36": "Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted.", + "markdown-rs/mdx_jsx_text#37": "Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted.", + "markdown-rs/mdx_jsx_text#122": "Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted.", + "markdown-rs/mdx_jsx_text#123": "Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted.", + "markdown-rs/mdx_jsx_text#128": "Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted.", + "micromark-extension-mdx-expression/micromark-extension-mdx-expression@3.0.1#6": "Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted.", + "micromark-extension-mdx-expression/micromark-extension-mdx-expression@3.0.1#7": "Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted.", + "micromark-extension-mdx-expression/micromark-extension-mdx-expression@3.0.1#9": "Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted.", + "micromark-extension-mdxjs-esm/3.0.0#11": "Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted.", + "micromark-extension-mdxjs-esm/3.0.0#12": "Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted.", + "markdown-rs/mdx_esm#16": "An ESM block continues over blank lines inside braces only when JavaScript says the statement is incomplete.", + "micromark-extension-mdxjs-esm/3.0.0#10": "An ESM block continues over blank lines inside braces only when JavaScript says the statement is incomplete.", + "markdown-rs/mdx_expression_flow#17": "A tab at the start of a line of an expression is dropped as one character of the indent.", + "markdown-rs/mdx_jsx_text#17": "An invalid numeric character reference in a JSX attribute is kept as written, not replaced with U+FFFD.", + "markdown-rs/code_fenced#34": "A null character is kept in values and replaced with U+FFFD only in HTML, so that positions stay the ones of the source.", + "markdown-rs/hard_break_trailing#11": "A null character is kept in values and replaced with U+FFFD only in HTML, so that positions stay the ones of the source.", + "markdown-rs/hard_break_trailing#12": "A null character is kept in values and replaced with U+FFFD only in HTML, so that positions stay the ones of the source.", + "markdown-rs/link_resource#72": "A null character is kept in values and replaced with U+FFFD only in HTML, so that positions stay the ones of the source.", + "markdown-rs/misc_zero#2": "A null character is kept in values and replaced with U+FFFD only in HTML, so that positions stay the ones of the source.", + "micromark-extension-mdx-expression/micromark-extension-mdx-expression@3.0.1#5": "A null character is kept in values and replaced with U+FFFD only in HTML, so that positions stay the ones of the source.", + "markdown-rs/autolink#14": "GFM autolink literals: an email after a backslash escape is text here, as in the CommonMark spec.", + "markdown-rs/gfm_autolink_literal#28": "GFM autolink literals: www after punctuation other than the ASCII ones of the GFM spec is text here.", + "markdown-rs/gfm_autolink_literal#33": "GFM autolink literals: the text of a `mailto:` or `xmpp:` link includes the scheme here, as in the GFM spec.", + "markdown-rs/code_fenced#36": "Blank lines at the end of an unclosed fence in a quote are one fewer in the value of the official packages.", + "markdown-rs/math_flow#35": "Blank lines at the end of an unclosed fence in a quote are one fewer in the value of the official packages.", + "markdown-rs/code_indented#23": "The lines of inline HTML lose all their indent here, and all but a space in the official packages.", + "markdown-rs/gfm_strikethrough#10": "A strikethrough inside `*...*` that CommonMark does not let open: emphasis does not open next to `~` here.", + "markdown-rs/gfm_table#19": "A table whose header row is empty is a table here, and text in the official packages.", + "markdown-rs/gfm_task_list_item#6": "A task list item whose checkbox is followed by a line ending is not a task here, and the whitespace after a checkbox is dropped.", + "markdown-rs/misc_tabs#15": "The official packages keep trailing whitespace of the info string, as the meta of a fence.", + "markdown-rs/misc_tabs#16": "The official packages keep trailing whitespace of the info string, as the meta of a fence.", + "markdown-rs/html_flow#147": "The official packages do not start HTML flow on a line that continues a paragraph in a quote, which CommonMark does.", + "markdown-rs/link_reference#46": "A reference followed by an empty label is a shortcut reference here, as in CommonMark, and text in the official packages." } \ No newline at end of file From ccf4a0b3374f11865eee98b9f69c328c2c053ea9 Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Thu, 8 Oct 2026 13:07:39 +0900 Subject: [PATCH 27/88] =?UTF-8?q?=F0=9F=90=9B=20fix(markdown):=20write=20f?= =?UTF-8?q?ragments,=20attributes,=20expressions,=20math=20and=20labels=20?= =?UTF-8?q?so=20they=20read=20back?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - A fragment is written as <></>, not < />. - A JSX attribute value is quoted with the quote it does not contain, and a character reference in it keeps its ampersand. - The lines of an expression are indented as they are read, so that reading it again does not remove more. - A math span or fence is longer than any run of dollar signs in it. - Definitions, footnotes and references keep the identifier they were written with, and an email that is not an autolink is written as a link. --- crates/mq-markdown/src/node.rs | 198 +++++++++++++++--- crates/mq-markdown/src/parser.rs | 2 +- crates/mq-markdown/src/parser/inline.rs | 4 +- .../mq-markdown/src/parser/inline/entity.rs | 2 +- crates/mq-markdown/src/parser/inline/html.rs | 6 + .../mq-markdown/tests/mdx_roundtrip_tests.rs | 29 +++ .../tests/parser_structure_tests.rs | 22 ++ 7 files changed, 233 insertions(+), 30 deletions(-) diff --git a/crates/mq-markdown/src/node.rs b/crates/mq-markdown/src/node.rs index 5e1d1c42b..b25eb992a 100644 --- a/crates/mq-markdown/src/node.rs +++ b/crates/mq-markdown/src/node.rs @@ -1312,7 +1312,7 @@ impl Node { let (us, ue) = &theme.link_url; format!( "[{}]: {}{}{}{}", - escape_label(label.as_deref().unwrap_or(ident)), + definition_label(label.as_deref(), ident), us, url.to_string_with(options), ue, @@ -1337,10 +1337,14 @@ impl Node { render_emphasis(values, delim, options, theme) } Self::Footnote(Footnote { values, ident, .. }) => { - format!("[^{}]: {}", escape_label(ident), render_values(values, options, theme)) + format!( + "[^{}]: {}", + escape_brackets(ident), + render_values(values, options, theme) + ) } - Self::FootnoteRef(FootnoteRef { label, .. }) => { - format!("[^{}]", escape_label(label.as_deref().unwrap_or_default())) + Self::FootnoteRef(FootnoteRef { label, ident, .. }) => { + format!("[^{}]", reference_label(label.as_deref(), ident)) } Self::Heading(Heading { depth, values, .. }) => { let (hs, he) = &theme.heading; @@ -1432,7 +1436,18 @@ impl Node { } Self::MathInline(MathInline { value, .. }) => { let (ms, me) = &theme.math; - format!("{}${}${}", ms, value, me) + let fence = math_span_fence(value); + // Padding keeps a dollar sign at an end from fusing with the fence, and a genuine space at + // both ends from being removed when it is read. + let all_spaces = value.chars().all(|c| c == ' '); + if value.starts_with('$') + || value.ends_with('$') + || (!all_spaces && value.starts_with(' ') && value.ends_with(' ')) + { + format!("{}{} {} {}{}", ms, fence, value, fence, me) + } else { + format!("{}{}{}{}{}", ms, fence, value, fence, me) + } } Self::Link(Link { url, title, values, .. }) => { let (ls, le) = &theme.link; @@ -1486,7 +1501,8 @@ impl Node { } Self::Math(Math { value, .. }) => { let (ms, me) = &theme.math; - format!("{}$$\n{}\n$${}", ms, value, me) + let fence = "$".repeat((longest_run(value, '$') + 1).max(2)); + format!("{}{}\n{}\n{}{}", ms, fence, value, fence, me) } // Only escape text parsed from real markdown (has a position); programmatic // values like JSON output or attr lookups must stay untouched. @@ -1498,7 +1514,7 @@ impl Node { } } Self::MdxFlowExpression(mdx_flow_expression) => { - format!("{{{}}}", mdx_flow_expression.value) + format!("{{{}}}", indent_expression(&mdx_flow_expression.value)) } Self::MdxJsxFlowElement(mdx_jsx_flow_element) => { let name = mdx_jsx_flow_element.name.as_deref().unwrap_or_default(); @@ -1516,18 +1532,24 @@ impl Node { }; if mdx_jsx_flow_element.children.is_empty() { - format!("<{}{} />", name, attributes,) + // A fragment has no self-closing form. + if name.is_empty() { + format!("<{}></>", attributes) + } else { + format!("<{}{} />", name, attributes,) + } } else { // Each tag and the children sit on their own lines, or the children would read as // inline content of the tag. let children = render_values_block(&mdx_jsx_flow_element.children, options, theme); - format!( - "<{}{}>\n{}\n</{}>", - name, - attributes, - indent_continuation(&format!(" {}", children), 2), - name - ) + // The lines of an expression are indented as they are written, which indenting them + // again would add to each time they are read. + let children = if mdx_jsx_flow_element.children.iter().any(has_multiline_expression) { + children + } else { + indent_continuation(&format!(" {}", children), 2) + }; + format!("<{}{}>\n{}\n</{}>", name, attributes, children, name) } } Self::MdxJsxTextElement(mdx_jsx_text_element) => { @@ -1546,7 +1568,12 @@ impl Node { }; if mdx_jsx_text_element.children.is_empty() { - format!("<{}{} />", name, attributes,) + // A fragment has no self-closing form. + if name.is_empty() { + format!("<{}></>", attributes) + } else { + format!("<{}{} />", name, attributes,) + } } else { format!( "<{}{}>{}</{}>", @@ -1558,7 +1585,7 @@ impl Node { } } Self::MdxTextExpression(mdx_text_expression) => { - format!("{{{}}}", mdx_text_expression.value) + format!("{{{}}}", indent_expression(&mdx_text_expression.value)) } Self::MdxJsEsm(mdxjs_esm) => mdxjs_esm.value.to_string(), Self::Strong(Strong { values, .. }) => { @@ -3184,11 +3211,15 @@ impl Node { fn mdx_attribute_content_to_string(attr: &MdxAttributeContent) -> SmolStr { match attr { - MdxAttributeContent::Expression(value) => format!("{{{}}}", value).into(), + MdxAttributeContent::Expression(value) => format!("{{{}}}", indent_expression(value)).into(), MdxAttributeContent::Property(property) => match &property.value { Some(value) => match value { - MdxAttributeValue::Expression(value) => format!("{}={{{}}}", property.name, value).into(), - MdxAttributeValue::Literal(literal) => format!("{}=\"{}\"", property.name, literal).into(), + MdxAttributeValue::Expression(value) => { + format!("{}={{{}}}", property.name, indent_expression(value)).into() + } + MdxAttributeValue::Literal(literal) => { + format!("{}={}", property.name, quote_attribute_value(literal)).into() + } }, None => property.name.clone(), }, @@ -3196,6 +3227,65 @@ impl Node { } } +/// Whether `node` or a node in it is an expression of more than one line. +fn has_multiline_expression(node: &Node) -> bool { + match node { + Node::MdxFlowExpression(expression) => expression.value.contains('\n'), + Node::MdxTextExpression(expression) => expression.value.contains('\n'), + Node::MdxJsxTextElement(MdxJsxTextElement { attributes, .. }) + | Node::MdxJsxFlowElement(MdxJsxFlowElement { attributes, .. }) + if attributes.iter().any(|attribute| match attribute { + MdxAttributeContent::Expression(value) => value.contains('\n'), + MdxAttributeContent::Property(MdxJsxAttribute { + value: Some(MdxAttributeValue::Expression(value)), + .. + }) => value.contains('\n'), + _ => false, + }) => + { + true + } + other => other.children().iter().any(has_multiline_expression), + } +} + +/// Quotes the value of a JSX attribute with the quote it does not contain. Character references that the +/// value holds as text are written with their `&` as a reference. +fn quote_attribute_value(value: &str) -> String { + let mut escaped = String::with_capacity(value.len()); + for (index, c) in value.char_indices() { + if c == '&' && starts_reference(&value[index + 1..]) { + escaped.push_str("&amp;"); + } else { + escaped.push(c); + } + } + if !escaped.contains('"') { + format!("\"{escaped}\"") + } else if !escaped.contains('\'') { + format!("'{escaped}'") + } else { + format!("\"{}\"", escaped.replace('"', "&quot;")) + } +} + +/// The content of an expression as it is written: the lines after the first are indented, as the +/// indentation of up to two characters is removed from them when they are read. +fn indent_expression(value: &str) -> String { + value + .split('\n') + .enumerate() + .map(|(index, line)| { + if index == 0 || line.is_empty() || line == "\r" { + line.to_string() + } else { + format!(" {line}") + } + }) + .collect::<Vec<_>>() + .join("\n") +} + pub(crate) fn values_to_string(values: &[Node], options: &RenderOptions) -> String { render_values(values, options, &ColorTheme::PLAIN) } @@ -3482,15 +3572,17 @@ pub(crate) fn list_own_prefix_width(ordered: bool, index: usize, start: Option<u marker_len + 1 } +/// The length of the longest run of `ch` in `s`. +fn longest_run(s: &str, ch: char) -> usize { + s.split('\n') + .flat_map(|line| line.split(|c| c != ch).map(str::len)) + .max() + .unwrap_or(0) +} + /// Longest fence-char run in `value` + 1 (min 3); switches to tilde when `info` /// has a backtick, since a backtick-fenced info string can't contain one. fn code_fence(value: &str, info: &str) -> String { - let longest_run = |s: &str, ch: char| -> usize { - s.split('\n') - .flat_map(|line| line.split(|c| c != ch).map(str::len)) - .max() - .unwrap_or(0) - }; if info.contains('`') { let len = (longest_run(value, '~') + 1).max(3); "~".repeat(len) @@ -3500,6 +3592,19 @@ fn code_fence(value: &str, info: &str) -> String { } } +/// The label to write in a definition: the identifier as it was written when the label is only that +/// without its escapes and character references, which would not read back as the same identifier. +fn definition_label(label: Option<&str>, ident: &str) -> String { + match label { + Some(label) if crate::parser::normalize(label) == ident => escape_label(label), + Some(label) if crate::parser::normalize(label) == crate::parser::normalize(&crate::parser::unescape(ident)) => { + ident.to_string() + } + Some(label) => escape_label(label), + None => ident.to_string(), + } +} + /// The label to write in the second brackets of a full reference. The label as written is kept when it /// still resolves to `ident`. fn reference_label(label: Option<&str>, ident: &str) -> String { @@ -3525,6 +3630,45 @@ fn code_span_fence(value: &str) -> String { "`".repeat(max_run + 1) } +/// The shortest run of dollar signs, from one, that is not the length of a run in `value`, so that no +/// run inside a math span can be mistaken for the closing. +fn math_span_fence(value: &str) -> String { + let mut lengths = std::collections::HashSet::new(); + let mut current = 0; + for c in value.chars() { + if c == '$' { + current += 1; + } else if current > 0 { + lengths.insert(current); + current = 0; + } + } + if current > 0 { + lengths.insert(current); + } + "$".repeat((1..).find(|length| !lengths.contains(length)).unwrap_or(1)) +} + +/// Escapes the brackets that are not escaped yet, for an identifier that keeps the escapes it was written with. +fn escape_brackets(s: &str) -> String { + let mut result = String::with_capacity(s.len()); + let mut chars = s.chars().peekable(); + while let Some(c) = chars.next() { + match c { + '\\' if chars.peek().is_some_and(char::is_ascii_punctuation) => { + result.push(c); + result.extend(chars.next()); + } + '[' | ']' => { + result.push('\\'); + result.push(c); + } + _ => result.push(c), + } + } + result +} + fn escape_label(s: &str) -> String { if !s.bytes().any(|b| matches!(b, b'\\' | b'[' | b']')) { return s.to_string(); @@ -3547,7 +3691,7 @@ fn autolink_target(url: &str, values: &[Node]) -> Option<String> { }; let target = if value == url { url - } else if url.strip_prefix("mailto:") == Some(value.as_str()) && value.contains('@') { + } else if url.strip_prefix("mailto:") == Some(value.as_str()) && crate::parser::is_autolink_email(value) { value.as_str() } else { return None; diff --git a/crates/mq-markdown/src/parser.rs b/crates/mq-markdown/src/parser.rs index 8a503f37a..8b6d275d5 100644 --- a/crates/mq-markdown/src/parser.rs +++ b/crates/mq-markdown/src/parser.rs @@ -23,7 +23,7 @@ mod tree; use crate::node::Node; -pub(crate) use inline::normalize; +pub(crate) use inline::{is_autolink_email, normalize, unescape}; /// Parses `content` into a flat list of nodes. pub(crate) fn parse(content: &str) -> miette::Result<Vec<Node>> { diff --git a/crates/mq-markdown/src/parser/inline.rs b/crates/mq-markdown/src/parser/inline.rs index 7c28a26d5..a1a7277fe 100644 --- a/crates/mq-markdown/src/parser/inline.rs +++ b/crates/mq-markdown/src/parser/inline.rs @@ -14,7 +14,9 @@ mod literal; mod punctuation; mod tail; -pub(super) use entity::{decode_references, remove_line_indent, unescape}; +pub(crate) use entity::unescape; +pub(super) use entity::{decode_references, remove_line_indent}; +pub(crate) use html::is_autolink_email; pub(crate) use link::normalize; pub(super) use tail::{destination, title_at}; diff --git a/crates/mq-markdown/src/parser/inline/entity.rs b/crates/mq-markdown/src/parser/inline/entity.rs index 54bf92354..2badf69ff 100644 --- a/crates/mq-markdown/src/parser/inline/entity.rs +++ b/crates/mq-markdown/src/parser/inline/entity.rs @@ -44,7 +44,7 @@ pub(super) fn decode(src: &str, pos: usize) -> Option<(usize, String)> { } /// Decodes backslash escapes and character references, as in link destinations and titles. -pub(in crate::parser) fn unescape(value: &str) -> String { +pub(crate) fn unescape(value: &str) -> String { let bytes = value.as_bytes(); let mut result = String::with_capacity(value.len()); let mut from = 0; diff --git a/crates/mq-markdown/src/parser/inline/html.rs b/crates/mq-markdown/src/parser/inline/html.rs index 0ad4dd4a8..9a01cef3f 100644 --- a/crates/mq-markdown/src/parser/inline/html.rs +++ b/crates/mq-markdown/src/parser/inline/html.rs @@ -49,6 +49,12 @@ pub(super) fn autolink(src: &str, pos: usize) -> Option<Autolink> { }) } +/// Whether `text` is an email address that an autolink (`<text>`) can hold. +pub(crate) fn is_autolink_email(text: &str) -> bool { + let closed = format!("{text}>"); + email(closed.as_bytes(), 0) == Some(text.len()) +} + /// Parses an email address for an autolink at `start`, returning the offset of the closing `>`. /// The characters allowed differ slightly from `CommonMark`, following `markdown-rs`. fn email(bytes: &[u8], start: usize) -> Option<usize> { diff --git a/crates/mq-markdown/tests/mdx_roundtrip_tests.rs b/crates/mq-markdown/tests/mdx_roundtrip_tests.rs index 3b225c376..574b24810 100644 --- a/crates/mq-markdown/tests/mdx_roundtrip_tests.rs +++ b/crates/mq-markdown/tests/mdx_roundtrip_tests.rs @@ -34,3 +34,32 @@ fn flow_elements_survive_a_round_trip(#[case] input: &str) { fn flow_elements_keep_tags_and_children_on_their_own_lines(#[case] input: &str, #[case] expected: &str) { assert_eq!(Markdown::from_mdx_str(input).unwrap().to_string().trim_end(), expected); } + +#[rstest] +#[case::fragment("<></>", "<></>")] +#[case::fragment_with_children("<>\n\nx\n\n</>", "<>\n x\n</>")] +#[case::double_quote_in_a_value("<a b='c\"d' />", "<a b='c\"d' />")] +#[case::both_quotes_in_a_value("<a b='&quot;&apos;' />", "<a b=\"&quot;'\" />")] +#[case::reference_in_a_value("<a b='&amp;copy;' />", "<a b=\"&amp;copy;\" />")] +#[case::lone_ampersand_in_a_value("<a b='c&d' />", "<a b=\"c&d\" />")] +fn jsx_elements_are_written_so_that_they_read_back(#[case] input: &str, #[case] expected: &str) { + let markdown = Markdown::from_mdx_str(input).unwrap(); + let written = markdown.to_string(); + assert_eq!(written.trim_end(), expected); + let reparsed = Markdown::from_mdx_str(&written).unwrap(); + assert_eq!(describe_all(&markdown), describe_all(&reparsed), "{written}"); +} + +#[rstest] +#[case::flow("{a\n b\n c}")] +#[case::text("x {a\n b} y")] +#[case::attribute("<a b={c\n d} />")] +#[case::spread("<a {...b\n ,c} />")] +#[case::in_an_element("<A>\n\n{`\n x\n`}\n\n</A>")] +fn expressions_keep_their_indentation_when_written_and_read_again(#[case] input: &str) { + let first = Markdown::from_mdx_str(input).unwrap(); + let second = Markdown::from_mdx_str(&first.to_string()).unwrap(); + let third = Markdown::from_mdx_str(&second.to_string()).unwrap(); + assert_eq!(describe_all(&first), describe_all(&second), "{first}"); + assert_eq!(second.to_string(), third.to_string()); +} diff --git a/crates/mq-markdown/tests/parser_structure_tests.rs b/crates/mq-markdown/tests/parser_structure_tests.rs index 36cc3b58a..a6caaf7de 100644 --- a/crates/mq-markdown/tests/parser_structure_tests.rs +++ b/crates/mq-markdown/tests/parser_structure_tests.rs @@ -404,6 +404,28 @@ fn line_endings_in_text_elements_are_kept(#[case] input: &str, #[case] expected: ); } +/// What `to_string` writes for these reads back as the same nodes. +#[rstest] +#[case::math_with_a_dollar("$$ foo $ bar $$\n", "$$foo $ bar$$")] +#[case::math_of_dollars("$ $$ $\n", "$ $$ $")] +#[case::math_block_with_a_fence_inside("$$$\naaa\n$$\n$$$$\n", "$$$\naaa\n$$\n$$$")] +#[case::definition_with_an_escape("[+]: a\n[\\;]: b\n", "[+]: a\n[\\;]: b")] +#[case::definition_with_a_reference("[&amp;]: a\n\n[&amp;]\n", "[&amp;]: a\n\n[&][&amp;]")] +#[case::footnote_with_a_reference("Call.[^a&copy;b].\n\n[^a&copy;b]: y\n", "Call.[^a&copy;b].\n\n[^a&copy;b]: y")] +#[case::email_that_is_not_an_autolink("<asd@-example.com>\n", "\\<[asd\\@-example.com](mailto:asd@-example.com)\\>")] +fn writing_keeps_what_is_read_back(#[case] input: &str, #[case] expected: &str) { + let markdown = Markdown::from_markdown_str(input).unwrap(); + let written = markdown.to_string(); + assert_eq!(written.trim_end(), expected, "{:?}", markdown.nodes); + let reparsed = Markdown::from_markdown_str(&written).unwrap(); + assert_eq!( + mq_markdown::to_html(&written), + mq_markdown::to_html(input), + "{written:?} {:?}", + reparsed.nodes + ); +} + /// References keep the notation they were written in, as long as it resolves to the same definition. #[rstest] #[case::full_keeps_case_and_spaces("[x][Foo Bar]\n\n[foo bar]: /u\n", "[x][Foo Bar]")] From 2fc14feb2220ab831544ad878ee72b354fe4425e Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Thu, 8 Oct 2026 13:07:45 +0900 Subject: [PATCH 28/88] =?UTF-8?q?=E2=9C=85=20test(markdown):=20check=20tha?= =?UTF-8?q?t=20the=20corpus=20is=20written=20back=20as=20the=20same=20node?= =?UTF-8?q?s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crates/mq-markdown/tests/mdast_official.rs | 109 +++++++++++++++++- .../mdast_official/round_trip_known.json | 12 ++ 2 files changed, 120 insertions(+), 1 deletion(-) create mode 100644 crates/mq-markdown/tests/mdast_official/round_trip_known.json diff --git a/crates/mq-markdown/tests/mdast_official.rs b/crates/mq-markdown/tests/mdast_official.rs index b5fe485a3..26845d0b9 100644 --- a/crates/mq-markdown/tests/mdast_official.rs +++ b/crates/mq-markdown/tests/mdast_official.rs @@ -1,5 +1,6 @@ //! Writes the nodes mq makes of each input of the corpus as JSON, for `mdast_official/compare.mjs` to -//! compare with the mdast of the official packages. Run it with `just test-mdast-official`. +//! compare with the mdast of the official packages, and checks that what mq writes back reads as the +//! same nodes. Run it with `just test-mdast-official`. #![cfg(feature = "json")] use mq_markdown::Markdown; @@ -26,3 +27,109 @@ fn write_nodes_of_the_corpus() { .collect(); std::fs::write(std::env::var("MDAST_OUT").unwrap(), json!(results).to_string()).unwrap(); } + +/// Normalizes nodes for the comparison of what is written with what was read: positions and the spelling +/// of references, fences and rules are left out, line endings are the same and adjacent text is joined. +fn normalize(value: &mut Value) { + match value { + Value::Object(object) => { + for key in ["position", "label", "fence"] { + object.remove(key); + } + if object.get("type").is_some_and(|kind| kind == "HorizontalRule") { + object.remove("marker"); + } + for (key, value) in object.iter_mut() { + normalize(value); + if let (Value::String(text), true) = (value, key == "value") { + *text = text.replace("\r\n", "\n").replace('\r', "\n"); + } + } + if object.get("type").is_some_and(|kind| kind == "Html") + && let Some(Value::String(text)) = object.get_mut("value") + { + *text = text.trim_end_matches('\n').to_string(); + } + } + Value::Array(array) => { + array.iter_mut().for_each(normalize); + let mut joined: Vec<Value> = Vec::with_capacity(array.len()); + for item in array.drain(..) { + if let Some(last) = joined.last_mut() + && item["type"] == "Text" + && last["type"] == "Text" + { + let text = format!("{}{}", last["value"].as_str().unwrap(), item["value"].as_str().unwrap()); + last["value"] = json!(text); + continue; + } + joined.push(item); + } + *array = joined; + } + _ => {} + } +} + +fn parse(mode: &str, input: &str) -> Option<Value> { + let markdown = match mode { + "mdx" => Markdown::from_mdx_str(input), + _ => Markdown::from_markdown_str(input), + } + .ok()?; + let mut nodes: Value = serde_json::from_str(&markdown.to_json().ok()?).unwrap(); + normalize(&mut nodes); + Some(nodes) +} + +/// Inputs whose written form reads as other nodes, by id, which `round_trip_known.json` lists. +#[test] +#[ignore = "needs the corpus; run via `just test-mdast-official`"] +fn the_corpus_round_trips() { + let corpus = std::fs::read_to_string(std::env::var("MDAST_CORPUS").unwrap()).unwrap(); + let corpus: Vec<Value> = serde_json::from_str(&corpus).unwrap(); + let known: Value = serde_json::from_str(include_str!("mdast_official/round_trip_known.json")).unwrap(); + let mut unexpected = Vec::new(); + let mut fixed = Vec::new(); + for entry in &corpus { + let (id, mode, input) = ( + entry["id"].as_str().unwrap(), + entry["mode"].as_str().unwrap(), + entry["input"].as_str().unwrap(), + ); + let Some(before) = parse(mode, input) else { continue }; + let written = match mode { + "mdx" => Markdown::from_mdx_str(input), + _ => Markdown::from_markdown_str(input), + } + .unwrap() + .to_string(); + let same = parse(mode, &written).as_ref() == Some(&before); + match (same, known.get(id).is_some()) { + (false, false) => unexpected.push(format!( + "{id} {input:?}\n written: {written:?}\n before: {}\n after: {}", + before, + parse(mode, &written).map_or("error".to_string(), |after| after.to_string()) + )), + (true, true) => fixed.push(id.to_string()), + _ => {} + } + } + let ids: Vec<&str> = corpus.iter().map(|entry| entry["id"].as_str().unwrap()).collect(); + for id in known.as_object().unwrap().keys() { + assert!( + ids.contains(&id.as_str()), + "round_trip_known.json has an id that is not in the corpus: {id}" + ); + } + println!("{} inputs, {} known", corpus.len(), known.as_object().unwrap().len()); + assert!( + unexpected.is_empty(), + "do not read back as the same nodes:\n{}", + unexpected.join("\n") + ); + assert!( + fixed.is_empty(), + "now the same, remove from round_trip_known.json: {fixed:?}" + ); +} diff --git a/crates/mq-markdown/tests/mdast_official/round_trip_known.json b/crates/mq-markdown/tests/mdast_official/round_trip_known.json new file mode 100644 index 000000000..d51e1c5f8 --- /dev/null +++ b/crates/mq-markdown/tests/mdast_official/round_trip_known.json @@ -0,0 +1,12 @@ +{ + "mdx: import a from \"b\"\n": "Text that starts with import or export is written without the escape that would keep it from being read as ESM.", + "markdown-rs/mdx_esm#7": "Text that starts with import or export is written without the escape that would keep it from being read as ESM.", + "mdx:a\\{b}": "A brace in MDX text is written without the escape, as the options do not tell Markdown from MDX.", + "mdx:a &#123;b}": "A brace in MDX text is written without the escape, as the options do not tell Markdown from MDX.", + "markdown-rs/mdx_jsx_text#113": "A brace in MDX text is written without the escape, as the options do not tell Markdown from MDX.", + "gfm-extensions#19": "Text that reads as an email address or a URL once written is linked, as it cannot be written so that it stays text.", + "markdown-rs/gfm_autolink_literal#31": "Text that reads as an email address or a URL once written is linked, as it cannot be written so that it stays text.", + "gfm-extensions#23": "A footnote definition with blocks, such as code, is written on one line.", + "markdown-rs/misc_line_ending#20": "Line endings of a lone carriage return are not kept when written.", + "micromark-extension-mdxjs-esm/3.0.0#3": "Text that starts with import or export is written without the escape that would keep it from being read as ESM." +} \ No newline at end of file From eb38ef8b770c99e3f5d70530944df5216ccc84ea Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Thu, 8 Oct 2026 13:07:51 +0900 Subject: [PATCH 29/88] =?UTF-8?q?=F0=9F=90=9B=20fix(markdown):=20keep=20a?= =?UTF-8?q?=20blank=20line=20before=20a=20table=20and=20an=20asterisk=20ne?= =?UTF-8?q?xt=20to=20letters=20when=20writing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - A table after a footnote or a block quote is written after a blank line, or it reads back as part of the block. - Emphasis at an end of a strong keeps its asterisks when a letter or digit is next to it, as an underscore does not open or close there. --- crates/mq-markdown/src/markdown.rs | 10 +++++++++- crates/mq-markdown/src/node.rs | 19 +++++++++++++++++-- .../tests/parser_structure_tests.rs | 8 ++++++++ 3 files changed, 34 insertions(+), 3 deletions(-) diff --git a/crates/mq-markdown/src/markdown.rs b/crates/mq-markdown/src/markdown.rs index a1437a66d..7f39ae6bd 100644 --- a/crates/mq-markdown/src/markdown.rs +++ b/crates/mq-markdown/src/markdown.rs @@ -102,11 +102,19 @@ impl Markdown { buffer.push('\n'); } else if let Some(pos) = node.position() { // Insert newlines before the first row of a table - let new_line_count = pre_position + let mut new_line_count = pre_position .as_ref() .map(|p| pos.start.line.saturating_sub(p.end.line)) .unwrap_or_else(|| if is_first { 0 } else { 1 }) .min(2); + // A footnote ends where the next line starts, and a single newline after a + // block quote reads back as lazy continuation: a table needs a blank line. + if new_line_count < 2 + && i > 0 + && matches!(nodes[i - 1], Node::Footnote(_) | Node::Blockquote(_)) + { + new_line_count = 2; + } for _ in 0..new_line_count { buffer.push('\n'); } diff --git a/crates/mq-markdown/src/node.rs b/crates/mq-markdown/src/node.rs index b25eb992a..224de01ac 100644 --- a/crates/mq-markdown/src/node.rs +++ b/crates/mq-markdown/src/node.rs @@ -1596,8 +1596,15 @@ impl Node { .enumerate() .map(|(index, value)| match value { // Emphasis at an end uses the other delimiter, or `*` and `**` fuse into `***`. - Self::Emphasis(Emphasis { values, .. }) if index == 0 || index == last => { - render_emphasis(values, "_", options, theme) + // An underscore does not open or close next to a letter or digit, so those keep `*`. + Self::Emphasis(Emphasis { values: inner, .. }) + if (index == 0 || index == last) + && !index + .checked_sub(1) + .is_some_and(|before| ends_with_word_char(&values[before])) + && !values.get(index + 1).is_some_and(starts_with_word_char) => + { + render_emphasis(inner, "_", options, theme) } value => value.render_with_theme(options, theme), }) @@ -3347,6 +3354,14 @@ pub(crate) fn render_cell_values(values: &[Node], options: &RenderOptions, theme result } +fn ends_with_word_char(node: &Node) -> bool { + matches!(node, Node::Text(Text { value, .. }) if value.ends_with(|c: char| c.is_alphanumeric())) +} + +fn starts_with_word_char(node: &Node) -> bool { + matches!(node, Node::Text(Text { value, .. }) if value.starts_with(|c: char| c.is_alphanumeric())) +} + /// Renders `value`, escaping a `!` that ends its text when `next` starts a link, which would make an image. pub(crate) fn render_before( value: &Node, diff --git a/crates/mq-markdown/tests/parser_structure_tests.rs b/crates/mq-markdown/tests/parser_structure_tests.rs index a6caaf7de..136297bb4 100644 --- a/crates/mq-markdown/tests/parser_structure_tests.rs +++ b/crates/mq-markdown/tests/parser_structure_tests.rs @@ -412,6 +412,14 @@ fn line_endings_in_text_elements_are_kept(#[case] input: &str, #[case] expected: #[case::definition_with_an_escape("[+]: a\n[\\;]: b\n", "[+]: a\n[\\;]: b")] #[case::definition_with_a_reference("[&amp;]: a\n\n[&amp;]\n", "[&amp;]: a\n\n[&][&amp;]")] #[case::footnote_with_a_reference("Call.[^a&copy;b].\n\n[^a&copy;b]: y\n", "Call.[^a&copy;b].\n\n[^a&copy;b]: y")] +#[case::table_after_a_footnote( + "[^f0]: a\n\n| 1 | 2 |\n|-|-|\n| a | b |\n", + "[^f0]: a\n\n| 1 | 2 |\n| - | - |\n| a | b |" +)] +#[case::table_after_a_quote("> a\n\n| 1 | 2 |\n|-|-|\n| a | b |\n", "> a\n\n| 1 | 2 |\n| - | - |\n| a | b |")] +#[case::emphasis_after_a_letter_in_strong("**a*b c***\n", "**a*b c***")] +#[case::emphasis_before_a_letter_in_strong("***a b*c**\n", "***a b*c**")] +#[case::emphasis_at_an_end_of_strong("**a *b***\n", "**a _b_**")] #[case::email_that_is_not_an_autolink("<asd@-example.com>\n", "\\<[asd\\@-example.com](mailto:asd@-example.com)\\>")] fn writing_keeps_what_is_read_back(#[case] input: &str, #[case] expected: &str) { let markdown = Markdown::from_markdown_str(input).unwrap(); From 651e70fb414ba0bd9073bfa9ff10fc5ca6ce2f18 Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Thu, 8 Oct 2026 13:07:58 +0900 Subject: [PATCH 30/88] =?UTF-8?q?=E2=9C=85=20test(markdown):=20compare=20r?= =?UTF-8?q?andom=20documents=20with=20the=20official=20packages?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A grammar of Markdown and MDX constructs makes the documents from a seed (MDAST_SEED, MDAST_RANDOM). They go through the same mdast comparison and the write-back check as the rest of the corpus. --- .../tests/mdast_official/corpus.mjs | 9 ++ .../tests/mdast_official/generate.mjs | 120 ++++++++++++++++++ .../tests/mdast_official/known.json | 8 +- .../mdast_official/round_trip_known.json | 3 +- 4 files changed, 138 insertions(+), 2 deletions(-) create mode 100644 crates/mq-markdown/tests/mdast_official/generate.mjs diff --git a/crates/mq-markdown/tests/mdast_official/corpus.mjs b/crates/mq-markdown/tests/mdast_official/corpus.mjs index 418695ca3..801e7b948 100644 --- a/crates/mq-markdown/tests/mdast_official/corpus.mjs +++ b/crates/mq-markdown/tests/mdast_official/corpus.mjs @@ -1,6 +1,7 @@ // Writes the inputs to compare to the file in the first argument: the hand-written corpora and the // examples of the specs, which are fetched over the network. import fs from 'node:fs' +import {generate} from './generate.mjs' const dir = new URL('.', import.meta.url) const read = (file) => JSON.parse(fs.readFileSync(new URL(file, dir), 'utf8')) @@ -125,5 +126,13 @@ for (const url of MICROMARK_TESTS) { const name = url.split('/').slice(-4, -2).join('/').replace('%40', '@') addTests(name, 'mdx', firstArguments(await fetchText(url), ['micromark'])) } +// Random documents. MDAST_RANDOM is how many of each kind, MDAST_SEED the seed. +const random = Number(process.env.MDAST_RANDOM ?? 500) +const seed = Number(process.env.MDAST_SEED ?? 1) +for (const [mode, offset] of [['markdown', 0], ['mdx', 1]]) { + generate(random, seed + offset, mode === 'mdx').forEach((input, index) => { + corpus.push({id: `random-${mode}-${seed}#${index + 1}`, mode, input}) + }) +} fs.writeFileSync(process.argv[2], JSON.stringify(corpus)) console.log(`${corpus.length} inputs`) diff --git a/crates/mq-markdown/tests/mdast_official/generate.mjs b/crates/mq-markdown/tests/mdast_official/generate.mjs new file mode 100644 index 000000000..7289887a9 --- /dev/null +++ b/crates/mq-markdown/tests/mdast_official/generate.mjs @@ -0,0 +1,120 @@ +// Random documents built from a grammar of Markdown and MDX constructs. The same seed makes the same +// documents, so the ids of the inputs stay the same and known.json can list them. + +/** A small seeded generator (mulberry32). */ +function random(seed) { + let state = seed >>> 0 + return () => { + state = (state + 0x6d2b79f5) >>> 0 + let t = state + t = Math.imul(t ^ (t >>> 15), t | 1) + t ^= t + Math.imul(t ^ (t >>> 7), t | 61) + return ((t ^ (t >>> 14)) >>> 0) / 4294967296 + } +} + +const WORDS = ['a', 'bc', 'foo', 'x1', 'Zed', 'été', '日本', 'a_b', '1', 'q'] + +export function generate(count, seed, mdx) { + const next = random(seed) + const pick = (list) => list[Math.floor(next() * list.length)] + const chance = (p) => next() < p + const word = () => pick(WORDS) + const words = (max = 4) => Array.from({length: 1 + Math.floor(next() * max)}, word).join(' ') + + const inline = (depth = 0) => { + const parts = [] + for (let i = 0, n = 1 + Math.floor(next() * 3); i < n; i++) { + const choices = [ + () => words(), + () => `*${depth < 2 ? inline(depth + 1) : word()}*`, + () => `**${depth < 2 ? inline(depth + 1) : word()}**`, + () => `\`${word()}\``, + () => `[${word()}](/${word()}${chance(0.3) ? ` "${word()}"` : ''})`, + () => `![${word()}](/${word()}.png)`, + () => `[${word()}][r${Math.floor(next() * 2)}]`, + () => `${word()} \n${word()}`, + () => `\\${pick(['*', '_', '[', '#', '`'])}${word()}`, + () => `&${pick(['amp', 'lt', 'copy'])};`, + ] + if (!mdx) { + choices.push( + () => `~~${word()}~~`, + () => `$${word()}$`, + () => `[^f${Math.floor(next() * 2)}]`, + () => `<${pick(['b', 'span'])}>${word()}</${pick(['b', 'span'])}>`, + ) + } else { + choices.push( + () => `<${pick(['B', 'a.b', 'x:y'])}${attributes()}>${depth < 2 ? inline(depth + 1) : word()}</${pick(['B', 'a.b', 'x:y'])}>`, + () => `<${pick(['B', 'C'])}${attributes()} />`, + () => `{${expression()}}`, + ) + } + parts.push(pick(choices)()) + } + return parts.join(chance(0.7) ? ' ' : '') + } + + const expression = () => pick(['a', '1 + 1', '"x"', "'}'", '`t${a}`', 'a.b', '[1, 2]', '{a: 1}', 'f(a, b)', '/* c */ a']) + const attributes = () => + Array.from({length: Math.floor(next() * 3)}, () => + pick([` a="${word()}"`, ` b='${word()}'`, ` c={${expression()}}`, ' d', ` {...${pick(['p', 'q.r'])}}`, ` e = "${word()}"`]), + ).join('') + + const block = (depth = 0) => { + const choices = [ + () => inline(), + () => `${'#'.repeat(1 + Math.floor(next() * 6))} ${inline()}`, + () => `${inline()}\n${chance(0.5) ? '===' : '---'}`, + () => pick(['***', '---', '___', '- - -']), + () => `\`\`\`${pick(['', 'js', 'rs x'])}\n${words()}\n${chance(0.3) ? '\n' : ''}\`\`\``, + () => ` ${words()}`, + () => `> ${block(depth + 1).replaceAll('\n', '\n> ')}`, + () => list(depth), + () => `[r${Math.floor(next() * 2)}]: /${word()}${chance(0.3) ? ` "${word()}"` : ''}`, + ] + if (!mdx) { + choices.push( + () => `| ${word()} | ${word()} |\n|${pick([':-', '-', '-:', ':-:'])}|${pick(['-', ':-'])}|\n| ${word()} | ${word()} |`, + () => `$$\n${word()}\n$$`, + () => `[^f${Math.floor(next() * 2)}]: ${inline()}`, + () => `<div>\n${word()}\n</div>`, + () => `<!-- ${word()} -->`, + ) + } else { + choices.push( + () => `<${pick(['A', 'B'])}${attributes()} />`, + () => `{${expression()}}`, + () => { + const name = pick(['A', 'B', 'a.b']) + return `<${name}${attributes()}>\n\n${depth < 2 ? block(depth + 1) : word()}\n\n</${name}>` + }, + () => `<${pick(['A', 'B'])}>${inline()}</${pick(['A', 'B'])}>`, + ) + } + return pick(choices)() + } + + const list = (depth) => { + const ordered = chance(0.4) + const items = Array.from({length: 1 + Math.floor(next() * 3)}, (_, i) => { + const marker = ordered ? `${i + 1}.` : pick(['-', '*', '+']) + const task = !ordered && !mdx && chance(0.2) ? pick(['[ ] ', '[x] ']) : '' + const nested = depth < 2 && chance(0.3) ? `\n${list(depth + 1).replaceAll(/^/gm, ' ')}` : '' + return `${marker} ${task}${inline()}${nested}` + }) + return items.join(chance(0.2) ? '\n\n' : '\n') + } + + const documents = [] + for (let i = 0; i < count; i++) { + const blocks = Array.from({length: 1 + Math.floor(next() * 5)}, () => block()) + // A document that starts with a line of dashes makes the official parser read what follows as text. + if (/^(---|- - -)/.test(blocks[0])) blocks[0] = '***' + if (mdx && chance(0.15)) blocks.unshift(pick(['import a from "b"', 'export const x = 1'])) + if (!mdx && chance(0.1)) blocks.unshift(`---\ntitle: ${word()}\n---`) + documents.push(blocks.join('\n\n') + '\n') + } + return documents +} diff --git a/crates/mq-markdown/tests/mdast_official/known.json b/crates/mq-markdown/tests/mdast_official/known.json index 8c3f2eac7..a2b5b02f6 100644 --- a/crates/mq-markdown/tests/mdast_official/known.json +++ b/crates/mq-markdown/tests/mdast_official/known.json @@ -73,5 +73,11 @@ "markdown-rs/misc_tabs#15": "The official packages keep trailing whitespace of the info string, as the meta of a fence.", "markdown-rs/misc_tabs#16": "The official packages keep trailing whitespace of the info string, as the meta of a fence.", "markdown-rs/html_flow#147": "The official packages do not start HTML flow on a line that continues a paragraph in a quote, which CommonMark does.", - "markdown-rs/link_reference#46": "A reference followed by an empty label is a shortcut reference here, as in CommonMark, and text in the official packages." + "markdown-rs/link_reference#46": "A reference followed by an empty label is a shortcut reference here, as in CommonMark, and text in the official packages.", + "random-markdown-1#90": "The official packages match a delimiter run that is partly used without the rule of three, which CommonMark applies to the length of the whole run.", + "random-markdown-1#200": "The official packages match a delimiter run that is partly used without the rule of three, which CommonMark applies to the length of the whole run.", + "random-markdown-1#298": "The official packages match a delimiter run that is partly used without the rule of three, which CommonMark applies to the length of the whole run.", + "random-mdx-1#102": "The official packages match a delimiter run that is partly used without the rule of three, which CommonMark applies to the length of the whole run.", + "random-markdown-1#162": "The official packages mark a list as spread when a blank line follows its last item, before a list with another marker.", + "random-markdown-1#205": "The official packages mark a list as spread when a blank line follows its last item, before a list with another marker." } \ No newline at end of file diff --git a/crates/mq-markdown/tests/mdast_official/round_trip_known.json b/crates/mq-markdown/tests/mdast_official/round_trip_known.json index d51e1c5f8..b7e219742 100644 --- a/crates/mq-markdown/tests/mdast_official/round_trip_known.json +++ b/crates/mq-markdown/tests/mdast_official/round_trip_known.json @@ -8,5 +8,6 @@ "markdown-rs/gfm_autolink_literal#31": "Text that reads as an email address or a URL once written is linked, as it cannot be written so that it stays text.", "gfm-extensions#23": "A footnote definition with blocks, such as code, is written on one line.", "markdown-rs/misc_line_ending#20": "Line endings of a lone carriage return are not kept when written.", - "micromark-extension-mdxjs-esm/3.0.0#3": "Text that starts with import or export is written without the escape that would keep it from being read as ESM." + "micromark-extension-mdxjs-esm/3.0.0#3": "Text that starts with import or export is written without the escape that would keep it from being read as ESM.", + "random-markdown-1#311": "Emphasis next to a run of asterisks that closes a strong and an image is written so that the runs fuse." } \ No newline at end of file From dcea13d59957e27361343054dcc3b6211a4c888d Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Thu, 8 Oct 2026 13:08:04 +0900 Subject: [PATCH 31/88] =?UTF-8?q?=E2=9C=A8=20feat(markdown):=20escape=20br?= =?UTF-8?q?aces=20and=20ESM-like=20text=20when=20writing=20MDX?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RenderOptions gains mdx, which from_mdx_str sets and mq sets for MDX input (-I mdx, a .mdx file, the wasm option). Text with a brace is written with the escape, and a line that starts with import or export as a character reference, so that the text reads back as text. --- crates/mq-markdown/src/markdown.rs | 6 ++- crates/mq-markdown/src/node.rs | 26 ++++++++++++- .../mdast_official/round_trip_known.json | 6 --- .../mq-markdown/tests/mdx_roundtrip_tests.rs | 20 ++++++++++ crates/mq-run/src/cli.rs | 37 ++++++++++++------- crates/mq-wasm/src/script.rs | 1 + 6 files changed, 74 insertions(+), 22 deletions(-) diff --git a/crates/mq-markdown/src/markdown.rs b/crates/mq-markdown/src/markdown.rs index 7f39ae6bd..2fdea2a5d 100644 --- a/crates/mq-markdown/src/markdown.rs +++ b/crates/mq-markdown/src/markdown.rs @@ -286,7 +286,10 @@ impl Markdown { pub fn from_mdx_str(content: &str) -> miette::Result<Self> { Ok(Self { nodes: parse_mdx_nodes(content)?, - options: RenderOptions::default(), + options: RenderOptions { + mdx: true, + ..RenderOptions::default() + }, }) } @@ -771,6 +774,7 @@ mod tests { list_style: Some(ListStyle::Star), link_title_style: TitleSurroundStyle::default(), link_url_style: UrlSurroundStyle::default(), + ..RenderOptions::default() }); let formatted = md.to_string(); diff --git a/crates/mq-markdown/src/node.rs b/crates/mq-markdown/src/node.rs index 224de01ac..1d9d2593c 100644 --- a/crates/mq-markdown/src/node.rs +++ b/crates/mq-markdown/src/node.rs @@ -152,6 +152,8 @@ pub struct RenderOptions { pub list_style: Option<ListStyle>, pub link_url_style: UrlSurroundStyle, pub link_title_style: TitleSurroundStyle, + /// Write for MDX: text that MDX would read as an expression or as ESM is escaped. + pub mdx: bool, } #[derive(Debug, Clone, Default, PartialEq)] @@ -1508,7 +1510,7 @@ impl Node { // values like JSON output or attr lookups must stay untouched. Self::Text(Text { value, position }) => { if position.is_some() { - escape_text(value.clone()) + escape_text_with(value.clone(), options.mdx) } else { value.clone() } @@ -3763,6 +3765,16 @@ fn is_email_domain(rest: &str) -> bool { /// syntax on re-parse. `# - + > =` only unsafe at line start, `. )` only after leading digits; /// a leading `\t` and runs of 2+ `\n` (only possible via decoded entities) use `&#N;` instead. fn escape_text(value: String) -> String { + escape_text_with(value, false) +} + +/// Whether `text` starts with the `import ` or `export ` that makes a block ESM in MDX. +fn starts_esm(text: &str) -> bool { + text.starts_with("import ") || text.starts_with("export ") +} + +/// `escape_text`, and for MDX also the braces and the `import` or `export` that starts a line. +fn escape_text_with(value: String, mdx: bool) -> String { // Skip the copy when nothing needs escaping; byte scan avoids UTF-8 decoding. let needs_escaping = value.bytes().any(|b| { matches!( @@ -3790,7 +3802,8 @@ fn escape_text(value: String) -> String { | b'@' ) }); - if !needs_escaping { + let mdx_special = mdx && (value.contains('{') || value.contains("import ") || value.contains("export ")); + if !needs_escaping && !mdx_special { return value; } @@ -3819,6 +3832,15 @@ fn escape_text(value: String) -> String { continue; } '\\' | '`' | '*' | '_' | '[' | ']' | '|' | '~' | '$' | '<' => result.push('\\'), + '{' if mdx => result.push('\\'), + // The first letter as a character reference keeps the line from starting an ESM block. + 'i' | 'e' if mdx && at_line_start && starts_esm(&value[value.len() - rest.len() - 1..]) => { + result.push_str(if c == 'i' { "&#105;" } else { "&#101;" }); + at_line_start = false; + leading_digits = false; + previous = Some(c); + continue; + } '#' | '-' | '+' | '>' | '=' if at_line_start => result.push('\\'), '.' | ')' if leading_digits => result.push('\\'), '&' if starts_reference(rest) => result.push('\\'), diff --git a/crates/mq-markdown/tests/mdast_official/round_trip_known.json b/crates/mq-markdown/tests/mdast_official/round_trip_known.json index b7e219742..59fd9122f 100644 --- a/crates/mq-markdown/tests/mdast_official/round_trip_known.json +++ b/crates/mq-markdown/tests/mdast_official/round_trip_known.json @@ -1,13 +1,7 @@ { - "mdx: import a from \"b\"\n": "Text that starts with import or export is written without the escape that would keep it from being read as ESM.", - "markdown-rs/mdx_esm#7": "Text that starts with import or export is written without the escape that would keep it from being read as ESM.", - "mdx:a\\{b}": "A brace in MDX text is written without the escape, as the options do not tell Markdown from MDX.", - "mdx:a &#123;b}": "A brace in MDX text is written without the escape, as the options do not tell Markdown from MDX.", - "markdown-rs/mdx_jsx_text#113": "A brace in MDX text is written without the escape, as the options do not tell Markdown from MDX.", "gfm-extensions#19": "Text that reads as an email address or a URL once written is linked, as it cannot be written so that it stays text.", "markdown-rs/gfm_autolink_literal#31": "Text that reads as an email address or a URL once written is linked, as it cannot be written so that it stays text.", "gfm-extensions#23": "A footnote definition with blocks, such as code, is written on one line.", "markdown-rs/misc_line_ending#20": "Line endings of a lone carriage return are not kept when written.", - "micromark-extension-mdxjs-esm/3.0.0#3": "Text that starts with import or export is written without the escape that would keep it from being read as ESM.", "random-markdown-1#311": "Emphasis next to a run of asterisks that closes a strong and an image is written so that the runs fuse." } \ No newline at end of file diff --git a/crates/mq-markdown/tests/mdx_roundtrip_tests.rs b/crates/mq-markdown/tests/mdx_roundtrip_tests.rs index 574b24810..c16f61ddb 100644 --- a/crates/mq-markdown/tests/mdx_roundtrip_tests.rs +++ b/crates/mq-markdown/tests/mdx_roundtrip_tests.rs @@ -63,3 +63,23 @@ fn expressions_keep_their_indentation_when_written_and_read_again(#[case] input: assert_eq!(describe_all(&first), describe_all(&second), "{first}"); assert_eq!(second.to_string(), third.to_string()); } + +#[rstest] +#[case::brace("a\\{b}", "a\\{b}")] +#[case::brace_from_a_reference("a &#123;b}", "a \\{b}")] +#[case::esm_looking_text(" import a from \"b\"", "&#105;mport a from \"b\"")] +#[case::export_looking_text(" export default c", "&#101;xport default c")] +#[case::text_that_only_starts_like_esm("importx a", "importx a")] +fn text_is_escaped_for_mdx(#[case] input: &str, #[case] expected: &str) { + let markdown = Markdown::from_mdx_str(input).unwrap(); + let written = markdown.to_string(); + assert_eq!(written.trim_end(), expected); + let reparsed = Markdown::from_mdx_str(&written).unwrap(); + assert_eq!(describe_all(&markdown), describe_all(&reparsed), "{written}"); +} + +#[test] +fn text_is_not_escaped_for_markdown() { + let markdown = Markdown::from_markdown_str("a {b} and\n\nimport a from \"b\"\n").unwrap(); + assert_eq!(markdown.to_string(), "a {b} and\n\nimport a from \"b\"\n"); +} diff --git a/crates/mq-run/src/cli.rs b/crates/mq-run/src/cli.rs index b278d0969..be54a11d6 100644 --- a/crates/mq-run/src/cli.rs +++ b/crates/mq-run/src/cli.rs @@ -2055,6 +2055,16 @@ impl Cli { Ok(()) } + /// Whether the input of `file` is read as MDX, so that what is written for it is escaped as MDX. + fn is_mdx_input(&self, file: &Option<PathBuf>) -> bool { + match self.explicit_input_format() { + Some(format) => matches!(format, InputFormat::Mdx), + None => file + .as_ref() + .is_some_and(|file| matches!(InputFormat::from_path(file), InputFormat::Mdx)), + } + } + fn explicit_input_format(&self) -> Option<InputFormat> { self.input .program @@ -2226,7 +2236,7 @@ impl Cli { }; if self.quiet { - return self.print(runtime_values); + return self.print(runtime_values, self.is_mdx_input(file)); } if let Some(input) = grep_input { @@ -2240,7 +2250,7 @@ impl Cli { )?; grep::print_grep(runtime_values, &input, file, handle, before, after) } else { - self.print(runtime_values) + self.print(runtime_values, self.is_mdx_input(file)) } } @@ -2632,7 +2642,7 @@ impl Cli { vec![mq_lang::RuntimeValue::String(Shared::new("".to_string()))].into_iter(), ) .map_err(|e| *e)?; - self.print(separator)?; + self.print(separator, false)?; } self.emit_results(runtime_values, grep_input, file) @@ -3044,7 +3054,7 @@ impl Cli { value } - fn build_markdown(&self, runtime_values: &[mq_lang::RuntimeValue]) -> mq_markdown::Markdown { + fn build_markdown(&self, runtime_values: &[mq_lang::RuntimeValue], mdx: bool) -> mq_markdown::Markdown { let mut markdown = mq_markdown::Markdown::new(runtime_values.iter().flat_map(Self::runtime_value_to_nodes).collect()); markdown.set_options(mq_markdown::RenderOptions { @@ -3062,6 +3072,7 @@ impl Cli { LinkUrlStyle::None => mq_markdown::UrlSurroundStyle::None, LinkUrlStyle::Angle => mq_markdown::UrlSurroundStyle::Angle, }, + mdx, }); markdown } @@ -3069,7 +3080,7 @@ impl Cli { /// Renders `runtime_values` to a byte buffer without writing anywhere. /// `emit_diff` always passes `colorize: false` — it needs plain text to diff /// against the uncolored original, and colors the diff lines itself. - fn render(&self, runtime_values: &[mq_lang::RuntimeValue], colorize: bool) -> miette::Result<Vec<u8>> { + fn render(&self, runtime_values: &[mq_lang::RuntimeValue], colorize: bool, mdx: bool) -> miette::Result<Vec<u8>> { let mut buf = Vec::new(); match self.resolved_output_format() { @@ -3095,20 +3106,20 @@ impl Cli { } } OutputFormat::Html => { - let markdown = self.build_markdown(runtime_values); + let markdown = self.build_markdown(runtime_values, mdx); buf.extend_from_slice(markdown.to_html().as_bytes()); } OutputFormat::Text => { - let markdown = self.build_markdown(runtime_values); + let markdown = self.build_markdown(runtime_values, mdx); buf.extend_from_slice(markdown.to_text().as_bytes()); } OutputFormat::Markdown if colorize => { - let markdown = self.build_markdown(runtime_values); + let markdown = self.build_markdown(runtime_values, mdx); let theme = mq_markdown::ColorTheme::from_env(); buf.extend_from_slice(markdown.to_colored_string_with_theme(&theme).as_bytes()); } OutputFormat::Markdown => { - let markdown = self.build_markdown(runtime_values); + let markdown = self.build_markdown(runtime_values, mdx); buf.extend_from_slice(markdown.to_string().as_bytes()); } OutputFormat::Table => { @@ -3117,7 +3128,7 @@ impl Cli { buf.extend_from_slice(format!("{}\n", table).as_bytes()); } OutputFormat::Grep => { - let markdown = self.build_markdown(runtime_values); + let markdown = self.build_markdown(runtime_values, mdx); buf.extend_from_slice(markdown.to_string().as_bytes()); } OutputFormat::Gron => { @@ -3156,7 +3167,7 @@ impl Cli { Ok(buf) } - fn print(&self, runtime_values: mq_lang::RuntimeValues) -> miette::Result<()> { + fn print(&self, runtime_values: mq_lang::RuntimeValues, mdx: bool) -> miette::Result<()> { let stripped_values: Option<Vec<mq_lang::RuntimeValue>> = self.output.no_position.then(|| { runtime_values .values() @@ -3185,7 +3196,7 @@ impl Cli { )?; let colorize = self.output.color_output && !Self::is_no_color(); - let buf = self.render(runtime_values, colorize)?; + let buf = self.render(runtime_values, colorize, mdx)?; Self::write_ignore_pipe(&mut handle, &buf)?; handle.finish()?; @@ -3200,7 +3211,7 @@ impl Cli { content: &ContentData, ) -> miette::Result<()> { let original = content.as_str().unwrap_or(""); - let rendered = self.render(runtime_values.values(), false)?; + let rendered = self.render(runtime_values.values(), false, self.is_mdx_input(file))?; let rendered = String::from_utf8_lossy(&rendered); if original != rendered { diff --git a/crates/mq-wasm/src/script.rs b/crates/mq-wasm/src/script.rs index da956962d..20d785e73 100644 --- a/crates/mq-wasm/src/script.rs +++ b/crates/mq-wasm/src/script.rs @@ -708,6 +708,7 @@ pub async fn run(code: &str, content: &str, options: JsValue) -> Result<String, UrlSurroundStyle::None => mq_markdown::UrlSurroundStyle::None, }) .unwrap_or_default(), + mdx: matches!(options.input_format, Some(InputFormat::Mdx)), }); markdown.to_string() }) From faa94669d2c0ebf796591cda3a7426acada75d34 Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Thu, 8 Oct 2026 13:08:10 +0900 Subject: [PATCH 32/88] =?UTF-8?q?=F0=9F=90=9B=20fix(markdown):=20match=20t?= =?UTF-8?q?he=20official=20packages=20on=20tables,=20task=20items,=20ESM?= =?UTF-8?q?=20and=20expressions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - A header of a lone pipe has no cells, so it does not start a table. - A task marker at the end of a line counts when the item goes on in the next line. - Character references of any length in a JSX attribute are decoded, and replaced with U+FFFD when out of range. - The tab at the start of a line of an expression is dropped in columns, and what is left stays as spaces. - An ESM block goes on past a blank line while a bracket is open. --- crates/mq-markdown/src/parser/block.rs | 20 +++++-- crates/mq-markdown/src/parser/block/rule.rs | 40 +++++++++++-- .../mq-markdown/src/parser/inline/entity.rs | 12 +++- crates/mq-markdown/src/parser/mdx.rs | 21 +++++-- crates/mq-markdown/src/parser/table.rs | 4 ++ .../tests/mdast_official/known.json | 13 ++--- .../tests/parser_structure_tests.rs | 58 +++++++++++++++++++ 7 files changed, 143 insertions(+), 25 deletions(-) diff --git a/crates/mq-markdown/src/parser/block.rs b/crates/mq-markdown/src/parser/block.rs index 1e0c87966..0da208cd0 100644 --- a/crates/mq-markdown/src/parser/block.rs +++ b/crates/mq-markdown/src/parser/block.rs @@ -608,7 +608,10 @@ fn list_item( index += 1; } - let checkbox = inner.iter().find(|line| !line.is_blank()).and_then(task_checkbox); + let checkbox = inner + .iter() + .position(|line| !line.is_blank()) + .and_then(|index| task_checkbox(&inner[index], inner.get(index + 1))); // Trailing blank lines belong to whatever follows the item, but still extend its position. let reaches_end = index == lines.len(); @@ -651,13 +654,20 @@ fn list_item( /// Recognises a GFM task marker (`[ ] ` or `[x] `) at the start of an item's content, returning /// the checked state and the bytes to skip. -fn task_checkbox(line: &Line<'_>) -> Option<bool> { +fn task_checkbox(line: &Line<'_>, next: Option<&Line<'_>>) -> Option<bool> { if line.mdx { return None; } - match line.text.as_bytes().get(..4)? { - [b'[', b' ', b']', b' ' | b'\t'] => Some(false), - [b'[', b'x' | b'X', b']', b' ' | b'\t'] => Some(true), + let bytes = line.text.as_bytes(); + let checked = match bytes.get(..3)? { + [b'[', b' ', b']'] => false, + [b'[', b'x' | b'X', b']'] => true, + _ => return None, + }; + // Whitespace follows the marker, which a line ending is when the item goes on in the next line. + match bytes.get(3) { + Some(b' ' | b'\t') => Some(checked), + None if next.is_some_and(|next| !next.is_blank()) => Some(checked), _ => None, } } diff --git a/crates/mq-markdown/src/parser/block/rule.rs b/crates/mq-markdown/src/parser/block/rule.rs index 909c3e7e0..ad93678b6 100644 --- a/crates/mq-markdown/src/parser/block/rule.rs +++ b/crates/mq-markdown/src/parser/block/rule.rs @@ -229,6 +229,30 @@ impl BlockRule for Table { } } +/// How many brackets are open after `text`, given the `depth` before it. Strings and a comment to the end +/// of the line are skipped. +fn open_brackets(text: &str, mut depth: i32) -> i32 { + let mut chars = text.chars().peekable(); + while let Some(char) = chars.next() { + match char { + '{' | '(' | '[' => depth += 1, + '}' | ')' | ']' => depth -= 1, + '\'' | '"' | '`' => { + while let Some(next) = chars.next() { + if next == '\\' { + chars.next(); + } else if next == char { + break; + } + } + } + '/' if chars.peek() == Some(&'/') => break, + _ => {} + } + } + depth +} + /// MDX `import` and `export`, which go on to the next blank line. pub(super) struct Esm; @@ -242,11 +266,17 @@ impl BlockRule for Esm { { return None; } - // Without a JavaScript parser the module is taken as it is. - let end = cx.lines[cx.index..] - .iter() - .position(Line::is_blank) - .map_or(cx.lines.len(), |offset| cx.index + offset); + // Without a JavaScript parser the module goes on to the next blank line, or past it when a bracket + // is still open. + let mut depth = 0i32; + let mut end = cx.lines.len(); + for (offset, line) in cx.lines[cx.index..].iter().enumerate() { + if line.is_blank() && depth <= 0 { + end = cx.index + offset; + break; + } + depth = open_brackets(line.text, depth); + } let parts = cx.lines[cx.index..end] .iter() .map(|line| (line.text, line.eol)) diff --git a/crates/mq-markdown/src/parser/inline/entity.rs b/crates/mq-markdown/src/parser/inline/entity.rs index 2badf69ff..4b5552f43 100644 --- a/crates/mq-markdown/src/parser/inline/entity.rs +++ b/crates/mq-markdown/src/parser/inline/entity.rs @@ -4,6 +4,12 @@ use super::entities::ENTITIES; /// Decodes the character reference that starts at `pos`, returning the offset after it and its value. pub(super) fn decode(src: &str, pos: usize) -> Option<(usize, String)> { + decode_with(src, pos, true) +} + +/// `decode`, where a numeric reference of any length is one when `limited` is not set, and is +/// replaced with U+FFFD when it is out of range. +fn decode_with(src: &str, pos: usize, limited: bool) -> Option<(usize, String)> { let bytes = src.as_bytes(); debug_assert_eq!(bytes[pos], b'&'); let rest = &bytes[pos + 1..]; @@ -13,6 +19,7 @@ pub(super) fn decode(src: &str, pos: usize) -> Option<(usize, String)> { Some(b'x' | b'X') => (16, 2, 6), _ => (10, 1, 7), }; + let max = if limited { max } else { usize::MAX }; let digits = rest[digits_from..] .iter() .take_while(|b| (radix == 16 && b.is_ascii_hexdigit()) || b.is_ascii_digit()) @@ -20,7 +27,8 @@ pub(super) fn decode(src: &str, pos: usize) -> Option<(usize, String)> { if digits == 0 || digits > max || rest.get(digits_from + digits) != Some(&b';') { return None; } - let number = u32::from_str_radix(&src[pos + 1 + digits_from..pos + 1 + digits_from + digits], radix).ok()?; + let number = + u32::from_str_radix(&src[pos + 1 + digits_from..pos + 1 + digits_from + digits], radix).unwrap_or(u32::MAX); // Control characters other than whitespace are not allowed, nor are surrogates and // out of range numbers. let char = match char::from_u32(number) { @@ -84,7 +92,7 @@ pub(in crate::parser) fn decode_references(value: &str) -> String { while index < bytes.len() { if bytes[index] == b'&' - && let Some((end, decoded)) = decode(value, index) + && let Some((end, decoded)) = decode_with(value, index, false) { result.push_str(&value[from..index]); result.push_str(&decoded); diff --git a/crates/mq-markdown/src/parser/mdx.rs b/crates/mq-markdown/src/parser/mdx.rs index ace22781d..09e09356a 100644 --- a/crates/mq-markdown/src/parser/mdx.rs +++ b/crates/mq-markdown/src/parser/mdx.rs @@ -260,7 +260,7 @@ impl Cursor<'_> { } impl Cursor<'_> { - /// The value of an expression: its content, without up to two whitespace characters at the start + /// The value of an expression: its content, without up to two columns of whitespace at the start /// of each line after the first. fn expression_value(&self, start: usize, end: usize) -> SmolStr { let content = &self.src[start..end]; @@ -268,15 +268,28 @@ impl Cursor<'_> { return SmolStr::new(content); } let mut value = String::with_capacity(content.len()); - // How many more whitespace characters may be dropped at the start of the current line. - let mut droppable = 0; + // How many more columns of whitespace may be dropped at the start of the current line, and the + // column that the line is at. + let mut droppable = 0usize; + let mut column = 0usize; for char in content.chars() { match char { '\n' | '\r' => { droppable = 2; + column = 0; value.push(char); } - ' ' | '\t' if droppable > 0 => droppable -= 1, + ' ' if droppable > 0 => { + droppable -= 1; + column += 1; + } + // What is left of a tab that is dropped in part stays as spaces. + '\t' if droppable > 0 => { + let width = 4 - column % 4; + value.extend(std::iter::repeat_n(' ', width.saturating_sub(droppable))); + droppable = droppable.saturating_sub(width); + column += width; + } _ => { droppable = 0; value.push(char); diff --git a/crates/mq-markdown/src/parser/table.rs b/crates/mq-markdown/src/parser/table.rs index 83f2983a5..abf19bad6 100644 --- a/crates/mq-markdown/src/parser/table.rs +++ b/crates/mq-markdown/src/parser/table.rs @@ -112,6 +112,10 @@ fn header<'a>( return None; } + // A lone pipe is both the leading and the trailing one of a row, which then has no cells. + if head.text.trim_matches([' ', '\t']) == "|" { + return None; + } let cells = split_row(head); let aligns = split_row(delimiter) .iter() diff --git a/crates/mq-markdown/tests/mdast_official/known.json b/crates/mq-markdown/tests/mdast_official/known.json index a2b5b02f6..e5c3ed773 100644 --- a/crates/mq-markdown/tests/mdast_official/known.json +++ b/crates/mq-markdown/tests/mdast_official/known.json @@ -51,10 +51,6 @@ "micromark-extension-mdx-expression/micromark-extension-mdx-expression@3.0.1#9": "Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted.", "micromark-extension-mdxjs-esm/3.0.0#11": "Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted.", "micromark-extension-mdxjs-esm/3.0.0#12": "Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted.", - "markdown-rs/mdx_esm#16": "An ESM block continues over blank lines inside braces only when JavaScript says the statement is incomplete.", - "micromark-extension-mdxjs-esm/3.0.0#10": "An ESM block continues over blank lines inside braces only when JavaScript says the statement is incomplete.", - "markdown-rs/mdx_expression_flow#17": "A tab at the start of a line of an expression is dropped as one character of the indent.", - "markdown-rs/mdx_jsx_text#17": "An invalid numeric character reference in a JSX attribute is kept as written, not replaced with U+FFFD.", "markdown-rs/code_fenced#34": "A null character is kept in values and replaced with U+FFFD only in HTML, so that positions stay the ones of the source.", "markdown-rs/hard_break_trailing#11": "A null character is kept in values and replaced with U+FFFD only in HTML, so that positions stay the ones of the source.", "markdown-rs/hard_break_trailing#12": "A null character is kept in values and replaced with U+FFFD only in HTML, so that positions stay the ones of the source.", @@ -66,12 +62,11 @@ "markdown-rs/gfm_autolink_literal#33": "GFM autolink literals: the text of a `mailto:` or `xmpp:` link includes the scheme here, as in the GFM spec.", "markdown-rs/code_fenced#36": "Blank lines at the end of an unclosed fence in a quote are one fewer in the value of the official packages.", "markdown-rs/math_flow#35": "Blank lines at the end of an unclosed fence in a quote are one fewer in the value of the official packages.", - "markdown-rs/code_indented#23": "The lines of inline HTML lose all their indent here, and all but a space in the official packages.", + "markdown-rs/code_indented#23": "The lines of inline HTML lose all their indent here, as in CommonMark, and all but a space in the official packages.", "markdown-rs/gfm_strikethrough#10": "A strikethrough inside `*...*` that CommonMark does not let open: emphasis does not open next to `~` here.", - "markdown-rs/gfm_table#19": "A table whose header row is empty is a table here, and text in the official packages.", - "markdown-rs/gfm_task_list_item#6": "A task list item whose checkbox is followed by a line ending is not a task here, and the whitespace after a checkbox is dropped.", - "markdown-rs/misc_tabs#15": "The official packages keep trailing whitespace of the info string, as the meta of a fence.", - "markdown-rs/misc_tabs#16": "The official packages keep trailing whitespace of the info string, as the meta of a fence.", + "markdown-rs/gfm_task_list_item#6": "The text after a checkbox that ends its line starts with a line ending in the official packages, and the whitespace after a checkbox is dropped here.", + "markdown-rs/misc_tabs#15": "The info string of a fence is trimmed here, as in CommonMark, and the official packages keep the trailing whitespace as the meta.", + "markdown-rs/misc_tabs#16": "The info string of a fence is trimmed here, as in CommonMark, and the official packages keep the trailing whitespace as the meta.", "markdown-rs/html_flow#147": "The official packages do not start HTML flow on a line that continues a paragraph in a quote, which CommonMark does.", "markdown-rs/link_reference#46": "A reference followed by an empty label is a shortcut reference here, as in CommonMark, and text in the official packages.", "random-markdown-1#90": "The official packages match a delimiter run that is partly used without the rule of three, which CommonMark applies to the length of the whole run.", diff --git a/crates/mq-markdown/tests/parser_structure_tests.rs b/crates/mq-markdown/tests/parser_structure_tests.rs index 136297bb4..ff988eaee 100644 --- a/crates/mq-markdown/tests/parser_structure_tests.rs +++ b/crates/mq-markdown/tests/parser_structure_tests.rs @@ -434,6 +434,64 @@ fn writing_keeps_what_is_read_back(#[case] input: &str, #[case] expected: &str) ); } +#[rstest] +#[case::lone_pipe_header("|\n|-|\n|d|\n", false)] +#[case::empty_cell_header("||\n|-|\n|d|\n", true)] +#[case::space_cell_header("| |\n|-|\n|d|\n", true)] +fn a_header_of_a_lone_pipe_has_no_cells(#[case] input: &str, #[case] table: bool) { + let nodes = Markdown::from_markdown_str(input).unwrap().nodes; + assert_eq!( + nodes.iter().any(|node| matches!(node, Node::TableCell(_))), + table, + "{nodes:?}" + ); +} + +#[rstest] +#[case::next_line("- [ ]\n a\n", Some(false))] +#[case::checked_next_line("* [x]\r\n a\n", Some(true))] +#[case::alone("- [ ]\n", None)] +#[case::before_a_blank_line("- [ ]\n\npara\n", None)] +#[case::before_an_item("- [x]\n- b\n", None)] +#[case::with_a_space("- [x] a\n", Some(true))] +fn a_task_marker_is_followed_by_whitespace(#[case] input: &str, #[case] checked: Option<bool>) { + let nodes = Markdown::from_markdown_str(input).unwrap().nodes; + let Node::List(list) = &nodes[0] else { + panic!("{nodes:?}") + }; + assert_eq!(list.checked, checked, "{nodes:?}"); +} + +#[test] +fn mdx_attribute_references_of_any_length_are_decoded() { + let markdown = Markdown::from_mdx_str("<a b='&#987654321; &#x110000; &#65;' />").unwrap(); + assert_eq!(markdown.to_string().trim_end(), "<a b=\"\u{FFFD} \u{FFFD} A\" />"); +} + +#[test] +fn mdx_expression_keeps_what_is_left_of_a_tab_at_the_start_of_a_line() { + let nodes = Markdown::from_mdx_str("{`\n\t`}").unwrap().nodes; + assert!( + matches!(&nodes[..], [Node::MdxFlowExpression(e)] if e.value == "`\n `"), + "{nodes:?}" + ); +} + +#[rstest] +#[case::bracket_open_over_blank_lines("export {\n\n a\n\n} from 'b'\n\nc", 2)] +#[case::paren_open("import (\n\n'a'\n\n)\n\nb", 2)] +#[case::string_with_a_bracket("import a from \"{\"\n\nb", 2)] +#[case::closed("import a from 'b'\n\nc", 2)] +fn mdx_esm_goes_on_while_a_bracket_is_open(#[case] input: &str, #[case] blocks: usize) { + let nodes = Markdown::from_mdx_str(input).unwrap().nodes; + assert_eq!(nodes.len(), blocks, "{nodes:?}"); + assert!(matches!(nodes[0], Node::MdxJsEsm(_)), "{nodes:?}"); + assert!( + matches!(&nodes[1], Node::Text(t) if t.value.trim().len() == 1), + "{nodes:?}" + ); +} + /// References keep the notation they were written in, as long as it resolves to the same definition. #[rstest] #[case::full_keeps_case_and_spaces("[x][Foo Bar]\n\n[foo bar]: /u\n", "[x][Foo Bar]")] From c5d184a8c256567b4fb9d6ffc92207c94a50bfae Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Thu, 8 Oct 2026 13:08:16 +0900 Subject: [PATCH 33/88] =?UTF-8?q?=F0=9F=90=9B=20fix(markdown):=20open=20em?= =?UTF-8?q?phasis=20next=20to=20strikethrough=20and=20keep=20the=20space?= =?UTF-8?q?=20after=20a=20task=20marker?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - A tilde next to a run of asterisks or underscores lets the run open or close, so a*~b~*c is emphasis around strikethrough. - A task marker takes one space, tab or line ending, so the text keeps the rest of the whitespace. --- crates/mq-markdown/src/parser/block.rs | 2 +- crates/mq-markdown/src/parser/inline.rs | 2 +- .../mq-markdown/src/parser/inline/emphasis.rs | 28 ++++++++++++----- crates/mq-markdown/src/parser/tree.rs | 20 ++++++++++--- .../tests/mdast_official/known.json | 3 +- .../mdast_official/round_trip_known.json | 3 +- .../tests/parser_structure_tests.rs | 30 +++++++++++++++++++ 7 files changed, 72 insertions(+), 16 deletions(-) diff --git a/crates/mq-markdown/src/parser/block.rs b/crates/mq-markdown/src/parser/block.rs index 0da208cd0..934fbc92b 100644 --- a/crates/mq-markdown/src/parser/block.rs +++ b/crates/mq-markdown/src/parser/block.rs @@ -685,7 +685,7 @@ fn remove_task_marker(children: &mut Vec<Block>) -> bool { if !matches!(source.text.as_bytes().get(..3), Some([b'[', b' ' | b'x' | b'X', b']'])) { return false; } - source.remove_prefix(3); + source.remove_prefix_and_one(3); if source.text.is_empty() { children.remove(0); } diff --git a/crates/mq-markdown/src/parser/inline.rs b/crates/mq-markdown/src/parser/inline.rs index a1a7277fe..19c7c2159 100644 --- a/crates/mq-markdown/src/parser/inline.rs +++ b/crates/mq-markdown/src/parser/inline.rs @@ -628,7 +628,7 @@ impl Scanner<'_> { let before = src[..start].chars().next_back(); let after = src[end..].chars().next(); - let (can_open, can_close) = emphasis::flanking(ch, before, after); + let (can_open, can_close) = emphasis::flanking(ch, before, after, !self.context.references.mdx); if !can_open && !can_close { self.pos = end; return; diff --git a/crates/mq-markdown/src/parser/inline/emphasis.rs b/crates/mq-markdown/src/parser/inline/emphasis.rs index 96e18af88..9f3ad455d 100644 --- a/crates/mq-markdown/src/parser/inline/emphasis.rs +++ b/crates/mq-markdown/src/parser/inline/emphasis.rs @@ -25,16 +25,30 @@ fn kind(char: Option<char>) -> Kind { /// Whether a run of `ch` between `before` and `after` can open and can close, by the rules of /// `CommonMark` for left- and right-flanking runs, which `~` follows too. -pub(super) fn flanking(ch: u8, before: Option<char>, after: Option<char>) -> (bool, bool) { - let (before_kind, after_kind) = (kind(before), kind(after)); - - let open = after_kind == Kind::Other || (after_kind == Kind::Punctuation && before_kind != Kind::Other); - let close = before_kind == Kind::Other || (before_kind == Kind::Punctuation && after_kind != Kind::Other); +/// +/// With `strikethrough`, a `~` next to a run of `*` or `_` counts as what lets the run open or close, so +/// that `a*~b~*c` is emphasis around strikethrough, and `$**~~b~~` still closes. +pub(super) fn flanking(ch: u8, before: Option<char>, after: Option<char>, strikethrough: bool) -> (bool, bool) { + let tilde = |char: Option<char>| strikethrough && ch != b'~' && char == Some('~'); + // For opening a `~` after the run is a letter and a `~` before it is punctuation, and for closing the + // other way round. + let (before_for_open, after_for_open) = ( + if tilde(before) { Kind::Punctuation } else { kind(before) }, + if tilde(after) { Kind::Other } else { kind(after) }, + ); + let (before_for_close, after_for_close) = ( + if tilde(before) { Kind::Other } else { kind(before) }, + if tilde(after) { Kind::Punctuation } else { kind(after) }, + ); + + let open = after_for_open == Kind::Other || (after_for_open == Kind::Punctuation && before_for_open != Kind::Other); + let close = + before_for_close == Kind::Other || (before_for_close == Kind::Punctuation && after_for_close != Kind::Other); if ch == b'_' { ( - open && (before_kind != Kind::Other || !close), - close && (after_kind != Kind::Other || !open), + open && (before_for_open != Kind::Other || !close), + close && (after_for_close != Kind::Other || !open), ) } else { (open, close) diff --git a/crates/mq-markdown/src/parser/tree.rs b/crates/mq-markdown/src/parser/tree.rs index 79b878831..a80a922f4 100644 --- a/crates/mq-markdown/src/parser/tree.rs +++ b/crates/mq-markdown/src/parser/tree.rs @@ -40,11 +40,23 @@ impl InlineSource { } } - /// Removes the first `marker` bytes and the whitespace that follows them, line endings included. - pub(super) fn remove_prefix(&mut self, marker: usize) { + /// Removes the first `marker` bytes and the one space, tab or line ending that follows them, and the + /// line ending after a space or a tab. + pub(super) fn remove_prefix_and_one(&mut self, marker: usize) { + let eol = |rest: &str| { + if rest.starts_with("\r\n") { + 2 + } else { + usize::from(rest.starts_with(['\n', '\r'])) + } + }; let rest = &self.text[marker..]; - let end = self.text.len() - rest.trim_start_matches([' ', '\t', '\n', '\r']).len(); - self.drain_to(end); + let one = if rest.starts_with([' ', '\t']) { + 1 + eol(&rest[1..]) + } else { + eol(rest) + }; + self.drain_to(marker + one); } /// Removes the first line with its line ending and returns it without the line ending. diff --git a/crates/mq-markdown/tests/mdast_official/known.json b/crates/mq-markdown/tests/mdast_official/known.json index e5c3ed773..6c4f9654f 100644 --- a/crates/mq-markdown/tests/mdast_official/known.json +++ b/crates/mq-markdown/tests/mdast_official/known.json @@ -63,8 +63,7 @@ "markdown-rs/code_fenced#36": "Blank lines at the end of an unclosed fence in a quote are one fewer in the value of the official packages.", "markdown-rs/math_flow#35": "Blank lines at the end of an unclosed fence in a quote are one fewer in the value of the official packages.", "markdown-rs/code_indented#23": "The lines of inline HTML lose all their indent here, as in CommonMark, and all but a space in the official packages.", - "markdown-rs/gfm_strikethrough#10": "A strikethrough inside `*...*` that CommonMark does not let open: emphasis does not open next to `~` here.", - "markdown-rs/gfm_task_list_item#6": "The text after a checkbox that ends its line starts with a line ending in the official packages, and the whitespace after a checkbox is dropped here.", + "markdown-rs/gfm_task_list_item#6": "The text after a checkbox that ends its line starts with a line ending in the JavaScript packages, and not in markdown-rs.", "markdown-rs/misc_tabs#15": "The info string of a fence is trimmed here, as in CommonMark, and the official packages keep the trailing whitespace as the meta.", "markdown-rs/misc_tabs#16": "The info string of a fence is trimmed here, as in CommonMark, and the official packages keep the trailing whitespace as the meta.", "markdown-rs/html_flow#147": "The official packages do not start HTML flow on a line that continues a paragraph in a quote, which CommonMark does.", diff --git a/crates/mq-markdown/tests/mdast_official/round_trip_known.json b/crates/mq-markdown/tests/mdast_official/round_trip_known.json index 59fd9122f..4b52db244 100644 --- a/crates/mq-markdown/tests/mdast_official/round_trip_known.json +++ b/crates/mq-markdown/tests/mdast_official/round_trip_known.json @@ -3,5 +3,6 @@ "markdown-rs/gfm_autolink_literal#31": "Text that reads as an email address or a URL once written is linked, as it cannot be written so that it stays text.", "gfm-extensions#23": "A footnote definition with blocks, such as code, is written on one line.", "markdown-rs/misc_line_ending#20": "Line endings of a lone carriage return are not kept when written.", - "random-markdown-1#311": "Emphasis next to a run of asterisks that closes a strong and an image is written so that the runs fuse." + "random-markdown-1#311": "Emphasis next to a run of asterisks that closes a strong and an image is written so that the runs fuse.", + "markdown-rs/gfm_task_list_item#6": "Line endings of a lone carriage return are not kept when written." } \ No newline at end of file diff --git a/crates/mq-markdown/tests/parser_structure_tests.rs b/crates/mq-markdown/tests/parser_structure_tests.rs index ff988eaee..705df23db 100644 --- a/crates/mq-markdown/tests/parser_structure_tests.rs +++ b/crates/mq-markdown/tests/parser_structure_tests.rs @@ -492,6 +492,36 @@ fn mdx_esm_goes_on_while_a_bracket_is_open(#[case] input: &str, #[case] blocks: ); } +#[rstest] +#[case::strikethrough_in_emphasis("a*~b~*c\n", "<p>a<em><del>b</del></em>c</p>\n")] +#[case::strikethrough_not_next_to_a_word("a ~*b*~ c\n", "<p>a <del><em>b</em></del> c</p>\n")] +#[case::tildes_in_a_word("a~*b*~c\n", "<p>a~<em>b</em>~c</p>\n")] +#[case::strong_around_strikethrough("a**~b~**c\n", "<p>a<strong><del>b</del></strong>c</p>\n")] +#[case::underscore_next_to_tildes("a_~b~_c\n", "<p>a_<del>b</del>_c</p>\n")] +#[case::punctuation_that_is_not_a_tilde("a*.b.*c\n", "<p>a*.b.*c</p>\n")] +fn emphasis_next_to_strikethrough(#[case] input: &str, #[case] expected: &str) { + assert_eq!(mq_markdown::to_html(input), expected); +} + +#[rstest] +#[case::more_spaces("- [ ] b\n", " b")] +#[case::one_space("- [ ] b\n", "b")] +#[case::tab("- [x]\tb\n", "b")] +#[case::next_line("- [ ]\n b\n", "b")] +#[case::space_and_next_line("- [ ] \n b\n", "b")] +#[case::crlf("* [x]\r\n a\n", "a")] +fn a_task_marker_takes_one_whitespace_character(#[case] input: &str, #[case] expected: &str) { + let nodes = Markdown::from_markdown_str(input).unwrap().nodes; + let Node::List(list) = &nodes[0] else { + panic!("{nodes:?}") + }; + assert_eq!( + list.values.iter().map(Node::value).collect::<String>(), + expected, + "{nodes:?}" + ); +} + /// References keep the notation they were written in, as long as it resolves to the same definition. #[rstest] #[case::full_keeps_case_and_spaces("[x][Foo Bar]\n\n[foo bar]: /u\n", "[x][Foo Bar]")] From b459a31a48be488ce20ad15662b6a2194d6aa2ae Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Thu, 8 Oct 2026 13:08:22 +0900 Subject: [PATCH 34/88] =?UTF-8?q?=E2=9C=85=20test(markdown):=20note=20that?= =?UTF-8?q?=20markdown-rs=20reads=20null=20characters=20and=20open=20tags?= =?UTF-8?q?=20as=20mq=20does?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/mdast_official/known.json | 42 +++++++++---------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/crates/mq-markdown/tests/mdast_official/known.json b/crates/mq-markdown/tests/mdast_official/known.json index 6c4f9654f..9140b1fc9 100644 --- a/crates/mq-markdown/tests/mdast_official/known.json +++ b/crates/mq-markdown/tests/mdast_official/known.json @@ -5,21 +5,21 @@ "mdx:<a {b} />": "Expressions are not parsed as JavaScript, so invalid JavaScript is accepted.", "mdx:{a\n\nb}": "Expressions are not parsed as JavaScript, so invalid JavaScript is accepted.", "mdx:{'a\nb'}": "Expressions are not parsed as JavaScript, so invalid JavaScript is accepted.", - "mdx:<a": "A tag that is not closed before the end of its text is text here, and an error in the official packages.", - "mdx:<a b": "A tag that is not closed before the end of its text is text here, and an error in the official packages.", - "mdx:<a b=": "A tag that is not closed before the end of its text is text here, and an error in the official packages.", - "mdx:x <a": "A tag that is not closed before the end of its text is text here, and an error in the official packages.", - "mdx:x <a b": "A tag that is not closed before the end of its text is text here, and an error in the official packages.", - "mdx:x <a b=": "A tag that is not closed before the end of its text is text here, and an error in the official packages.", - "mdx:<a\n\nb": "A tag that is not closed before the end of its text is text here, and an error in the official packages.", - "mdx:<a\nb": "A tag that is not closed before the end of its text is text here, and an error in the official packages.", - "mdx:a\n<a": "A tag that is not closed before the end of its text is text here, and an error in the official packages.", - "mdx:x <": "A tag that is not closed before the end of its text is text here, and an error in the official packages.", - "mdx:<": "A tag that is not closed before the end of its text is text here, and an error in the official packages.", - "mdx:x <a\n\nb": "A tag that is not closed before the end of its text is text here, and an error in the official packages.", - "mdx:x <a b\n\nc": "A tag that is not closed before the end of its text is text here, and an error in the official packages.", - "mdx:- <a": "A tag that is not closed before the end of its text is text here, and an error in the official packages.", - "mdx:> <a": "A tag that is not closed before the end of its text is text here, and an error in the official packages.", + "mdx:<a": "A tag that is not closed before the end of its text is text here and in markdown-rs, and an error in the JavaScript packages.", + "mdx:<a b": "A tag that is not closed before the end of its text is text here and in markdown-rs, and an error in the JavaScript packages.", + "mdx:<a b=": "A tag that is not closed before the end of its text is text here and in markdown-rs, and an error in the JavaScript packages.", + "mdx:x <a": "A tag that is not closed before the end of its text is text here and in markdown-rs, and an error in the JavaScript packages.", + "mdx:x <a b": "A tag that is not closed before the end of its text is text here and in markdown-rs, and an error in the JavaScript packages.", + "mdx:x <a b=": "A tag that is not closed before the end of its text is text here and in markdown-rs, and an error in the JavaScript packages.", + "mdx:<a\n\nb": "A tag that is not closed before the end of its text is text here and in markdown-rs, and an error in the JavaScript packages.", + "mdx:<a\nb": "A tag that is not closed before the end of its text is text here and in markdown-rs, and an error in the JavaScript packages.", + "mdx:a\n<a": "A tag that is not closed before the end of its text is text here and in markdown-rs, and an error in the JavaScript packages.", + "mdx:x <": "A tag that is not closed before the end of its text is text here and in markdown-rs, and an error in the JavaScript packages.", + "mdx:<": "A tag that is not closed before the end of its text is text here and in markdown-rs, and an error in the JavaScript packages.", + "mdx:x <a\n\nb": "A tag that is not closed before the end of its text is text here and in markdown-rs, and an error in the JavaScript packages.", + "mdx:x <a b\n\nc": "A tag that is not closed before the end of its text is text here and in markdown-rs, and an error in the JavaScript packages.", + "mdx:- <a": "A tag that is not closed before the end of its text is text here and in markdown-rs, and an error in the JavaScript packages.", + "mdx:> <a": "A tag that is not closed before the end of its text is text here and in markdown-rs, and an error in the JavaScript packages.", "commonmark#606": "GFM autolink literals: an email after a backslash escape is text here, as in the CommonMark spec.", "gfm#602": "GFM autolink literals: an email after a backslash escape is text here, as in the CommonMark spec.", "gfm-extensions#19": "GFM autolink literals: the text of a `mailto:` link includes the scheme here, as in the GFM spec.", @@ -51,12 +51,12 @@ "micromark-extension-mdx-expression/micromark-extension-mdx-expression@3.0.1#9": "Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted.", "micromark-extension-mdxjs-esm/3.0.0#11": "Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted.", "micromark-extension-mdxjs-esm/3.0.0#12": "Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted.", - "markdown-rs/code_fenced#34": "A null character is kept in values and replaced with U+FFFD only in HTML, so that positions stay the ones of the source.", - "markdown-rs/hard_break_trailing#11": "A null character is kept in values and replaced with U+FFFD only in HTML, so that positions stay the ones of the source.", - "markdown-rs/hard_break_trailing#12": "A null character is kept in values and replaced with U+FFFD only in HTML, so that positions stay the ones of the source.", - "markdown-rs/link_resource#72": "A null character is kept in values and replaced with U+FFFD only in HTML, so that positions stay the ones of the source.", - "markdown-rs/misc_zero#2": "A null character is kept in values and replaced with U+FFFD only in HTML, so that positions stay the ones of the source.", - "micromark-extension-mdx-expression/micromark-extension-mdx-expression@3.0.1#5": "A null character is kept in values and replaced with U+FFFD only in HTML, so that positions stay the ones of the source.", + "markdown-rs/code_fenced#34": "A null character is kept in values, as in markdown-rs, and replaced with U+FFFD only in HTML; the JavaScript packages replace it in values too.", + "markdown-rs/hard_break_trailing#11": "A null character is kept in values, as in markdown-rs, and replaced with U+FFFD only in HTML; the JavaScript packages replace it in values too.", + "markdown-rs/hard_break_trailing#12": "A null character is kept in values, as in markdown-rs, and replaced with U+FFFD only in HTML; the JavaScript packages replace it in values too.", + "markdown-rs/link_resource#72": "A null character is kept in values, as in markdown-rs, and replaced with U+FFFD only in HTML; the JavaScript packages replace it in values too.", + "markdown-rs/misc_zero#2": "A null character is kept in values, as in markdown-rs, and replaced with U+FFFD only in HTML; the JavaScript packages replace it in values too.", + "micromark-extension-mdx-expression/micromark-extension-mdx-expression@3.0.1#5": "A null character is kept in values, as in markdown-rs, and replaced with U+FFFD only in HTML; the JavaScript packages replace it in values too.", "markdown-rs/autolink#14": "GFM autolink literals: an email after a backslash escape is text here, as in the CommonMark spec.", "markdown-rs/gfm_autolink_literal#28": "GFM autolink literals: www after punctuation other than the ASCII ones of the GFM spec is text here.", "markdown-rs/gfm_autolink_literal#33": "GFM autolink literals: the text of a `mailto:` or `xmpp:` link includes the scheme here, as in the GFM spec.", From 4b1cb3548ef457bed101ff63b70b174eb4f251db Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Thu, 8 Oct 2026 13:08:28 +0900 Subject: [PATCH 35/88] =?UTF-8?q?=E2=9C=A8=20feat(markdown):=20export=20th?= =?UTF-8?q?e=20types=20of=20JSX=20attributes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MdxAttributeContent, MdxJsxAttribute and MdxAttributeValue are public, so that the attributes of an element can be read. --- crates/mq-markdown/src/lib.rs | 7 +++--- crates/mq-markdown/src/node.rs | 3 +++ .../mq-markdown/tests/mdx_roundtrip_tests.rs | 25 +++++++++++++++++++ 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/crates/mq-markdown/src/lib.rs b/crates/mq-markdown/src/lib.rs index b5e995aae..5c011b3a1 100644 --- a/crates/mq-markdown/src/lib.rs +++ b/crates/mq-markdown/src/lib.rs @@ -92,9 +92,10 @@ pub use markdown::{Markdown, to_html}; pub use node::{ Blockquote, Break, Code, CodeInline, ColorTheme, Definition, Delete, Emphasis, Footnote, FootnoteRef, Fragment, Heading, HorizontalRule, HorizontalRuleMarker, Html, Image, ImageRef, Link, LinkRef, List, ListMarker, ListStyle, - Math, MathInline, MdxFlowExpression, MdxJsEsm, MdxJsxFlowElement, MdxJsxTextElement, MdxTextExpression, Node, - Point, Position, RenderOptions, Strong, TableAlign, TableAlignKind, TableCell, TableRow, Text, Title, - TitleSurroundStyle, Toml, Url, UrlSurroundStyle, Yaml, attr_value::AttrValue, + Math, MathInline, MdxAttributeContent, MdxAttributeValue, MdxFlowExpression, MdxJsEsm, MdxJsxAttribute, + MdxJsxFlowElement, MdxJsxTextElement, MdxTextExpression, Node, Point, Position, RenderOptions, Strong, TableAlign, + TableAlignKind, TableCell, TableRow, Text, Title, TitleSurroundStyle, Toml, Url, UrlSurroundStyle, Yaml, + attr_value::AttrValue, }; pub use node_kind::{AttrSpec, AttrType, NodeKind}; diff --git a/crates/mq-markdown/src/node.rs b/crates/mq-markdown/src/node.rs index 1d9d2593c..0a8fb5c23 100644 --- a/crates/mq-markdown/src/node.rs +++ b/crates/mq-markdown/src/node.rs @@ -750,6 +750,7 @@ pub struct MdxJsxFlowElement { derive(serde::Serialize, serde::Deserialize), serde(rename_all = "camelCase", tag = "type", content = "value") )] +/// An attribute of a JSX element: a property such as `a="b"`, or an expression such as `{...props}`. pub enum MdxAttributeContent { Expression(SmolStr), Property(MdxJsxAttribute), @@ -761,6 +762,7 @@ pub enum MdxAttributeContent { derive(serde::Serialize, serde::Deserialize), serde(rename_all = "camelCase", tag = "type") )] +/// A property of a JSX element, with the value it has after its quotes or braces. pub struct MdxJsxAttribute { pub name: SmolStr, pub value: Option<MdxAttributeValue>, @@ -772,6 +774,7 @@ pub struct MdxJsxAttribute { derive(serde::Serialize, serde::Deserialize), serde(rename_all = "camelCase", tag = "type", content = "value") )] +/// The value of a JSX property: a string, or an expression in braces. pub enum MdxAttributeValue { Expression(SmolStr), Literal(SmolStr), diff --git a/crates/mq-markdown/tests/mdx_roundtrip_tests.rs b/crates/mq-markdown/tests/mdx_roundtrip_tests.rs index c16f61ddb..69b061c33 100644 --- a/crates/mq-markdown/tests/mdx_roundtrip_tests.rs +++ b/crates/mq-markdown/tests/mdx_roundtrip_tests.rs @@ -83,3 +83,28 @@ fn text_is_not_escaped_for_markdown() { let markdown = Markdown::from_markdown_str("a {b} and\n\nimport a from \"b\"\n").unwrap(); assert_eq!(markdown.to_string(), "a {b} and\n\nimport a from \"b\"\n"); } + +#[test] +fn attributes_of_an_element_can_be_read() { + use mq_markdown::{MdxAttributeContent, MdxAttributeValue, MdxJsxAttribute}; + + let markdown = Markdown::from_mdx_str("<a b=\"c\" d={e} f {...g} />").unwrap(); + let Node::MdxJsxFlowElement(element) = &markdown.nodes[0] else { + panic!("{:?}", markdown.nodes) + }; + let property = |name: &str, value: Option<MdxAttributeValue>| { + MdxAttributeContent::Property(MdxJsxAttribute { + name: name.into(), + value, + }) + }; + assert_eq!( + element.attributes, + vec![ + property("b", Some(MdxAttributeValue::Literal("c".into()))), + property("d", Some(MdxAttributeValue::Expression("e".into()))), + property("f", None), + MdxAttributeContent::Expression("...g".into()), + ] + ); +} From a8282cfcf011764825ad5ed9890b2bb94a367892 Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Thu, 8 Oct 2026 13:08:34 +0900 Subject: [PATCH 36/88] =?UTF-8?q?=E2=9C=A8=20feat(markdown):=20read=20fron?= =?UTF-8?q?tmatter=20at=20the=20start=20of=20MDX?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit YAML between --- lines and TOML between +++ lines are frontmatter in MDX as in Markdown, as most MDX files start with it. markdown-rs and the MDX packages leave it to an extension, so the comparison with the packages turns the extension on. --- crates/mq-markdown/src/parser.rs | 4 +++- crates/mq-markdown/src/parser/block.rs | 2 +- crates/mq-markdown/src/parser/snapshots.txt | 8 ++++++-- .../tests/mdast_official/compare.mjs | 5 ++++- .../tests/mdast_official/corpus-mdx.json | 7 ++++++- .../mq-markdown/tests/mdx_roundtrip_tests.rs | 19 +++++++++++++++++++ 6 files changed, 39 insertions(+), 6 deletions(-) diff --git a/crates/mq-markdown/src/parser.rs b/crates/mq-markdown/src/parser.rs index 8b6d275d5..46e3def7a 100644 --- a/crates/mq-markdown/src/parser.rs +++ b/crates/mq-markdown/src/parser.rs @@ -123,7 +123,9 @@ mod tests { ("no_autolink", "<http://a.b>"), ("no_gfm", "| a |\n|-|\n\n~a~ www.a.b [^a]\n\n- [ ] a\n"), ("no_math", "$a$\n\n$$\na\n$$\n"), - ("no_frontmatter", "---\na\n---\n"), + ("frontmatter", "---\na\n---\n"), + ("frontmatter_toml", "+++\na = 1\n+++\n\n<A />"), + ("frontmatter_unclosed", "---\na\n"), ("esm", "import a from 'b'\n\nexport const c = 1\n"), ("definition", "[a]: /u\n\n[a]\n"), ("list_deep_indent", "- a\n"), diff --git a/crates/mq-markdown/src/parser/block.rs b/crates/mq-markdown/src/parser/block.rs index 934fbc92b..ff288eb12 100644 --- a/crates/mq-markdown/src/parser/block.rs +++ b/crates/mq-markdown/src/parser/block.rs @@ -38,7 +38,7 @@ pub(super) fn parse(src: &str, mdx: bool) -> Vec<Block> { if let Some(first) = lines.first_mut().filter(|line| line.text.starts_with('\u{feff}')) { *first = first.skip('\u{feff}'.len_utf8()); } - match frontmatter(&lines).filter(|_| !mdx) { + match frontmatter(&lines) { Some((node, next)) => { let mut blocks = vec![Block::Node(node)]; blocks.extend(parse_blocks(&lines[next..], 0, Interrupt::default())); diff --git a/crates/mq-markdown/src/parser/snapshots.txt b/crates/mq-markdown/src/parser/snapshots.txt index bca07c272..dcc285c72 100644 --- a/crates/mq-markdown/src/parser/snapshots.txt +++ b/crates/mq-markdown/src/parser/snapshots.txt @@ -1287,8 +1287,12 @@ nodes: error: Unexpected character `/` (U+002F) before local name nodes: [Text(Text { value: "| a |\n|-|", @1:1-2:4 }), Text(Text { value: "~a~ www.a.b [^a]", @4:1-4:17 }), List(List { values: [Text(Text { value: "[ ] a", @6:3-6:8 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @6:3-6:8 })] ## mdx no_math "$a$\n\n$$\na\n$$\n" nodes: [Text(Text { value: "$a$", @1:1-1:4 }), Text(Text { value: "$$\na\n$$", @3:1-5:3 })] -## mdx no_frontmatter "---\na\n---\n" -nodes: [HorizontalRule(HorizontalRule { marker: Some(Dash), @1:1-1:4 }), Heading(Heading { depth: 2, values: [Text(Text { value: "a", @2:1-2:2 })], @2:1-3:4 })] +## mdx frontmatter "---\na\n---\n" +nodes: [Yaml(Yaml { value: "a", @1:1-3:4 })] +## mdx frontmatter_toml "+++\na = 1\n+++\n\n<A />" +nodes: [Toml(Toml { value: "a = 1", @1:1-3:4 }), MdxJsxFlowElement(MdxJsxFlowElement { children: [], @5:1-5:6, name: Some("A"), attributes: [] })] +## mdx frontmatter_unclosed "---\na\n" +nodes: [HorizontalRule(HorizontalRule { marker: Some(Dash), @1:1-1:4 }), Text(Text { value: "a", @2:1-2:2 })] ## mdx esm "import a from 'b'\n\nexport const c = 1\n" nodes: [MdxJsEsm(MdxJsEsm { value: "import a from 'b'", @1:1-1:18 }), MdxJsEsm(MdxJsEsm { value: "export const c = 1", @3:1-3:19 })] ## mdx definition "[a]: /u\n\n[a]\n" diff --git a/crates/mq-markdown/tests/mdast_official/compare.mjs b/crates/mq-markdown/tests/mdast_official/compare.mjs index c79b2e3c0..defdeacae 100644 --- a/crates/mq-markdown/tests/mdast_official/compare.mjs +++ b/crates/mq-markdown/tests/mdast_official/compare.mjs @@ -165,7 +165,10 @@ const OPTIONS = { extensions: [gfm(), frontmatter(['yaml', 'toml']), math()], mdastExtensions: [gfmFromMarkdown(), frontmatterFromMarkdown(['yaml', 'toml']), mathFromMarkdown()], }, - mdx: {extensions: [mdxjs()], mdastExtensions: [mdxFromMarkdown()]}, + mdx: { + extensions: [mdxjs(), frontmatter(['yaml', 'toml'])], + mdastExtensions: [mdxFromMarkdown(), frontmatterFromMarkdown(['yaml', 'toml'])], + }, } function official({mode, input}) { diff --git a/crates/mq-markdown/tests/mdast_official/corpus-mdx.json b/crates/mq-markdown/tests/mdast_official/corpus-mdx.json index 1e138b595..6223a95be 100644 --- a/crates/mq-markdown/tests/mdast_official/corpus-mdx.json +++ b/crates/mq-markdown/tests/mdast_official/corpus-mdx.json @@ -194,5 +194,10 @@ "<a>{'}'}</a>", "{'a\nb'}", "{a / b}", -"{a /* x */ + 1}" +"{a /* x */ + 1}", +"---\ntitle: x\n---\n\n<A />\n", +"+++\na = 1\n+++\n\nb\n", +"---\na\n", +"---\n\n---\n\nx\n", +"---\ntitle: x\n---\nimport a from \"b\"\n\n{a}\n" ] \ No newline at end of file diff --git a/crates/mq-markdown/tests/mdx_roundtrip_tests.rs b/crates/mq-markdown/tests/mdx_roundtrip_tests.rs index 69b061c33..bf67be146 100644 --- a/crates/mq-markdown/tests/mdx_roundtrip_tests.rs +++ b/crates/mq-markdown/tests/mdx_roundtrip_tests.rs @@ -108,3 +108,22 @@ fn attributes_of_an_element_can_be_read() { ] ); } + +#[rstest] +#[case::yaml("---\ntitle: x\n---\n\n# h\n", "yaml")] +#[case::toml("+++\ntitle = 1\n+++\n\n# h\n", "toml")] +fn mdx_starts_with_frontmatter(#[case] input: &str, #[case] kind: &str) { + let markdown = Markdown::from_mdx_str(input).unwrap(); + assert_eq!(markdown.nodes[0].name(), kind, "{:?}", markdown.nodes); + assert_eq!(markdown.to_string(), input); +} + +#[test] +fn mdx_without_a_closing_line_has_no_frontmatter() { + let markdown = Markdown::from_mdx_str("---\ntitle: x\n").unwrap(); + assert!( + markdown.nodes.iter().all(|node| node.name() != "yaml"), + "{:?}", + markdown.nodes + ); +} From bb96d8f999d202b74e4c21fe93b1c293c87d7e3d Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Thu, 8 Oct 2026 13:08:40 +0900 Subject: [PATCH 37/88] =?UTF-8?q?=E2=9C=A8=20feat(run):=20add=20--no-front?= =?UTF-8?q?matter=20to=20read=20Markdown=20and=20MDX=20without=20frontmatt?= =?UTF-8?q?er?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit mq-markdown gains ParseOptions with frontmatter, used by from_markdown_str_with and from_mdx_str_with, and mq-lang parse_markdown_input_with and parse_mdx_input_with. The flag is checked against the input format, as --no-header is. --- crates/mq-lang/src/lib.rs | 14 ++- crates/mq-markdown/src/lib.rs | 2 +- crates/mq-markdown/src/markdown.rs | 36 ++++++-- crates/mq-markdown/src/parser.rs | 16 +++- crates/mq-markdown/src/parser/block.rs | 4 +- crates/mq-markdown/src/parser/render_html.rs | 2 +- .../mq-markdown/tests/mdx_roundtrip_tests.rs | 34 ++++++++ crates/mq-run/src/cli.rs | 86 ++++++++++++++++++- 8 files changed, 178 insertions(+), 16 deletions(-) diff --git a/crates/mq-lang/src/lib.rs b/crates/mq-lang/src/lib.rs index f2c89d382..0b3462bd9 100644 --- a/crates/mq-lang/src/lib.rs +++ b/crates/mq-lang/src/lib.rs @@ -212,7 +212,12 @@ pub(crate) fn parse_in_module( /// Parses an MDX string and returns an iterator over `Value` nodes. pub fn parse_mdx_input(input: &str) -> miette::Result<Vec<RuntimeValue>> { - let mdx = mq_markdown::Markdown::from_mdx_str(input)?; + parse_mdx_input_with(input, mq_markdown::ParseOptions::default()) +} + +/// Parses an MDX string with the given options and returns an iterator over `Value` nodes. +pub fn parse_mdx_input_with(input: &str, options: mq_markdown::ParseOptions) -> miette::Result<Vec<RuntimeValue>> { + let mdx = mq_markdown::Markdown::from_mdx_str_with(input, options)?; Ok(mdx.nodes.into_iter().map(RuntimeValue::from).collect()) } @@ -233,7 +238,12 @@ pub fn parse_html_input_with_options( /// Parses a Markdown string and returns an iterator over `Value` nodes. pub fn parse_markdown_input(input: &str) -> miette::Result<Vec<RuntimeValue>> { - let md = mq_markdown::Markdown::from_markdown_str(input)?; + parse_markdown_input_with(input, mq_markdown::ParseOptions::default()) +} + +/// Parses a Markdown string with the given options and returns an iterator over `Value` nodes. +pub fn parse_markdown_input_with(input: &str, options: mq_markdown::ParseOptions) -> miette::Result<Vec<RuntimeValue>> { + let md = mq_markdown::Markdown::from_markdown_str_with(input, options)?; Ok(md.nodes.into_iter().map(RuntimeValue::from).collect()) } diff --git a/crates/mq-markdown/src/lib.rs b/crates/mq-markdown/src/lib.rs index 5c011b3a1..153326c4b 100644 --- a/crates/mq-markdown/src/lib.rs +++ b/crates/mq-markdown/src/lib.rs @@ -88,7 +88,7 @@ mod markdown; mod node; mod node_kind; mod parser; -pub use markdown::{Markdown, to_html}; +pub use markdown::{Markdown, ParseOptions, to_html}; pub use node::{ Blockquote, Break, Code, CodeInline, ColorTheme, Definition, Delete, Emphasis, Footnote, FootnoteRef, Fragment, Heading, HorizontalRule, HorizontalRuleMarker, Html, Image, ImageRef, Link, LinkRef, List, ListMarker, ListStyle, diff --git a/crates/mq-markdown/src/markdown.rs b/crates/mq-markdown/src/markdown.rs index 2fdea2a5d..73792cb1e 100644 --- a/crates/mq-markdown/src/markdown.rs +++ b/crates/mq-markdown/src/markdown.rs @@ -16,6 +16,20 @@ use table_layout::{TableLayout, write_padded_cell}; mod list_order; mod table_layout; +/// What reading Markdown or MDX turns on. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ParseOptions { + /// Read YAML between `---` lines, or TOML between `+++` lines, at the start of the input as frontmatter. + /// Without it these lines are a rule and text, or a heading. + pub frontmatter: bool, +} + +impl Default for ParseOptions { + fn default() -> Self { + Self { frontmatter: true } + } +} + #[derive(Debug, Clone)] pub struct Markdown { pub nodes: Vec<Node>, @@ -284,8 +298,13 @@ impl Markdown { } pub fn from_mdx_str(content: &str) -> miette::Result<Self> { + Self::from_mdx_str_with(content, ParseOptions::default()) + } + + /// Reads MDX with the given options. + pub fn from_mdx_str_with(content: &str, options: ParseOptions) -> miette::Result<Self> { Ok(Self { - nodes: parse_mdx_nodes(content)?, + nodes: parse_mdx_nodes(content, options)?, options: RenderOptions { mdx: true, ..RenderOptions::default() @@ -334,7 +353,12 @@ impl Markdown { } pub fn from_markdown_str(content: &str) -> miette::Result<Self> { - let nodes = parse_nodes(content)?; + Self::from_markdown_str_with(content, ParseOptions::default()) + } + + /// Reads Markdown with the given options. + pub fn from_markdown_str_with(content: &str, options: ParseOptions) -> miette::Result<Self> { + let nodes = parse_nodes(content, options)?; Ok(Self { nodes, @@ -344,13 +368,13 @@ impl Markdown { } /// Parses `content` into nodes. -fn parse_nodes(content: &str) -> miette::Result<Vec<Node>> { - crate::parser::parse(content) +fn parse_nodes(content: &str, options: ParseOptions) -> miette::Result<Vec<Node>> { + crate::parser::parse_with(content, options.frontmatter) } /// Parses `content` as MDX into nodes. -fn parse_mdx_nodes(content: &str) -> miette::Result<Vec<Node>> { - crate::parser::parse_mdx(content) +fn parse_mdx_nodes(content: &str, options: ParseOptions) -> miette::Result<Vec<Node>> { + crate::parser::parse_mdx_with(content, options.frontmatter) } /// Converts Markdown to HTML. diff --git a/crates/mq-markdown/src/parser.rs b/crates/mq-markdown/src/parser.rs index 46e3def7a..73e26f623 100644 --- a/crates/mq-markdown/src/parser.rs +++ b/crates/mq-markdown/src/parser.rs @@ -26,8 +26,14 @@ use crate::node::Node; pub(crate) use inline::{is_autolink_email, normalize, unescape}; /// Parses `content` into a flat list of nodes. +#[cfg(test)] pub(crate) fn parse(content: &str) -> miette::Result<Vec<Node>> { - resolve::resolve(block::parse(content, false), false).map_err(|message| miette::miette!(message)) + parse_with(content, true) +} + +/// Parses `content` into a flat list of nodes, with frontmatter at the start when `frontmatter` is set. +pub(crate) fn parse_with(content: &str, frontmatter: bool) -> miette::Result<Vec<Node>> { + resolve::resolve(block::parse(content, false, frontmatter), false).map_err(|message| miette::miette!(message)) } /// Renders `content` as HTML. @@ -36,8 +42,14 @@ pub(crate) fn to_html(content: &str) -> String { } /// Parses `content` as MDX: no indented code, HTML, autolinks or GFM, but expressions and JSX. +#[cfg(test)] pub(crate) fn parse_mdx(content: &str) -> miette::Result<Vec<Node>> { - resolve::resolve(block::parse(content, true), true).map_err(|message| miette::miette!(message)) + parse_mdx_with(content, true) +} + +/// Parses `content` as MDX, with frontmatter at the start when `frontmatter` is set. +pub(crate) fn parse_mdx_with(content: &str, frontmatter: bool) -> miette::Result<Vec<Node>> { + resolve::resolve(block::parse(content, true, frontmatter), true).map_err(|message| miette::miette!(message)) } #[cfg(test)] diff --git a/crates/mq-markdown/src/parser/block.rs b/crates/mq-markdown/src/parser/block.rs index ff288eb12..df745c78b 100644 --- a/crates/mq-markdown/src/parser/block.rs +++ b/crates/mq-markdown/src/parser/block.rs @@ -32,13 +32,13 @@ struct Interrupt { list: bool, } -pub(super) fn parse(src: &str, mdx: bool) -> Vec<Block> { +pub(super) fn parse(src: &str, mdx: bool, read_frontmatter: bool) -> Vec<Block> { let mut lines = split_lines(src, mdx); // A byte order mark at the start of the document is not content. if let Some(first) = lines.first_mut().filter(|line| line.text.starts_with('\u{feff}')) { *first = first.skip('\u{feff}'.len_utf8()); } - match frontmatter(&lines) { + match read_frontmatter.then(|| frontmatter(&lines)).flatten() { Some((node, next)) => { let mut blocks = vec![Block::Node(node)]; blocks.extend(parse_blocks(&lines[next..], 0, Interrupt::default())); diff --git a/crates/mq-markdown/src/parser/render_html.rs b/crates/mq-markdown/src/parser/render_html.rs index dc602a552..af950ecef 100644 --- a/crates/mq-markdown/src/parser/render_html.rs +++ b/crates/mq-markdown/src/parser/render_html.rs @@ -20,7 +20,7 @@ const SAFE_PROTOCOL_SRC: [&str; 2] = ["http", "https"]; /// Renders `content` as HTML. pub(super) fn render(content: &str) -> String { - let blocks = super::block::parse(content, false); + let blocks = super::block::parse(content, false, true); let mut references = References::default(); resolve::collect(&blocks, &mut references); diff --git a/crates/mq-markdown/tests/mdx_roundtrip_tests.rs b/crates/mq-markdown/tests/mdx_roundtrip_tests.rs index bf67be146..84f5dfa21 100644 --- a/crates/mq-markdown/tests/mdx_roundtrip_tests.rs +++ b/crates/mq-markdown/tests/mdx_roundtrip_tests.rs @@ -127,3 +127,37 @@ fn mdx_without_a_closing_line_has_no_frontmatter() { markdown.nodes ); } + +#[rstest] +#[case::mdx(true)] +#[case::markdown(false)] +fn frontmatter_can_be_left_to_the_text(#[case] mdx: bool) { + use mq_markdown::ParseOptions; + + let options = ParseOptions { frontmatter: false }; + let input = "---\ntitle: x\n---\n\n+++\na\n+++\n"; + let markdown = if mdx { + Markdown::from_mdx_str_with(input, options) + } else { + Markdown::from_markdown_str_with(input, options) + } + .unwrap(); + assert!( + markdown + .nodes + .iter() + .all(|node| !matches!(node.name().as_str(), "yaml" | "toml")), + "{:?}", + markdown.nodes + ); + assert_eq!(markdown.nodes[0].name().as_str(), "Horizontal_rule"); + assert_eq!(markdown.nodes[1].name().as_str(), "h2"); + + let default = if mdx { + Markdown::from_mdx_str(input) + } else { + Markdown::from_markdown_str(input) + } + .unwrap(); + assert_eq!(default.nodes[0].name().as_str(), "yaml"); +} diff --git a/crates/mq-run/src/cli.rs b/crates/mq-run/src/cli.rs index be54a11d6..d1eda97f9 100644 --- a/crates/mq-run/src/cli.rs +++ b/crates/mq-run/src/cli.rs @@ -466,6 +466,12 @@ struct ProgramArgs { #[arg(long = "no-header", default_value_t = false)] no_header: bool, + /// Do not read frontmatter at the start of Markdown or MDX input: the lines between `---` or + /// `+++` are then text, a heading or a rule. Applies to `-I markdown` and `-I mdx`, and to + /// `.md` and `.mdx` files + #[arg(long = "no-frontmatter", default_value_t = false)] + no_frontmatter: bool, + /// Search modules from the directory #[arg(short = 'L', long = "directory")] module_directories: Option<Vec<PathBuf>>, @@ -1809,6 +1815,7 @@ impl Cli { } self.validate_csv_options()?; + self.validate_frontmatter_option()?; self.check_output_permitted()?; match &self.commands { @@ -2044,6 +2051,22 @@ impl Cli { Ok(aggregate.map(|agg| format!("{} | {}", agg, query)).unwrap_or(query)) } + fn validate_frontmatter_option(&self) -> miette::Result<()> { + if self.input.program.no_frontmatter + && matches!(self.explicit_input_format(), Some(fmt) if !matches!(fmt, InputFormat::Markdown | InputFormat::Mdx)) + { + return Err(miette!("--no-frontmatter only applies to -I markdown or -I mdx")); + } + Ok(()) + } + + /// How Markdown and MDX input is read. + fn parse_options(&self) -> mq_markdown::ParseOptions { + mq_markdown::ParseOptions { + frontmatter: !self.input.program.no_frontmatter, + } + } + fn validate_csv_options(&self) -> miette::Result<()> { if (self.input.program.csv_delimiter.is_some() || self.input.program.no_header) && matches!(self.explicit_input_format(), Some(fmt) if !matches!(fmt, InputFormat::Csv | InputFormat::Tsv | InputFormat::Psv)) @@ -2171,8 +2194,8 @@ impl Cli { } }) { // Native formats - InputFormat::Markdown => mq_lang::parse_markdown_input(text)?, - InputFormat::Mdx => mq_lang::parse_mdx_input(text)?, + InputFormat::Markdown => mq_lang::parse_markdown_input_with(text, self.parse_options())?, + InputFormat::Mdx => mq_lang::parse_mdx_input_with(text, self.parse_options())?, InputFormat::Html => mq_lang::parse_html_input(text)?, InputFormat::Text => mq_lang::parse_text_input(text)?, InputFormat::Null => mq_lang::null_input(), @@ -5960,6 +5983,65 @@ mod tests { assert_eq!(cli.tabular_query_prefix(&InputFormat::Json), None); } + fn frontmatter_cli(format: Option<InputFormat>, no_frontmatter: bool) -> Cli { + Cli { + input: InputArgs { + program: ProgramArgs { + input_format: format, + no_frontmatter, + ..Default::default() + }, + ..Default::default() + }, + ..Cli::default() + } + } + + #[rstest] + #[case::markdown(InputFormat::Markdown)] + #[case::mdx(InputFormat::Mdx)] + fn test_frontmatter_is_read_unless_turned_off(#[case] format: InputFormat) { + let input = ContentData::Text("---\ntitle: x\n---\n\ntext".to_string()); + let names = |cli: &Cli| -> Vec<String> { + cli.resolve_input(&None, &input) + .unwrap() + .iter() + .map(|value| match value { + mq_lang::RuntimeValue::Markdown(node, _) => node.name().to_string(), + other => other.to_string(), + }) + .collect() + }; + + assert_eq!(names(&frontmatter_cli(Some(format.clone()), false)), ["yaml", "text"]); + assert_eq!( + names(&frontmatter_cli(Some(format), true)), + ["Horizontal_rule", "h2", "text"] + ); + } + + #[rstest] + #[case::markdown(Some(InputFormat::Markdown), true)] + #[case::mdx(Some(InputFormat::Mdx), true)] + #[case::by_extension(None, true)] + #[case::csv(Some(InputFormat::Csv), false)] + #[case::json(Some(InputFormat::Json), false)] + fn test_no_frontmatter_applies_to_markdown_and_mdx(#[case] format: Option<InputFormat>, #[case] valid: bool) { + assert_eq!( + frontmatter_cli(format, true).validate_frontmatter_option().is_ok(), + valid + ); + } + + #[test] + fn test_no_frontmatter_is_not_checked_when_it_is_not_given() { + assert!( + frontmatter_cli(Some(InputFormat::Csv), false) + .validate_frontmatter_option() + .is_ok() + ); + } + #[test] fn test_json_auto_parse() { let (_, temp_file_path) = create_file("auto_parse_test.json", r#"{"key": "value"}"#); From a704311ba8c3bf473de132ea9b94820d2caa1be2 Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Thu, 8 Oct 2026 13:08:47 +0900 Subject: [PATCH 38/88] =?UTF-8?q?=F0=9F=94=A7=20chore(typos):=20allow=20th?= =?UTF-8?q?e=20character=20reference=20form=20of=20import=20in=20the=20MDX?= =?UTF-8?q?=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .typos.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.typos.toml b/.typos.toml index 77c92976a..71cf2657a 100644 --- a/.typos.toml +++ b/.typos.toml @@ -10,6 +10,8 @@ ist = "ist" # Intentional misspelling in the fuzzy-match-headings cookbook recipe instalation = "instalation" juxt = "juxt" +# `import` with its first letter as a character reference (`&#105;mport`), as MDX text is written +mport = "mport" reakpoint = "reakpoint" sction = "sction" From 8480c07f75d375cbbd19bc796b5657d35573e014 Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Thu, 8 Oct 2026 13:08:53 +0900 Subject: [PATCH 39/88] =?UTF-8?q?=E2=9A=A1=20perf(markdown):=20look=20for?= =?UTF-8?q?=20an=20at=20sign=20once=20per=20text,=20not=20for=20each=20pai?= =?UTF-8?q?r=20of=20emphasis?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every pair of emphasis, strong or strikethrough that matched looked through the whole text for an at sign, so a paragraph with many pairs took time that grew with the square of its length. --- crates/mq-markdown/src/parser/inline.rs | 7 ++++--- crates/mq-markdown/tests/parser_limits_tests.rs | 2 ++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/crates/mq-markdown/src/parser/inline.rs b/crates/mq-markdown/src/parser/inline.rs index 19c7c2159..32f0ba7ab 100644 --- a/crates/mq-markdown/src/parser/inline.rs +++ b/crates/mq-markdown/src/parser/inline.rs @@ -102,7 +102,8 @@ pub(super) fn item_depth(item: &Item) -> usize { pub(super) struct Context<'a> { pub(super) source: &'a InlineSource, pub(super) references: &'a References, - /// Whether email addresses in plain text are linked. They are not inside link text. + /// Whether email addresses in plain text are linked: not inside link text, and not when there is no `@` + /// in the text, which is looked for once. pub(super) emails: bool, /// The first error in the content, which only MDX has. error: &'a RefCell<Option<String>>, @@ -136,7 +137,7 @@ pub(super) fn parse(source: &InlineSource, references: &References) -> Result<Ve let context = Context { source, references, - emails: !references.mdx, + emails: !references.mdx && source.text.contains('@'), error: &error, }; let mut scanner = Scanner { @@ -167,7 +168,7 @@ pub(super) fn parse(source: &InlineSource, references: &References) -> Result<Ve /// Converts items to nodes, merging adjacent text. Unused delimiters and openers become text. pub(super) fn to_nodes(items: Vec<Item>, context: &Context<'_>) -> Vec<Node> { let src = context.src(); - let items = if context.emails && src.contains('@') { + let items = if context.emails { literal::link_emails(src, items) } else { items diff --git a/crates/mq-markdown/tests/parser_limits_tests.rs b/crates/mq-markdown/tests/parser_limits_tests.rs index 8fa1d7c7e..a0beb0747 100644 --- a/crates/mq-markdown/tests/parser_limits_tests.rs +++ b/crates/mq-markdown/tests/parser_limits_tests.rs @@ -73,6 +73,8 @@ fn elements_nested_within_the_limit_parse() { #[case::quote_markers_on_a_line(format!("{}a\n", "> ".repeat(20_000)).repeat(20))] #[case::numbered_markers_on_a_line(format!("{}a\n", "1. ".repeat(20_000)).repeat(20))] #[case::long_labels(format!("[a]: /u\n{}", "[b ".repeat(100_000)))] +#[case::many_emphasis_pairs("*a* _b_ **c** __d__ ~~e~~ ".repeat(30_000))] +#[case::many_emphasis_pairs_and_an_email("*a* b@c.de ".repeat(30_000))] #[case::backtick_runs_of_every_length((1..3_000).map(|length| format!("e{}", "`".repeat(length))).collect::<String>())] #[case::dollar_runs_of_every_length((1..3_000).map(|length| format!("e{}", "$".repeat(length))).collect::<String>())] #[case::unclosed_comments("a <!--".repeat(300_000))] From 81c22614f592224cbdb4b7306552155b0eb5bbb5 Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Thu, 8 Oct 2026 13:08:59 +0900 Subject: [PATCH 40/88] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor(markdown):?= =?UTF-8?q?=20type=20the=20list=20that=20emphasis=20is=20matched=20in?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The slots of the list are identified by a SlotId that is never zero, so that the end of the list is an Option and not a number set aside, and the list, the keys of the openers a closer rejected and the search for an opener are types with their own methods. The time is the same. --- .../mq-markdown/src/parser/inline/emphasis.rs | 331 ++++++++++++------ 1 file changed, 215 insertions(+), 116 deletions(-) diff --git a/crates/mq-markdown/src/parser/inline/emphasis.rs b/crates/mq-markdown/src/parser/inline/emphasis.rs index 9f3ad455d..d5f058e29 100644 --- a/crates/mq-markdown/src/parser/inline/emphasis.rs +++ b/crates/mq-markdown/src/parser/inline/emphasis.rs @@ -6,6 +6,7 @@ use super::punctuation::is_punctuation; use super::{Context, Delim, Item, MAX_NESTING, item_depth, to_nodes}; use crate::node::{Delete, Emphasis, Node, Strong}; +use std::num::NonZeroUsize; #[derive(PartialEq)] enum Kind { @@ -71,50 +72,204 @@ fn matches(opener: &Delim, closer: &Delim) -> bool { !(closer.ch == b'~' && (closer.count != opener.count || closer.count > 2)) } -/// A node of the list the delimiters are matched in, which lets matched items be taken out between -/// two delimiters without moving the items after them. +/// The place of an item in `Slots`. +/// +/// It is not zero, so that an optional one takes no more room than a number. +#[derive(Clone, Copy, PartialEq, Eq)] +struct SlotId(NonZeroUsize); + +impl SlotId { + /// The slot at `index` of the vector of the slots. + fn at(index: usize) -> Self { + Self(NonZeroUsize::MIN.saturating_add(index)) + } + + fn index(self) -> usize { + self.0.get() - 1 + } +} + +/// A node of the list the delimiters are matched in. struct Slot { item: Option<Item>, - prev: usize, - next: usize, + prev: SlotId, + next: Option<SlotId>, } -const NIL: usize = usize::MAX; +/// The list the delimiters are matched in, which lets matched items be taken out between two delimiters +/// without moving the items after them. +/// +/// It starts with a slot that holds no item, so that every other slot has one before it. +struct Slots(Vec<Slot>); + +impl Slots { + const HEAD: SlotId = SlotId(NonZeroUsize::MIN); + + /// Moves `items` into a list. + fn new(items: &mut Vec<Item>) -> Self { + let total = items.len(); + let mut slots = Vec::with_capacity(total + 1); + slots.push(Slot { + item: None, + prev: Self::HEAD, + next: (total > 0).then(|| SlotId::at(1)), + }); + for (index, item) in items.drain(..).enumerate() { + slots.push(Slot { + item: Some(item), + prev: SlotId::at(index), + next: (index + 1 < total).then(|| SlotId::at(index + 2)), + }); + } + Self(slots) + } + + fn slot(&self, id: SlotId) -> &Slot { + &self.0[id.index()] + } + + fn slot_mut(&mut self, id: SlotId) -> &mut Slot { + &mut self.0[id.index()] + } + + fn first(&self) -> Option<SlotId> { + self.next(Self::HEAD) + } + + fn next(&self, id: SlotId) -> Option<SlotId> { + self.slot(id).next + } + + fn is_delim(&self, id: SlotId) -> bool { + matches!(self.slot(id).item, Some(Item::Delim(_))) + } + + fn delim(&self, id: SlotId) -> &Delim { + match &self.slot(id).item { + Some(Item::Delim(delim)) => delim, + _ => unreachable!("the slot holds a delimiter"), + } + } + + fn delim_mut(&mut self, id: SlotId) -> &mut Delim { + match &mut self.slot_mut(id).item { + Some(Item::Delim(delim)) => delim, + _ => unreachable!("the slot holds a delimiter"), + } + } + + /// The slots after `from` and before `to`, which comes later in the list. + fn between(&self, from: SlotId, to: SlotId) -> impl Iterator<Item = SlotId> + '_ { + std::iter::successors(self.next(from), |&id| self.next(id)).take_while(move |&id| id != to) + } + + /// Whether the node for the items between `opener` and `closer` stays within the nesting limit. + fn fits(&self, opener: SlotId, closer: SlotId) -> bool { + let depth = self + .between(opener, closer) + .filter_map(|id| self.slot(id).item.as_ref()) + .map(item_depth) + .max() + .unwrap_or(0); + depth < MAX_NESTING + } + + /// Takes the items between `opener` and `closer` out, and returns them with the depth of the deepest. + fn take_between(&mut self, opener: SlotId, closer: SlotId) -> (Vec<Item>, usize) { + let mut children = Vec::new(); + let mut depth = 0; + let mut cursor = self.next(opener); + while let Some(id) = cursor.filter(|&id| id != closer) { + if let Some(item) = self.slot_mut(id).item.take() { + depth = depth.max(item_depth(&item)); + children.push(item); + } + cursor = self.next(id); + } + (children, depth) + } + + /// Puts `item` between `opener` and `closer`, which are next to each other once the items between + /// them are taken out. + fn insert_between(&mut self, opener: SlotId, closer: SlotId, item: Item) { + let id = SlotId::at(self.0.len()); + self.0.push(Slot { + item: Some(item), + prev: opener, + next: Some(closer), + }); + self.slot_mut(opener).next = Some(id); + self.slot_mut(closer).prev = id; + } + + /// Takes the slot out of the list. + fn unlink(&mut self, id: SlotId) { + let (prev, next) = (self.slot(id).prev, self.slot(id).next); + self.slot_mut(prev).next = next; + if let Some(next) = next { + self.slot_mut(next).prev = prev; + } + self.slot_mut(id).item = None; + } + + /// Moves the items that are left, in order, to the end of `items`. + fn move_into(mut self, items: &mut Vec<Item>) { + let mut cursor = self.first(); + while let Some(id) = cursor { + items.extend(self.slot_mut(id).item.take()); + cursor = self.next(id); + } + } +} /// What decides whether a closer can match an opener: the openers a closer with the same key /// rejected once are rejected for every later closer with it. -fn key(closer: &Delim) -> usize { - let ch = match closer.ch { - b'*' => 0, - b'_' => 1, - _ => 2, - }; - // Strikethrough also goes by the length of the run. - let count = if ch == 2 { closer.count.min(2) } else { 0 }; - ((ch * 2 + usize::from(closer.can_open)) * 3 + closer.original % 3) * 3 + count -} +#[derive(Clone, Copy)] +struct Key(usize); -fn delim(slots: &[Slot], index: usize) -> &Delim { - match &slots[index].item { - Some(Item::Delim(delim)) => delim, - _ => unreachable!("the index is a delimiter"), +impl Key { + /// The number of different keys: the character, whether it can open, the rest of the length of the + /// run by three, and for strikethrough the length of the run. + const COUNT: usize = 3 * 2 * 3 * 3; + + fn of(closer: &Delim) -> Self { + let ch = match closer.ch { + b'*' => 0, + b'_' => 1, + _ => 2, + }; + // Strikethrough also goes by the length of the run. + let count = if ch == 2 { closer.count.min(2) } else { 0 }; + Self(((ch * 2 + usize::from(closer.can_open)) * 3 + closer.original % 3) * 3 + count) } } -fn delim_mut(slots: &mut [Slot], index: usize) -> &mut Delim { - match &mut slots[index].item { - Some(Item::Delim(delim)) => delim, - _ => unreachable!("the index is a delimiter"), +/// For each key, the offset that the openers a closer with it rejected start at or after. +struct Bottoms([usize; Key::COUNT]); + +impl Bottoms { + fn get(&self, key: Key) -> usize { + self.0[key.0] + } + + fn set(&mut self, key: Key, start: usize) { + self.0[key.0] = start; } } -fn unlink(slots: &mut [Slot], index: usize) { - let (prev, next) = (slots[index].prev, slots[index].next); - slots[prev].next = next; - if next != NIL { - slots[next].prev = prev; +/// The place in `stack` of the nearest opener that `closer` can match, not looking at openers that start +/// before `bottom`. +fn find_opener(slots: &Slots, stack: &[SlotId], closer: &Delim, bottom: usize) -> Option<usize> { + for (at, &id) in stack.iter().enumerate().rev() { + let candidate = slots.delim(id); + if candidate.start < bottom { + break; + } + if matches(candidate, closer) { + return Some(at); + } } - slots[index].item = None; + None } /// Matches the delimiters in `items`, replacing each pair and what is between them by a node. @@ -126,76 +281,43 @@ pub(super) fn process(items: &mut Vec<Item>, context: &Context<'_>) { return; } - let mut slots = Vec::with_capacity(items.len() + 1); - let total = items.len(); - slots.push(Slot { - item: None, - prev: NIL, - next: 1, - }); - for (index, item) in items.drain(..).enumerate() { - slots.push(Slot { - item: Some(item), - prev: index, - next: if index + 1 == total { NIL } else { index + 2 }, - }); - } + let mut slots = Slots::new(items); + let mut stack: Vec<SlotId> = Vec::new(); + let mut bottoms = Bottoms([0; Key::COUNT]); + let mut cursor = slots.first(); - let mut stack: Vec<usize> = Vec::new(); - let mut bottoms = [0usize; 54]; - let mut closer = slots[0].next; - - while closer != NIL { - if !matches!(slots[closer].item, Some(Item::Delim(_))) { - closer = slots[closer].next; + while let Some(closer) = cursor { + if !slots.is_delim(closer) { + cursor = slots.next(closer); continue; } - let current = delim(&slots, closer); - let (can_open, can_close, key) = (current.can_open, current.can_close, key(current)); - - let mut found = None; - if can_close { - for at in (0..stack.len()).rev() { - let candidate = delim(&slots, stack[at]); - if candidate.start < bottoms[key] { - break; - } - if matches(candidate, current) { - found = Some(at); - break; - } - } - } - let found = found.filter(|&at| fits(&slots, stack[at], closer)); + let current = slots.delim(closer); + let (can_open, can_close, key) = (current.can_open, current.can_close, Key::of(current)); + + let found = can_close + .then(|| find_opener(&slots, &stack, current, bottoms.get(key))) + .flatten() + .filter(|&at| slots.fits(stack[at], closer)); let Some(at) = found else { if can_close { - bottoms[key] = delim(&slots, closer).start; + bottoms.set(key, slots.delim(closer).start); } if can_open { stack.push(closer); } - closer = slots[closer].next; + cursor = slots.next(closer); continue; }; let opener = stack[at]; - let (open, close) = (delim(&slots, opener), delim(&slots, closer)); + let (open, close) = (slots.delim(opener), slots.delim(closer)); let used = if open.count > 1 && close.count > 1 { 2 } else { 1 }; let ch = close.ch; let start = open.start + open.count - used; let end = close.start + used; - let mut children = Vec::new(); - let mut depth = 0; - let mut index = slots[opener].next; - while index != closer { - if let Some(item) = slots[index].item.take() { - depth = depth.max(item_depth(&item)); - children.push(item); - } - index = slots[index].next; - } + let (children, depth) = slots.take_between(opener, closer); let values = to_nodes(children, context); let position = Some(context.position(start, end)); let node = match (ch, used) { @@ -203,50 +325,27 @@ pub(super) fn process(items: &mut Vec<Item>, context: &Context<'_>) { (_, 2) => Node::Strong(Strong { values, position }), _ => Node::Emphasis(Emphasis { values, position }), }; + slots.insert_between(opener, closer, Item::Node(node, depth + 1)); - let node_index = slots.len(); - slots.push(Slot { - item: Some(Item::Node(node, depth + 1)), - prev: opener, - next: closer, - }); - slots[opener].next = node_index; - slots[closer].prev = node_index; - - delim_mut(&mut slots, opener).count -= used; - let close = delim_mut(&mut slots, closer); + slots.delim_mut(opener).count -= used; + let close = slots.delim_mut(closer); close.start += used; close.count -= used; // Delimiters inside the new node cannot open anything else. stack.truncate(at + 1); - if delim(&slots, opener).count == 0 { - unlink(&mut slots, opener); + if slots.delim(opener).count == 0 { + slots.unlink(opener); stack.pop(); } - if delim(&slots, closer).count == 0 { - let next = slots[closer].next; - unlink(&mut slots, closer); - closer = next; - } - } - - let mut index = slots[0].next; - while index != NIL { - items.extend(slots[index].item.take()); - index = slots[index].next; + cursor = if slots.delim(closer).count == 0 { + let next = slots.next(closer); + slots.unlink(closer); + next + } else { + Some(closer) + }; } -} -/// Whether the node for the items between `opener` and `closer` stays within the nesting limit. -fn fits(slots: &[Slot], opener: usize, closer: usize) -> bool { - let mut depth = 0; - let mut index = slots[opener].next; - while index != closer { - if let Some(item) = &slots[index].item { - depth = depth.max(item_depth(item)); - } - index = slots[index].next; - } - depth < MAX_NESTING + slots.move_into(items); } From 54d889971dbb94e1bd86c2e74d5b49c999c2dc0b Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Thu, 8 Oct 2026 13:09:05 +0900 Subject: [PATCH 41/88] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor(markdown):?= =?UTF-8?q?=20share=20byte=20scanning=20helpers=20and=20fix=20stale=20pars?= =?UTF-8?q?er=20docs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Move eol_len, skip_blanks, skip_whitespace and the blanks-with-one-line-ending skip into parser/scan.rs instead of a copy per module. - Name the footnote continuation indent separately from the code indent, and derive Line::code_indent from CODE_INDENT. - Pass the parsed marker of the next item to list instead of parsing it again behind an unreachable!. - Use FxHashMap for the span run and brace tables like the rest of the parser. - Fix the block module doc, which still listed constructs as unsupported, and the dangling paragraph doc; drop notes about markdown-rs differences. --- crates/mq-markdown/src/parser.rs | 1 + crates/mq-markdown/src/parser/block.rs | 35 +++++---- crates/mq-markdown/src/parser/definition.rs | 33 +-------- crates/mq-markdown/src/parser/html_flow.rs | 9 +-- crates/mq-markdown/src/parser/inline.rs | 26 +++---- .../mq-markdown/src/parser/inline/emphasis.rs | 4 +- crates/mq-markdown/src/parser/inline/html.rs | 23 ++---- crates/mq-markdown/src/parser/inline/link.rs | 2 +- crates/mq-markdown/src/parser/inline/tail.rs | 28 ++------ crates/mq-markdown/src/parser/line.rs | 3 +- crates/mq-markdown/src/parser/mdx.rs | 8 +-- crates/mq-markdown/src/parser/scan.rs | 71 +++++++++++++++++++ crates/mq-markdown/src/parser/tree.rs | 18 ++--- 13 files changed, 135 insertions(+), 126 deletions(-) create mode 100644 crates/mq-markdown/src/parser/scan.rs diff --git a/crates/mq-markdown/src/parser.rs b/crates/mq-markdown/src/parser.rs index 73e26f623..4f7ee5560 100644 --- a/crates/mq-markdown/src/parser.rs +++ b/crates/mq-markdown/src/parser.rs @@ -18,6 +18,7 @@ mod mdx; mod mdx_flow; mod render_html; mod resolve; +mod scan; mod table; mod tree; diff --git a/crates/mq-markdown/src/parser/block.rs b/crates/mq-markdown/src/parser/block.rs index df745c78b..5e200b3f1 100644 --- a/crates/mq-markdown/src/parser/block.rs +++ b/crates/mq-markdown/src/parser/block.rs @@ -1,8 +1,8 @@ -//! Block structure parsing: resolves each line into leaf blocks. +//! Block structure parsing: resolves the lines into a tree of [`Block`]s. //! -//! Containers (blockquote, list) are resolved by stripping their prefix from each line and parsing -//! the remaining lines again, up to [`MAX_DEPTH`] levels. The remaining leaf blocks (HTML, table, -//! definition, frontmatter, math) are not handled yet, so their lines currently fall into paragraphs. +//! The constructs that can start at a line are the rules of [`rule::Rules`], tried in order, and a +//! paragraph when none starts. Containers (blockquote, list item, footnote definition) collect their +//! lines without the container prefix and parse them again, up to [`MAX_DEPTH`] levels. use super::code::Fence; use super::definition; @@ -20,6 +20,9 @@ use rule::{BlockRule, Cx, Rules, fallback}; /// Lines indented by this many columns or more are code, not other blocks. pub(super) const CODE_INDENT: usize = 4; +/// The indentation of the lines that continue a footnote definition. +const FOOTNOTE_INDENT: usize = 4; + /// Containers nested deeper than this are parsed as plain paragraph text, which bounds recursion. const MAX_DEPTH: usize = 128; @@ -351,6 +354,7 @@ fn blockquote( index } +#[derive(Clone)] struct ItemMarker { ordered: bool, kind: ListMarker, @@ -451,9 +455,9 @@ fn list( let mut items = Vec::new(); let mut spread = false; let mut index = start; + let mut marker = first.clone(); loop { - let marker = ItemMarker::parse(&lines[index]).unwrap_or_else(|| unreachable!("checked by the caller")); let (item, end) = list_item( lines, index, @@ -468,11 +472,16 @@ fn list( items.push(item); // Blank lines between items make the list loose. - let next = (end..lines.len()).find(|&i| !lines[i].is_blank()); - match next.filter(|&i| ItemMarker::parse(&lines[i]).is_some_and(|m| m.same_list(first))) { - Some(next) => { + let next = (end..lines.len()).find(|&i| !lines[i].is_blank()).and_then(|i| { + ItemMarker::parse(&lines[i]) + .filter(|m| m.same_list(first)) + .map(|m| (i, m)) + }); + match next { + Some((next, next_marker)) => { spread |= next > end; index = next; + marker = next_marker; } None => { index = end; @@ -808,8 +817,8 @@ fn ends_table(line: &Line<'_>) -> bool { || (!line.lazy && line.indent().0 < line.code_indent() && ItemMarker::parse(line).is_some()) } -/// Returns the index after the block and what it restricts on the first line of a following container. -/// +/// Collects a paragraph or a setext heading starting at `lines[start]`, with the definitions at its +/// start as blocks of their own. Returns the index after it and what it restricts on the next line. fn paragraph(lines: &[Line<'_>], start: usize, blocks: &mut Vec<Block>) -> (usize, Interrupt) { let mut index = start + 1; // Lines up to this one are taken by flow content that turned out to be text. @@ -936,7 +945,7 @@ fn footnote_marker<'a>(line: &Line<'a>) -> Option<FootnoteMarker<'a>> { } let (columns, indent) = line.indent(); let rest = line.text[indent..].strip_prefix("[^")?; - if columns >= CODE_INDENT { + if columns >= line.code_indent() { return None; } // A backslash escapes the bracket that follows it, as well as another backslash. @@ -986,12 +995,12 @@ fn footnote( while let Some(line) = lines.get(index) { if line.is_blank() { blanks.push(line.skip(line.text.len())); - } else if line.indent().0 >= CODE_INDENT { + } else if line.indent().0 >= FOOTNOTE_INDENT { for blank in &blanks { state.feed(blank); } inner.append(&mut blanks); - let stripped = line.skip_columns(CODE_INDENT); + let stripped = line.skip_columns(FOOTNOTE_INDENT); state.feed(&stripped); inner.push(stripped); } else if blanks.is_empty() && state.continues_with(line) && !blocks_lazy_continuation(lines, index) { diff --git a/crates/mq-markdown/src/parser/definition.rs b/crates/mq-markdown/src/parser/definition.rs index 6de759d4b..01de568e8 100644 --- a/crates/mq-markdown/src/parser/definition.rs +++ b/crates/mq-markdown/src/parser/definition.rs @@ -1,37 +1,10 @@ //! Link reference definitions: `[label]: destination "title"` at the start of a paragraph. use super::inline::{destination, normalize, remove_line_indent, title_at, unescape}; +use super::scan::{eol_len, skip_blanks, skip_blanks_and_eol}; use super::tree::InlineSource; use crate::node::{Definition, Node, Point, Position, Title, Url}; -/// Skips spaces and tabs. -fn skip_blanks(bytes: &[u8], mut index: usize) -> usize { - while matches!(bytes.get(index), Some(b' ' | b'\t')) { - index += 1; - } - index -} - -/// The length of the line ending at `index`, or 0. -fn eol_len(bytes: &[u8], index: usize) -> usize { - match bytes.get(index..) { - Some([b'\r', b'\n', ..]) => 2, - Some([b'\r' | b'\n', ..]) => 1, - _ => 0, - } -} - -/// Skips whitespace including at most one line ending. -fn skip_space(bytes: &[u8], index: usize) -> usize { - let index = skip_blanks(bytes, index); - let eol = eol_len(bytes, index); - if eol > 0 { - skip_blanks(bytes, index + eol) - } else { - index - } -} - /// Parses the label at `pos` (a `[`) followed by `:`, returning the raw label and the offset after. fn label(text: &str, pos: usize) -> Option<(&str, usize)> { let bytes = text.as_bytes(); @@ -68,7 +41,7 @@ fn parse_one(text: &str, pos: usize) -> Option<Parsed> { let bytes = text.as_bytes(); let (raw_label, after_colon) = label(text, pos)?; - let dest_start = skip_space(bytes, after_colon); + let dest_start = skip_blanks_and_eol(bytes, after_colon); let (url, after_dest) = destination(text, dest_start)?; // A bare destination cannot be empty. if after_dest == dest_start { @@ -107,7 +80,7 @@ fn parse_one(text: &str, pos: usize) -> Option<Parsed> { /// The start of a title after the destination: whitespace is required before it. fn title_start(bytes: &[u8], after_dest: usize) -> Option<usize> { - let start = skip_space(bytes, after_dest); + let start = skip_blanks_and_eol(bytes, after_dest); (start > after_dest).then_some(start) } diff --git a/crates/mq-markdown/src/parser/html_flow.rs b/crates/mq-markdown/src/parser/html_flow.rs index be65e3256..51638b689 100644 --- a/crates/mq-markdown/src/parser/html_flow.rs +++ b/crates/mq-markdown/src/parser/html_flow.rs @@ -1,5 +1,7 @@ //! HTML blocks: the seven kinds of `CommonMark`, ported from the rules of `markdown-rs`. +use super::scan::skip_blanks; + /// The kind of an HTML block, which decides how it ends. #[derive(Clone, Copy, PartialEq)] pub(super) enum Kind { @@ -135,13 +137,6 @@ fn tag(rest: &str, from: usize, closing: bool) -> Option<Kind> { } } -fn skip_blanks(bytes: &[u8], mut index: usize) -> usize { - while matches!(bytes.get(index), Some(b' ' | b'\t')) { - index += 1; - } - index -} - /// Whether the rest of a tag after its name is complete and alone on the line. fn complete_tag(bytes: &[u8], closing: bool) -> bool { let mut index = 0; diff --git a/crates/mq-markdown/src/parser/inline.rs b/crates/mq-markdown/src/parser/inline.rs index 32f0ba7ab..f007280a0 100644 --- a/crates/mq-markdown/src/parser/inline.rs +++ b/crates/mq-markdown/src/parser/inline.rs @@ -22,14 +22,15 @@ pub(super) use tail::{destination, title_at}; use super::mdx::{self, Fallback, Parsed, TagKind}; use super::resolve::References; +use super::scan::eol_len; use super::tree::InlineSource; use crate::node::{ Break, CodeInline, MathInline, MdxAttributeContent, MdxJsxTextElement, MdxTextExpression, Node, Position, Text, }; +use rustc_hash::FxHashMap; use smol_str::SmolStr; use std::borrow::Cow; use std::cell::{OnceCell, RefCell}; -use std::collections::HashMap; /// The value of a text item: a slice of the source, or a decoded string. pub(super) enum Value { @@ -85,8 +86,8 @@ pub(super) struct TextTag { end: usize, } -/// Emphasis and links nested deeper than this are text, and JSX elements are an error, which bounds the recursion of everything that -/// walks the nodes. +/// Emphasis and links nested deeper than this are text, and JSX elements are an error. This bounds the +/// recursion of everything that walks the nodes. pub(super) const MAX_NESTING: usize = 128; /// How deeply the item nests: nodes carry it, and the rest are text. @@ -292,8 +293,8 @@ fn text_element(context: &Context<'_>, tag: TextTag, children: Vec<Node>, end: u /// span asks, so that openers without a closer do not each scan to the end. #[derive(Default)] struct SpanRuns { - backticks: OnceCell<HashMap<usize, Vec<usize>>>, - dollars: OnceCell<HashMap<usize, Vec<usize>>>, + backticks: OnceCell<FxHashMap<usize, Vec<usize>>>, + dollars: OnceCell<FxHashMap<usize, Vec<usize>>>, } impl SpanRuns { @@ -303,7 +304,7 @@ impl SpanRuns { let starts = runs .get_or_init(|| { let bytes = src.as_bytes(); - let mut runs: HashMap<usize, Vec<usize>> = HashMap::new(); + let mut runs: FxHashMap<usize, Vec<usize>> = FxHashMap::default(); let mut index = 0; while index < bytes.len() { if bytes[index] == ch { @@ -509,7 +510,7 @@ impl Scanner<'_> { self.push(item, pos + 2); } Some(b'\n' | b'\r') => { - let end = pos + 1 + eol_len(&bytes[pos + 1..]); + let end = pos + 1 + eol_len(bytes, pos + 1); let node = Node::Break(Break { position: Some(self.context.position(pos, end)), }); @@ -542,7 +543,7 @@ impl Scanner<'_> { let src = self.src(); let bytes = src.as_bytes(); let pos = self.pos; - let eol = eol_len(&bytes[pos..]); + let eol = eol_len(bytes, pos); // Trailing whitespace before the line ending is dropped, or makes a hard break. let mut start = pos; @@ -758,15 +759,6 @@ impl Scanner<'_> { } } -/// The length of the line ending at the start of `bytes`. -fn eol_len(bytes: &[u8]) -> usize { - match bytes { - [b'\r', b'\n', ..] => 2, - [b'\r' | b'\n', ..] => 1, - _ => 0, - } -} - /// The value of a code or math span: the content as it is, except that one space or line ending is /// removed from both ends when both have one and the content is not only spaces and line endings. fn span_value(content: &str) -> SmolStr { diff --git a/crates/mq-markdown/src/parser/inline/emphasis.rs b/crates/mq-markdown/src/parser/inline/emphasis.rs index d5f058e29..634436f62 100644 --- a/crates/mq-markdown/src/parser/inline/emphasis.rs +++ b/crates/mq-markdown/src/parser/inline/emphasis.rs @@ -1,7 +1,7 @@ //! Emphasis, strong emphasis and strikethrough, resolved with a stack of delimiter runs. //! -//! This follows the reference implementation of `CommonMark`. `markdown-rs` differs in how runs next -//! to other runs open and close, and in using the remaining length of a run for the rule of three. +//! This follows the reference implementation of `CommonMark`: the rule of three goes by the length of +//! the whole runs. use super::punctuation::is_punctuation; use super::{Context, Delim, Item, MAX_NESTING, item_depth, to_nodes}; diff --git a/crates/mq-markdown/src/parser/inline/html.rs b/crates/mq-markdown/src/parser/inline/html.rs index 9a01cef3f..56d250baf 100644 --- a/crates/mq-markdown/src/parser/inline/html.rs +++ b/crates/mq-markdown/src/parser/inline/html.rs @@ -1,5 +1,7 @@ //! Autolinks (`<https://example.com>`) and raw inline HTML. +use crate::parser::scan::{is_whitespace, skip_whitespace}; + /// An autolink found at `<`. pub(super) struct Autolink { pub(super) url: String, @@ -142,10 +144,6 @@ pub(super) fn inline_html(src: &str, pos: usize, misses: &mut Misses) -> Option< open_tag(bytes).map(|length| pos + length) } -fn is_space(byte: u8) -> bool { - matches!(byte, b' ' | b'\t' | b'\n' | b'\r') -} - /// The length of a tag name at the start of `bytes`. fn tag_name(bytes: &[u8]) -> usize { if !bytes.first().is_some_and(u8::is_ascii_alphabetic) { @@ -157,20 +155,13 @@ fn tag_name(bytes: &[u8]) -> usize { .count() } -fn skip_spaces(bytes: &[u8], mut index: usize) -> usize { - while bytes.get(index).copied().is_some_and(is_space) { - index += 1; - } - index -} - /// The length of a closing tag such as `</a>` at the start of `bytes`. fn closing_tag(bytes: &[u8]) -> Option<usize> { let name = tag_name(&bytes[2..]); if name == 0 { return None; } - let index = skip_spaces(bytes, 2 + name); + let index = skip_whitespace(bytes, 2 + name); (bytes.get(index) == Some(&b'>')).then_some(index + 1) } @@ -183,7 +174,7 @@ fn open_tag(bytes: &[u8]) -> Option<usize> { let mut index = 1 + name; loop { - let after_space = skip_spaces(bytes, index); + let after_space = skip_whitespace(bytes, index); let has_space = after_space > index; match bytes.get(after_space)? { b'>' => return Some(after_space + 1), @@ -204,11 +195,11 @@ fn attribute(bytes: &[u8], start: usize) -> Option<usize> { .count(); let end = start + name; - let after_space = skip_spaces(bytes, end); + let after_space = skip_whitespace(bytes, end); if bytes.get(after_space) != Some(&b'=') { return Some(end); } - let value = skip_spaces(bytes, after_space + 1); + let value = skip_whitespace(bytes, after_space + 1); match bytes.get(value)? { quote @ (b'"' | b'\'') => { let close = bytes[value + 1..].iter().position(|b| b == quote)?; @@ -217,7 +208,7 @@ fn attribute(bytes: &[u8], start: usize) -> Option<usize> { _ => { let length = bytes[value..] .iter() - .take_while(|&&b| !is_space(b) && !matches!(b, b'"' | b'\'' | b'=' | b'<' | b'>' | b'`')) + .take_while(|&&b| !is_whitespace(b) && !matches!(b, b'"' | b'\'' | b'=' | b'<' | b'>' | b'`')) .count(); (length > 0).then_some(value + length) } diff --git a/crates/mq-markdown/src/parser/inline/link.rs b/crates/mq-markdown/src/parser/inline/link.rs index 2cd213af7..4b9142217 100644 --- a/crates/mq-markdown/src/parser/inline/link.rs +++ b/crates/mq-markdown/src/parser/inline/link.rs @@ -8,7 +8,7 @@ use crate::node::RenderOptions; use crate::node::{FootnoteRef, Image, ImageRef, Link, LinkRef, Node, Position, Text, Title, Url}; /// Normalizes a reference label: whitespace runs become one space, the ends are trimmed, and the case -/// is folded. `markdown-rs` drops the first gap between words, which this does not. +/// is folded. pub(crate) fn normalize(label: &str) -> String { let bytes = label.as_bytes(); let mut result = String::with_capacity(label.len()); diff --git a/crates/mq-markdown/src/parser/inline/tail.rs b/crates/mq-markdown/src/parser/inline/tail.rs index 20197b581..f60eb604d 100644 --- a/crates/mq-markdown/src/parser/inline/tail.rs +++ b/crates/mq-markdown/src/parser/inline/tail.rs @@ -3,6 +3,7 @@ use super::Context; use super::entity; use super::link::normalize; +use crate::parser::scan::{eol_len, skip_blanks_and_eol}; /// The destination and title of an inline link, and the offset after its closing parenthesis. pub(super) struct Tail { @@ -61,25 +62,6 @@ fn parse_label(src: &str, pos: usize) -> Option<(usize, &str)> { None } -/// Skips whitespace including at most one line ending. -fn skip_space(bytes: &[u8], mut index: usize) -> usize { - let mut line_endings = 0; - while let Some(&byte) = bytes.get(index) { - match byte { - b' ' | b'\t' => index += 1, - b'\n' | b'\r' => { - line_endings += 1; - if line_endings > 1 { - break; - } - index += super::eol_len(&bytes[index..]); - } - _ => break, - } - } - index -} - /// Parses `(destination "title")` at `pos`. pub(super) fn inline_tail(src: &str, pos: usize) -> Option<Tail> { let bytes = src.as_bytes(); @@ -87,10 +69,10 @@ pub(super) fn inline_tail(src: &str, pos: usize) -> Option<Tail> { return None; } - let index = skip_space(bytes, pos + 1); + let index = skip_blanks_and_eol(bytes, pos + 1); let (url, after_url) = destination(src, index)?; - let after_space = skip_space(bytes, after_url); + let after_space = skip_blanks_and_eol(bytes, after_url); if bytes.get(after_space) == Some(&b')') { return Some(Tail { url, @@ -104,7 +86,7 @@ pub(super) fn inline_tail(src: &str, pos: usize) -> Option<Tail> { } let (title, after_title) = title_at(src, after_space)?; - let closing = skip_space(bytes, after_title); + let closing = skip_blanks_and_eol(bytes, after_title); (bytes.get(closing) == Some(&b')')).then_some(Tail { url, title: Some(title), @@ -179,7 +161,7 @@ pub(in crate::parser) fn title_at(src: &str, pos: usize) -> Option<(String, usiz b'\\' => index += 1, b'\n' | b'\r' => { // A title cannot contain a blank line. - let length = super::eol_len(&bytes[index..]); + let length = eol_len(bytes, index); let mut next = index + length; while matches!(bytes.get(next), Some(b' ' | b'\t')) { next += 1; diff --git a/crates/mq-markdown/src/parser/line.rs b/crates/mq-markdown/src/parser/line.rs index eb7720864..c122bc9e8 100644 --- a/crates/mq-markdown/src/parser/line.rs +++ b/crates/mq-markdown/src/parser/line.rs @@ -1,5 +1,6 @@ //! Source lines and their positions. +use super::block::CODE_INDENT; use crate::node::Point; #[derive(Clone, Copy)] @@ -34,7 +35,7 @@ pub(super) struct Line<'a> { impl<'a> Line<'a> { /// The indentation from which a line is code. Without indented code, indentation never matters. pub(super) fn code_indent(&self) -> usize { - if self.mdx { usize::MAX } else { 4 } + if self.mdx { usize::MAX } else { CODE_INDENT } } /// The zero-based visual column where `text` starts in the document. diff --git a/crates/mq-markdown/src/parser/mdx.rs b/crates/mq-markdown/src/parser/mdx.rs index 09e09356a..7408244c5 100644 --- a/crates/mq-markdown/src/parser/mdx.rs +++ b/crates/mq-markdown/src/parser/mdx.rs @@ -2,9 +2,9 @@ //! so expressions are only checked for balanced braces, outside of strings, template literals and comments. use crate::node::{MdxAttributeContent, MdxAttributeValue, MdxJsxAttribute}; +use rustc_hash::FxHashMap; use smol_str::SmolStr; use std::cell::OnceCell; -use std::collections::HashMap; /// What the end of the input means for a construct it ends inside of. pub(super) enum Fallback { @@ -65,7 +65,7 @@ const UNCLOSED_EXPRESSION: &str = /// Where each `{` of a source closes, found in one pass the first time one is asked for, so that /// many unclosed braces do not each scan to the end of the input. #[derive(Default)] -pub(super) struct Braces(OnceCell<HashMap<usize, usize>>); +pub(super) struct Braces(OnceCell<FxHashMap<usize, usize>>); impl Braces { /// The offset of the `}` that closes the `{` at `open`, if there is one. @@ -83,9 +83,9 @@ enum Frame { /// Matches the braces of the expressions in `src` from `from`, which are not checked as JavaScript but /// skip over strings, template literals and comments. Text outside braces is not looked at. With /// `single`, `from` is a `{` and the search ends once it is closed. -fn match_braces(src: &str, from: usize, single: bool) -> HashMap<usize, usize> { +fn match_braces(src: &str, from: usize, single: bool) -> FxHashMap<usize, usize> { let bytes = src.as_bytes(); - let mut closes = HashMap::new(); + let mut closes = FxHashMap::default(); let mut stack = Vec::new(); let mut index = from; diff --git a/crates/mq-markdown/src/parser/scan.rs b/crates/mq-markdown/src/parser/scan.rs new file mode 100644 index 000000000..0a66d0b56 --- /dev/null +++ b/crates/mq-markdown/src/parser/scan.rs @@ -0,0 +1,71 @@ +//! Byte-level helpers shared by the block and inline parsers. + +/// The length of the line ending at `index`, or 0 when there is none. +pub(super) fn eol_len(bytes: &[u8], index: usize) -> usize { + match bytes.get(index..) { + Some([b'\r', b'\n', ..]) => 2, + Some([b'\r' | b'\n', ..]) => 1, + _ => 0, + } +} + +/// Whether `byte` is a space, a tab or a line ending. +pub(super) fn is_whitespace(byte: u8) -> bool { + matches!(byte, b' ' | b'\t' | b'\n' | b'\r') +} + +/// The offset after the spaces and tabs at `index`. +pub(super) fn skip_blanks(bytes: &[u8], mut index: usize) -> usize { + while matches!(bytes.get(index), Some(b' ' | b'\t')) { + index += 1; + } + index +} + +/// The offset after the spaces, tabs and line endings at `index`. +pub(super) fn skip_whitespace(bytes: &[u8], mut index: usize) -> usize { + while bytes.get(index).copied().is_some_and(is_whitespace) { + index += 1; + } + index +} + +/// The offset after the spaces and tabs at `index`, including at most one line ending among them. +pub(super) fn skip_blanks_and_eol(bytes: &[u8], index: usize) -> usize { + let index = skip_blanks(bytes, index); + match eol_len(bytes, index) { + 0 => index, + eol => skip_blanks(bytes, index + eol), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use rstest::rstest; + + #[rstest] + #[case::none("a", 0, 0)] + #[case::lf("a\nb", 1, 1)] + #[case::cr("a\rb", 1, 1)] + #[case::crlf("a\r\nb", 1, 2)] + #[case::past_end("a", 5, 0)] + fn eol_len_at(#[case] text: &str, #[case] index: usize, #[case] expected: usize) { + assert_eq!(eol_len(text.as_bytes(), index), expected); + } + + #[rstest] + #[case::blanks(" \t a", 0, 3)] + #[case::one_eol(" \n a", 0, 3)] + #[case::crlf(" \r\n\ta", 0, 4)] + #[case::two_eols(" \n\n a", 0, 2)] + #[case::nothing("a", 0, 0)] + fn skip_blanks_and_eol_at(#[case] text: &str, #[case] index: usize, #[case] expected: usize) { + assert_eq!(skip_blanks_and_eol(text.as_bytes(), index), expected); + } + + #[test] + fn skip_whitespace_crosses_line_endings() { + assert_eq!(skip_whitespace(b" \n\r\n\ta", 0), 5); + } +} diff --git a/crates/mq-markdown/src/parser/tree.rs b/crates/mq-markdown/src/parser/tree.rs index a80a922f4..ff04abd93 100644 --- a/crates/mq-markdown/src/parser/tree.rs +++ b/crates/mq-markdown/src/parser/tree.rs @@ -5,6 +5,7 @@ use super::line::visual_column; use super::mdx::TagKind; +use super::scan::eol_len; use crate::node::{ListMarker, MdxAttributeContent, Node, Point, Position, TableAlignKind}; /// Raw inline content of a paragraph, heading or table cell, with the source position of each line. @@ -43,18 +44,11 @@ impl InlineSource { /// Removes the first `marker` bytes and the one space, tab or line ending that follows them, and the /// line ending after a space or a tab. pub(super) fn remove_prefix_and_one(&mut self, marker: usize) { - let eol = |rest: &str| { - if rest.starts_with("\r\n") { - 2 - } else { - usize::from(rest.starts_with(['\n', '\r'])) - } - }; - let rest = &self.text[marker..]; - let one = if rest.starts_with([' ', '\t']) { - 1 + eol(&rest[1..]) + let bytes = self.text.as_bytes(); + let one = if matches!(bytes.get(marker), Some(b' ' | b'\t')) { + 1 + eol_len(bytes, marker + 1) } else { - eol(rest) + eol_len(bytes, marker) }; self.drain_to(marker + one); } @@ -67,7 +61,7 @@ impl InlineSource { return std::mem::take(&mut self.text); }; let line = self.text[..eol].to_string(); - let end = eol + if self.text[eol..].starts_with("\r\n") { 2 } else { 1 }; + let end = eol + eol_len(self.text.as_bytes(), eol); self.drain_to(end); line } From e53489a4bc923e4d0c00d91b2f38a333faf55aa5 Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Thu, 8 Oct 2026 13:09:11 +0900 Subject: [PATCH 42/88] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor(markdown):?= =?UTF-8?q?=20read=20the=20dialect=20from=20a=20Flavor=20instead=20of=20md?= =?UTF-8?q?x=20flags?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Line and References each carried an mdx bool, and call sites negated it to mean things such as "math is allowed" (Fence::open(rest, !line.mdx)). A Flavor enum now says which constructs a document has: indented code, HTML, GFM, math, Obsidian syntax, JSX, and whether spans drop line indentation. Inline parsing receives the flavor with the references in a Document, instead of finding it inside the reference table. --- crates/mq-markdown/src/parser.rs | 8 +++- crates/mq-markdown/src/parser/block.rs | 17 +++---- crates/mq-markdown/src/parser/block/rule.rs | 6 +-- crates/mq-markdown/src/parser/flavor.rs | 50 ++++++++++++++++++++ crates/mq-markdown/src/parser/inline.rs | 36 +++++++++----- crates/mq-markdown/src/parser/line.rs | 17 ++++--- crates/mq-markdown/src/parser/mdx_flow.rs | 4 +- crates/mq-markdown/src/parser/render_html.rs | 17 +++++-- crates/mq-markdown/src/parser/resolve.rs | 47 +++++++++--------- crates/mq-markdown/src/parser/table.rs | 2 +- 10 files changed, 143 insertions(+), 61 deletions(-) create mode 100644 crates/mq-markdown/src/parser/flavor.rs diff --git a/crates/mq-markdown/src/parser.rs b/crates/mq-markdown/src/parser.rs index 4f7ee5560..7bc24d042 100644 --- a/crates/mq-markdown/src/parser.rs +++ b/crates/mq-markdown/src/parser.rs @@ -11,6 +11,7 @@ mod block; mod callout; mod code; mod definition; +mod flavor; mod html_flow; mod inline; mod line; @@ -23,6 +24,7 @@ mod table; mod tree; use crate::node::Node; +use flavor::Flavor; pub(crate) use inline::{is_autolink_email, normalize, unescape}; @@ -34,7 +36,8 @@ pub(crate) fn parse(content: &str) -> miette::Result<Vec<Node>> { /// Parses `content` into a flat list of nodes, with frontmatter at the start when `frontmatter` is set. pub(crate) fn parse_with(content: &str, frontmatter: bool) -> miette::Result<Vec<Node>> { - resolve::resolve(block::parse(content, false, frontmatter), false).map_err(|message| miette::miette!(message)) + resolve::resolve(block::parse(content, Flavor::Markdown, frontmatter), Flavor::Markdown) + .map_err(|message| miette::miette!(message)) } /// Renders `content` as HTML. @@ -50,7 +53,8 @@ pub(crate) fn parse_mdx(content: &str) -> miette::Result<Vec<Node>> { /// Parses `content` as MDX, with frontmatter at the start when `frontmatter` is set. pub(crate) fn parse_mdx_with(content: &str, frontmatter: bool) -> miette::Result<Vec<Node>> { - resolve::resolve(block::parse(content, true, frontmatter), true).map_err(|message| miette::miette!(message)) + resolve::resolve(block::parse(content, Flavor::Mdx, frontmatter), Flavor::Mdx) + .map_err(|message| miette::miette!(message)) } #[cfg(test)] diff --git a/crates/mq-markdown/src/parser/block.rs b/crates/mq-markdown/src/parser/block.rs index 5e200b3f1..08aa953fe 100644 --- a/crates/mq-markdown/src/parser/block.rs +++ b/crates/mq-markdown/src/parser/block.rs @@ -6,6 +6,7 @@ use super::code::Fence; use super::definition; +use super::flavor::Flavor; use super::html_flow::{self, Kind as HtmlKind}; use super::inline; use super::line::{Line, split_lines}; @@ -35,8 +36,8 @@ struct Interrupt { list: bool, } -pub(super) fn parse(src: &str, mdx: bool, read_frontmatter: bool) -> Vec<Block> { - let mut lines = split_lines(src, mdx); +pub(super) fn parse(src: &str, flavor: Flavor, read_frontmatter: bool) -> Vec<Block> { + let mut lines = split_lines(src, flavor); // A byte order mark at the start of the document is not content. if let Some(first) = lines.first_mut().filter(|line| line.text.starts_with('\u{feff}')) { *first = first.skip('\u{feff}'.len_utf8()); @@ -227,7 +228,7 @@ impl<'a> LeafState<'a> { } return; } - if let Some(fence) = Fence::open(rest, !line.mdx) { + if let Some(fence) = Fence::open(rest, line.flavor.has_math()) { self.fence = Some(fence); self.paragraph = false; return; @@ -664,7 +665,7 @@ fn list_item( /// Recognises a GFM task marker (`[ ] ` or `[x] `) at the start of an item's content, returning /// the checked state and the bytes to skip. fn task_checkbox(line: &Line<'_>, next: Option<&Line<'_>>) -> Option<bool> { - if line.mdx { + if !line.flavor.has_gfm() { return None; } let bytes = line.text.as_bytes(); @@ -801,7 +802,7 @@ fn interrupts_paragraph(line: &Line<'_>) -> bool { let (columns, indent) = line.indent(); let rest = &line.text[indent..]; columns < line.code_indent() - && (Fence::open(rest, !line.mdx).is_some() + && (Fence::open(rest, line.flavor.has_math()).is_some() || atx_depth(rest).is_some() || is_thematic_break(line, indent) || blockquote_marker(line).is_some() @@ -940,7 +941,7 @@ struct FootnoteMarker<'a> { } fn footnote_marker<'a>(line: &Line<'a>) -> Option<FootnoteMarker<'a>> { - if line.mdx { + if !line.flavor.has_gfm() { return None; } let (columns, indent) = line.indent(); @@ -1094,7 +1095,7 @@ fn html_block(lines: &[Line<'_>], start: usize, kind: HtmlKind, blocks: &mut Vec last + 1 } -/// The kind of HTML block that starts at `rest`; MDX has none. +/// The kind of HTML block that starts at `rest`, in a flavor that has HTML. fn html_start(line: &Line<'_>, rest: &str) -> Option<HtmlKind> { - if line.mdx { None } else { html_flow::start(rest) } + line.flavor.has_html().then(|| html_flow::start(rest)).flatten() } diff --git a/crates/mq-markdown/src/parser/block/rule.rs b/crates/mq-markdown/src/parser/block/rule.rs index ad93678b6..6f99f1e3a 100644 --- a/crates/mq-markdown/src/parser/block/rule.rs +++ b/crates/mq-markdown/src/parser/block/rule.rs @@ -121,7 +121,7 @@ pub(super) struct FencedCode; impl BlockRule for FencedCode { fn parse(cx: &Cx<'_, '_>, blocks: &mut Vec<Block>) -> Option<Step> { - let fence = Fence::open(cx.rest, !cx.line.mdx)?; + let fence = Fence::open(cx.rest, cx.line.flavor.has_math())?; // The end of an unclosed fence without content is quirky right after a container. let own_end = cx.closed_container && !(cx.separated @@ -259,7 +259,7 @@ pub(super) struct Esm; impl BlockRule for Esm { fn parse(cx: &Cx<'_, '_>, blocks: &mut Vec<Block>) -> Option<Step> { let line = cx.line; - if !(line.mdx + if !(line.flavor.has_jsx() && cx.depth == 0 && cx.indent == 0 && (line.text.starts_with("import ") || line.text.starts_with("export "))) @@ -296,7 +296,7 @@ pub(super) struct MdxFlow; impl BlockRule for MdxFlow { fn parse(cx: &Cx<'_, '_>, blocks: &mut Vec<Block>) -> Option<Step> { - if !(cx.line.mdx && matches!(cx.rest.as_bytes().first(), Some(b'<' | b'{'))) { + if !(cx.line.flavor.has_jsx() && matches!(cx.rest.as_bytes().first(), Some(b'<' | b'{'))) { return None; } let FlowOutcome::Flow(next) = mdx_flow(cx.lines, cx.index, blocks) else { diff --git a/crates/mq-markdown/src/parser/flavor.rs b/crates/mq-markdown/src/parser/flavor.rs new file mode 100644 index 000000000..14525eddc --- /dev/null +++ b/crates/mq-markdown/src/parser/flavor.rs @@ -0,0 +1,50 @@ +//! The dialect a document is read in, and the constructs each has. + +/// The dialect a document is read in. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(super) enum Flavor { + /// `CommonMark` with GFM, math, frontmatter and the Obsidian syntax of the enabled features. + #[default] + Markdown, + /// MDX: `CommonMark` without indented code, HTML and autolinks, but with JSX, expressions and ESM. + Mdx, +} + +impl Flavor { + /// Indented code blocks. + pub(super) fn has_indented_code(self) -> bool { + self == Self::Markdown + } + + /// HTML blocks, raw inline HTML and autolinks in angle brackets. + pub(super) fn has_html(self) -> bool { + self == Self::Markdown + } + + /// GFM: tables, task list items, footnotes, strikethrough and autolink literals. + pub(super) fn has_gfm(self) -> bool { + self == Self::Markdown + } + + /// Math blocks and inline math. + pub(super) fn has_math(self) -> bool { + self == Self::Markdown + } + + /// Wikilinks and embeds, when their features are enabled. + #[cfg(any(feature = "wikilink", feature = "embed"))] + pub(super) fn has_obsidian(self) -> bool { + self == Self::Markdown + } + + /// JSX tags, expressions and `import`/`export` statements. + pub(super) fn has_jsx(self) -> bool { + self == Self::Mdx + } + + /// Whether code and math spans drop the indentation of their continuation lines, as the lines of a + /// paragraph do. In MDX the content of a span is kept as written. + pub(super) fn strips_span_indent(self) -> bool { + self == Self::Markdown + } +} diff --git a/crates/mq-markdown/src/parser/inline.rs b/crates/mq-markdown/src/parser/inline.rs index f007280a0..e7a4e4ce5 100644 --- a/crates/mq-markdown/src/parser/inline.rs +++ b/crates/mq-markdown/src/parser/inline.rs @@ -20,6 +20,7 @@ pub(crate) use html::is_autolink_email; pub(crate) use link::normalize; pub(super) use tail::{destination, title_at}; +use super::flavor::Flavor; use super::mdx::{self, Fallback, Parsed, TagKind}; use super::resolve::References; use super::scan::eol_len; @@ -98,11 +99,19 @@ pub(super) fn item_depth(item: &Item) -> usize { } } +/// What inline parsing needs to know about the whole document. +#[derive(Clone, Copy)] +pub(super) struct Document<'a> { + pub(super) references: &'a References, + pub(super) flavor: Flavor, +} + /// What the scanner and its helpers share. #[derive(Clone, Copy)] pub(super) struct Context<'a> { pub(super) source: &'a InlineSource, pub(super) references: &'a References, + pub(super) flavor: Flavor, /// Whether email addresses in plain text are linked: not inside link text, and not when there is no `@` /// in the text, which is looked for once. pub(super) emails: bool, @@ -129,7 +138,7 @@ impl Context<'_> { } /// Parses raw inline content into nodes. -pub(super) fn parse(source: &InlineSource, references: &References) -> Result<Vec<Node>, String> { +pub(super) fn parse(source: &InlineSource, doc: Document<'_>) -> Result<Vec<Node>, String> { if source.text.is_empty() { return Ok(Vec::new()); } @@ -137,8 +146,9 @@ pub(super) fn parse(source: &InlineSource, references: &References) -> Result<Ve let error = RefCell::new(None); let context = Context { source, - references, - emails: !references.mdx && source.text.contains('@'), + references: doc.references, + flavor: doc.flavor, + emails: doc.flavor.has_gfm() && source.text.contains('@'), error: &error, }; let mut scanner = Scanner { @@ -384,17 +394,17 @@ impl Scanner<'_> { b'\\' => self.escape(), b'&' => self.entity(), b'`' => self.span(b'`'), - b'$' if !self.context.references.mdx => self.span(b'$'), + b'$' if self.context.flavor.has_math() => self.span(b'$'), b'*' | b'_' => self.delimiter(), - b'~' if !self.context.references.mdx => self.delimiter(), + b'~' if self.context.flavor.has_gfm() => self.delimiter(), b'[' => self.open(false), b'!' if image => self.open(true), b']' => self.close(), - b'<' if !self.context.references.mdx => self.angle(), + b'<' if self.context.flavor.has_html() => self.angle(), b'<' => self.jsx(), - b'{' if self.context.references.mdx => self.expression(), + b'{' if self.context.flavor.has_jsx() => self.expression(), b'\n' | b'\r' => self.line_ending(), - b'h' | b'H' | b'w' | b'W' if !self.context.references.mdx => self.literal_url(), + b'h' | b'H' | b'w' | b'W' if self.context.flavor.has_gfm() => self.literal_url(), _ => self.pos += 1, } } @@ -404,7 +414,7 @@ impl Scanner<'_> { /// offset where it ends. The target is not empty, and neither part holds a bracket or a line ending. #[cfg(any(feature = "wikilink", feature = "embed"))] fn obsidian_link(&self) -> Option<(Node, usize)> { - if self.context.references.mdx { + if !self.context.flavor.has_obsidian() { return None; } let src = self.src(); @@ -594,10 +604,10 @@ impl Scanner<'_> { // In a table cell, `\|` stands for `|` inside code as well. let content = &src[start + size..close]; // The lines of a paragraph lose their leading whitespace before the inline content is read. - let content = if self.context.references.mdx { - Cow::Borrowed(content) - } else { + let content = if self.context.flavor.strips_span_indent() { remove_line_indent(content) + } else { + Cow::Borrowed(content) }; let content = content.as_ref(); let value = if self.context.source.table && content.contains("\\|") { @@ -630,7 +640,7 @@ impl Scanner<'_> { let before = src[..start].chars().next_back(); let after = src[end..].chars().next(); - let (can_open, can_close) = emphasis::flanking(ch, before, after, !self.context.references.mdx); + let (can_open, can_close) = emphasis::flanking(ch, before, after, self.context.flavor.has_gfm()); if !can_open && !can_close { self.pos = end; return; diff --git a/crates/mq-markdown/src/parser/line.rs b/crates/mq-markdown/src/parser/line.rs index c122bc9e8..efeac1fff 100644 --- a/crates/mq-markdown/src/parser/line.rs +++ b/crates/mq-markdown/src/parser/line.rs @@ -1,6 +1,7 @@ //! Source lines and their positions. use super::block::CODE_INDENT; +use super::flavor::Flavor; use crate::node::Point; #[derive(Clone, Copy)] @@ -24,8 +25,8 @@ pub(super) struct Line<'a> { /// Columns left over from a tab that a container consumed only in part. They count as leading /// whitespace of `text`. pub(super) pad: usize, - /// Whether the document is MDX, which has no indented code, HTML, autolinks or GFM. - pub(super) mdx: bool, + /// The dialect of the document. + pub(super) flavor: Flavor, /// For `*`, `-` and `_`: one past the offset in `origin` of the last byte that is neither that /// character, nor a space or tab, or 0 if there is none. A thematic break is a run of one of them /// and whitespace to the end of the line, so this tells at once, from any offset, that it is not. @@ -35,7 +36,11 @@ pub(super) struct Line<'a> { impl<'a> Line<'a> { /// The indentation from which a line is code. Without indented code, indentation never matters. pub(super) fn code_indent(&self) -> usize { - if self.mdx { usize::MAX } else { CODE_INDENT } + if self.flavor.has_indented_code() { + CODE_INDENT + } else { + usize::MAX + } } /// The zero-based visual column where `text` starts in the document. @@ -187,7 +192,7 @@ fn last_others(text: &str) -> [usize; 3] { } /// Splits `src` into lines on `\n`, `\r\n` and `\r`. A trailing terminator does not add a line. -pub(super) fn split_lines(src: &str, mdx: bool) -> Vec<Line<'_>> { +pub(super) fn split_lines(src: &str, flavor: Flavor) -> Vec<Line<'_>> { let bytes = src.as_bytes(); let mut lines = Vec::new(); let (mut start, mut index) = (0, 0); @@ -213,7 +218,7 @@ pub(super) fn split_lines(src: &str, mdx: bool) -> Vec<Line<'_>> { origin: &src[start..end], tabs: src[start..end].contains('\t'), pad: 0, - mdx, + flavor, others: last_others(&src[start..end]), }); start = index; @@ -231,7 +236,7 @@ pub(super) fn split_lines(src: &str, mdx: bool) -> Vec<Line<'_>> { origin: &src[start..], tabs: src[start..].contains('\t'), pad: 0, - mdx, + flavor, others: last_others(&src[start..]), }); } diff --git a/crates/mq-markdown/src/parser/mdx_flow.rs b/crates/mq-markdown/src/parser/mdx_flow.rs index 41acd080d..58ccd3c04 100644 --- a/crates/mq-markdown/src/parser/mdx_flow.rs +++ b/crates/mq-markdown/src/parser/mdx_flow.rs @@ -174,7 +174,7 @@ pub(super) fn blocks_lazy_continuation(lines: &[Line<'_>], index: usize) -> bool pub(super) fn probe_mdx_flow(lines: &[Line<'_>], index: usize) -> Option<FlowOutcome> { let line = &lines[index]; - (line.mdx + (line.flavor.has_jsx() && !line.lazy && matches!( line.text.trim_start_matches([' ', '\t']).as_bytes().first(), @@ -185,7 +185,7 @@ pub(super) fn probe_mdx_flow(lines: &[Line<'_>], index: usize) -> Option<FlowOut /// Whether a single MDX line looks like flow content, for tracking what a paragraph can continue. pub(super) fn looks_like_mdx_flow(line: &Line<'_>) -> bool { - if !line.mdx + if !line.flavor.has_jsx() || !matches!( line.text.trim_start_matches([' ', '\t']).as_bytes().first(), Some(b'<' | b'{') diff --git a/crates/mq-markdown/src/parser/render_html.rs b/crates/mq-markdown/src/parser/render_html.rs index af950ecef..a28d649dd 100644 --- a/crates/mq-markdown/src/parser/render_html.rs +++ b/crates/mq-markdown/src/parser/render_html.rs @@ -4,7 +4,8 @@ //! The output is the same as that of `markdown-rs`, line endings included: blocks start on a line of //! their own, and the line ending that the source has after the last block is kept. -use super::inline; +use super::flavor::Flavor; +use super::inline::{self, Document}; use super::resolve::{self, References}; use super::tree::{Block, FencedBlock, InlineBlock, InlineKind, InlineSource, ListBlock, TableItem}; use crate::node::{Code, Node, TableAlignKind}; @@ -20,7 +21,7 @@ const SAFE_PROTOCOL_SRC: [&str; 2] = ["http", "https"]; /// Renders `content` as HTML. pub(super) fn render(content: &str) -> String { - let blocks = super::block::parse(content, false, true); + let blocks = super::block::parse(content, Flavor::Markdown, true); let mut references = References::default(); resolve::collect(&blocks, &mut references); @@ -183,6 +184,14 @@ struct Html { } impl Html { + /// The document that inline content is parsed in: the definitions of this one, as Markdown. + fn document(&self) -> Document<'_> { + Document { + references: &self.references, + flavor: Flavor::Markdown, + } + } + fn line_ending(&mut self) { self.out.push_str(self.eol); } @@ -304,7 +313,7 @@ impl Html { } fn inline_block(&mut self, block: &InlineBlock) { - let nodes = inline::parse(&block.source, &self.references).unwrap_or_default(); + let nodes = inline::parse(&block.source, self.document()).unwrap_or_default(); match &block.kind { InlineKind::Paragraph => { let tight = self.tight.last().copied().unwrap_or(false); @@ -418,7 +427,7 @@ impl Html { } self.out.push('>'); if let Some(Some(source)) = row.get(column) { - let nodes = inline::parse(source, &self.references).unwrap_or_default(); + let nodes = inline::parse(source, self.document()).unwrap_or_default(); self.inlines(&nodes); } self.out.push_str(&format!("</{tag}>")); diff --git a/crates/mq-markdown/src/parser/resolve.rs b/crates/mq-markdown/src/parser/resolve.rs index 6b079d944..91216c21d 100644 --- a/crates/mq-markdown/src/parser/resolve.rs +++ b/crates/mq-markdown/src/parser/resolve.rs @@ -1,7 +1,8 @@ //! Turns the block tree into nodes: parses the inline content, pairs JSX tags and flattens lists and //! tables. -use super::inline::{self, MAX_NESTING}; +use super::flavor::Flavor; +use super::inline::{self, Document, MAX_NESTING}; use super::mdx::TagKind; use super::tree::{Block, InlineBlock, InlineKind, JsxTag, ListBlock, QuoteBlock, TableItem}; use crate::node::{ @@ -16,19 +17,21 @@ pub(super) struct References { pub(super) definitions: FxHashSet<String>, /// Normalized labels of footnote definitions. pub(super) footnotes: FxHashSet<String>, - /// Whether the document is MDX. - pub(super) mdx: bool, } -pub(super) fn resolve(blocks: Vec<Block>, mdx: bool) -> Result<Vec<Node>, String> { - let mut references = References { - mdx, - ..References::default() - }; +pub(super) fn resolve(blocks: Vec<Block>, flavor: Flavor) -> Result<Vec<Node>, String> { + let mut references = References::default(); collect(&blocks, &mut references); let mut nodes = Vec::new(); - flatten(blocks, &references, &mut nodes)?; + flatten( + blocks, + Document { + references: &references, + flavor, + }, + &mut nodes, + )?; Ok(nodes) } @@ -104,7 +107,7 @@ fn element(tag: JsxTag, children: Vec<Node>, end: Option<crate::node::Point>) -> }) } -fn flatten(blocks: Vec<Block>, references: &References, nodes: &mut Vec<Node>) -> Result<(), String> { +fn flatten(blocks: Vec<Block>, doc: Document<'_>, nodes: &mut Vec<Node>) -> Result<(), String> { let mut frames = Frames { open: Vec::new(), root: Vec::new(), @@ -116,18 +119,18 @@ fn flatten(blocks: Vec<Block>, references: &References, nodes: &mut Vec<Node>) - Block::Fenced(fenced) => frames.push(fenced.node), Block::Inline(block) => { let mut out = Vec::new(); - inline_block(block, references, &mut out)?; + inline_block(block, doc, &mut out)?; out.into_iter().for_each(|node| frames.push(node)); } - Block::Quote(quote) => frames.push(quote_node(quote, references)?), + Block::Quote(quote) => frames.push(quote_node(quote, doc)?), Block::List(list) => { let mut out = Vec::new(); - list_nodes(list, 0, references, &mut out)?; + list_nodes(list, 0, doc, &mut out)?; out.into_iter().for_each(|node| frames.push(node)); } Block::Footnote(footnote) => { let mut values = Vec::new(); - flatten(footnote.children, references, &mut values)?; + flatten(footnote.children, doc, &mut values)?; frames.push(Node::Footnote(Footnote { ident: footnote.ident, values, @@ -144,7 +147,7 @@ fn flatten(blocks: Vec<Block>, references: &References, nodes: &mut Vec<Node>) - source, } => Node::TableCell(TableCell { values: match source { - Some(source) => inline::parse(&source, references)?, + Some(source) => inline::parse(&source, doc)?, None => Vec::new(), }, column, @@ -178,7 +181,7 @@ fn flatten(blocks: Vec<Block>, references: &References, nodes: &mut Vec<Node>) - } /// A block quote, or a callout when the `callout` feature is enabled and its first line is a callout header. -fn quote_node(quote: QuoteBlock, references: &References) -> Result<Node, String> { +fn quote_node(quote: QuoteBlock, doc: Document<'_>) -> Result<Node, String> { #[cfg(feature = "callout")] let (header, children) = { let mut children = quote.children; @@ -188,7 +191,7 @@ fn quote_node(quote: QuoteBlock, references: &References) -> Result<Node, String let children = quote.children; let mut values = Vec::new(); - flatten(children, references, &mut values)?; + flatten(children, doc, &mut values)?; let position = Some(quote.position); #[cfg(feature = "callout")] @@ -204,8 +207,8 @@ fn quote_node(quote: QuoteBlock, references: &References) -> Result<Node, String Ok(Node::Blockquote(Blockquote { values, position })) } -fn inline_block(block: InlineBlock, references: &References, nodes: &mut Vec<Node>) -> Result<(), String> { - let values = inline::parse(&block.source, references)?; +fn inline_block(block: InlineBlock, doc: Document<'_>, nodes: &mut Vec<Node>) -> Result<(), String> { + let values = inline::parse(&block.source, doc)?; match block.kind { InlineKind::Paragraph => nodes.extend(values), InlineKind::Heading { depth, position } => nodes.push(Node::Heading(Heading { @@ -218,11 +221,11 @@ fn inline_block(block: InlineBlock, references: &References, nodes: &mut Vec<Nod } /// Emits one flat `Node::List` per item, followed by the items of its nested lists one level deeper. -fn list_nodes(list: ListBlock, level: Level, references: &References, nodes: &mut Vec<Node>) -> Result<(), String> { +fn list_nodes(list: ListBlock, level: Level, doc: Document<'_>, nodes: &mut Vec<Node>) -> Result<(), String> { for (index, item) in list.items.into_iter().enumerate() { let (nested, others): (Vec<_>, Vec<_>) = item.children.into_iter().partition(|b| matches!(b, Block::List(_))); let mut values = Vec::new(); - flatten(others, references, &mut values)?; + flatten(others, doc, &mut values)?; let position = match ( values.first().and_then(Node::position), values.last().and_then(Node::position), @@ -248,7 +251,7 @@ fn list_nodes(list: ListBlock, level: Level, references: &References, nodes: &mu for block in nested { if let Block::List(sub_list) = block { - list_nodes(sub_list, level + 1, references, nodes)?; + list_nodes(sub_list, level + 1, doc, nodes)?; } } } diff --git a/crates/mq-markdown/src/parser/table.rs b/crates/mq-markdown/src/parser/table.rs index abf19bad6..f4ae3c727 100644 --- a/crates/mq-markdown/src/parser/table.rs +++ b/crates/mq-markdown/src/parser/table.rs @@ -95,7 +95,7 @@ fn header<'a>( starts_block: fn(&Line<'_>) -> bool, ) -> Option<(Vec<Cell<'a>>, Vec<TableAlignKind>)> { let head = &lines[start]; - if head.mdx { + if !head.flavor.has_gfm() { return None; } let delimiter = lines.get(start + 1)?; From 3c9698dc43abe7db1deb7f2cd32a3a612d3e3034 Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Thu, 8 Oct 2026 13:09:17 +0900 Subject: [PATCH 43/88] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor(markdown):?= =?UTF-8?q?=20name=20the=20parts=20of=20indentation=20and=20offset=20range?= =?UTF-8?q?s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Parser internals passed several pairs as bare tuples where the two halves were easy to swap: - Line::indent returns Indent { columns, bytes } instead of (usize, usize), and the block rule context holds one Indent. - Byte ranges of the inline source are a Span: text item slices, the link text of an autolink, MDX expression content and the email grouping range. - The line starts of an InlineSource are LineStart { offset, point }, looked up through InlineSource::line_index, and the definition extractor takes each line as a Position. - Item::Node names its nesting depth. - Tables compare indentation with Line::code_indent instead of a literal 4. --- crates/mq-markdown/src/parser/block.rs | 43 ++++++++------ crates/mq-markdown/src/parser/block/rule.rs | 22 ++++--- crates/mq-markdown/src/parser/code.rs | 9 ++- crates/mq-markdown/src/parser/definition.rs | 18 +++--- crates/mq-markdown/src/parser/inline.rs | 31 +++++----- .../mq-markdown/src/parser/inline/emphasis.rs | 2 +- crates/mq-markdown/src/parser/inline/html.rs | 10 ++-- crates/mq-markdown/src/parser/inline/link.rs | 5 +- .../mq-markdown/src/parser/inline/literal.rs | 35 +++++------ crates/mq-markdown/src/parser/line.rs | 19 ++++-- crates/mq-markdown/src/parser/mdx.rs | 21 +++---- crates/mq-markdown/src/parser/mdx_flow.rs | 11 +--- crates/mq-markdown/src/parser/render_html.rs | 4 +- crates/mq-markdown/src/parser/scan.rs | 18 ++++++ crates/mq-markdown/src/parser/table.rs | 8 ++- crates/mq-markdown/src/parser/tree.rs | 59 +++++++++++-------- 16 files changed, 185 insertions(+), 130 deletions(-) diff --git a/crates/mq-markdown/src/parser/block.rs b/crates/mq-markdown/src/parser/block.rs index 08aa953fe..a7d62c42c 100644 --- a/crates/mq-markdown/src/parser/block.rs +++ b/crates/mq-markdown/src/parser/block.rs @@ -9,7 +9,7 @@ use super::definition; use super::flavor::Flavor; use super::html_flow::{self, Kind as HtmlKind}; use super::inline; -use super::line::{Line, split_lines}; +use super::line::{Indent, Line, split_lines}; use super::mdx_flow::{FlowOutcome, absorbed_until, blocks_lazy_continuation, looks_like_mdx_flow, probe_mdx_flow}; use super::table; mod rule; @@ -104,14 +104,13 @@ fn parse_blocks(lines: &[Line<'_>], depth: usize, interrupting: Interrupt) -> Ve let closed_container = std::mem::take(&mut after_container); let separated = std::mem::take(&mut blank_between); - let (columns, indent) = line.indent(); + let indent = line.indent(); let cx = Cx { lines, index, line, - columns, indent, - rest: &line.text[indent..], + rest: &line.text[indent.bytes..], depth, containers, interrupting, @@ -174,7 +173,7 @@ impl<'a> LeafState<'a> { // Whether a paragraph was open when the line started, for markers deeper on the same line. let open = self.paragraph; loop { - let (columns, indent) = line.indent(); + let Indent { columns, bytes: indent } = line.indent(); let rest = &line.text[indent..]; if let Some(fence) = &self.fence { @@ -289,7 +288,7 @@ impl<'a> LeafState<'a> { /// Bytes to skip to get past a blockquote marker (`>` and one optional space), if `line` has one. fn blockquote_marker(line: &Line<'_>) -> Option<usize> { - let (columns, indent) = line.indent(); + let Indent { columns, bytes: indent } = line.indent(); let rest = &line.text[indent..]; if columns >= line.code_indent() || !rest.starts_with('>') { return None; @@ -300,7 +299,7 @@ fn blockquote_marker(line: &Line<'_>) -> Option<usize> { /// The line without its blockquote marker and the one space or column after it, if it has a marker. fn strip_blockquote(line: Line<'_>) -> Option<Line<'_>> { - let (columns, indent) = line.indent(); + let Indent { columns, bytes: indent } = line.indent(); if columns >= line.code_indent() || !line.text[indent..].starts_with('>') { return None; } @@ -372,7 +371,7 @@ struct ItemMarker { impl ItemMarker { fn parse(line: &Line<'_>) -> Option<Self> { - let (columns, indent) = line.indent(); + let Indent { columns, bytes: indent } = line.indent(); if columns >= line.code_indent() { return None; } @@ -402,7 +401,10 @@ impl ItemMarker { }; let after = &rest[marker_len..]; - let (spaces, blanks) = line.skip(indent + marker_len).indent(); + let Indent { + columns: spaces, + bytes: blanks, + } = line.skip(indent + marker_len).indent(); if blanks == 0 && !after.is_empty() { return None; } @@ -600,7 +602,7 @@ fn list_item( } else if line.lazy { state.feed(line); inner.push(*line); - } else if line.indent().0 >= marker.width { + } else if line.indent().columns >= marker.width { for blank in &blanks { state.feed(blank); } @@ -771,7 +773,7 @@ fn is_thematic_break(line: &Line<'_>, indent: usize) -> bool { /// Returns the setext heading depth when `line` is a setext underline. fn setext_depth(line: &Line<'_>) -> Option<u8> { - let (columns, indent) = line.indent(); + let Indent { columns, bytes: indent } = line.indent(); let content = line.text[indent..].trim_end_matches([' ', '\t']); let marker = content.chars().next()?; if line.lazy || columns >= line.code_indent() || !content.chars().all(|c| c == marker) { @@ -799,7 +801,7 @@ fn interrupts_paragraph(line: &Line<'_>) -> bool { if line.lazy { return false; } - let (columns, indent) = line.indent(); + let Indent { columns, bytes: indent } = line.indent(); let rest = &line.text[indent..]; columns < line.code_indent() && (Fence::open(rest, line.flavor.has_math()).is_some() @@ -815,7 +817,7 @@ fn interrupts_paragraph(line: &Line<'_>) -> bool { /// or that is numbered from other than one. fn ends_table(line: &Line<'_>) -> bool { interrupts_paragraph(line) - || (!line.lazy && line.indent().0 < line.code_indent() && ItemMarker::parse(line).is_some()) + || (!line.lazy && line.indent().columns < line.code_indent() && ItemMarker::parse(line).is_some()) } /// Collects a paragraph or a setext heading starting at `lines[start]`, with the definitions at its @@ -858,11 +860,14 @@ fn paragraph(lines: &[Line<'_>], start: usize, blocks: &mut Vec<Block>) -> (usiz let mut first = start; if lines[start].text.trim_start_matches([' ', '\t']).starts_with('[') { let source = paragraph_source(&lines[start..index]); - let starts = lines[start..index] + let spans = lines[start..index] .iter() - .map(|line| (line.point(0), line.end())) + .map(|line| Position { + start: line.point(0), + end: line.end(), + }) .collect::<Vec<_>>(); - let (definitions, used) = definition::extract(&source, &starts); + let (definitions, used) = definition::extract(&source, &spans); blocks.extend(definitions.into_iter().map(Block::Node)); first += used; } @@ -944,7 +949,7 @@ fn footnote_marker<'a>(line: &Line<'a>) -> Option<FootnoteMarker<'a>> { if !line.flavor.has_gfm() { return None; } - let (columns, indent) = line.indent(); + let Indent { columns, bytes: indent } = line.indent(); let rest = line.text[indent..].strip_prefix("[^")?; if columns >= line.code_indent() { return None; @@ -996,7 +1001,7 @@ fn footnote( while let Some(line) = lines.get(index) { if line.is_blank() { blanks.push(line.skip(line.text.len())); - } else if line.indent().0 >= FOOTNOTE_INDENT { + } else if line.indent().columns >= FOOTNOTE_INDENT { for blank in &blanks { state.feed(blank); } @@ -1050,7 +1055,7 @@ fn html_block(lines: &[Line<'_>], start: usize, kind: HtmlKind, blocks: &mut Vec } } _ => { - let (_, indent) = first.indent(); + let indent = first.indent().bytes; let from = indent + html_flow::first_line_offset(kind); if !html_flow::ends_in(kind, &first.text[from..]) { let mut ended = false; diff --git a/crates/mq-markdown/src/parser/block/rule.rs b/crates/mq-markdown/src/parser/block/rule.rs index 6f99f1e3a..f0e3c9827 100644 --- a/crates/mq-markdown/src/parser/block/rule.rs +++ b/crates/mq-markdown/src/parser/block/rule.rs @@ -10,7 +10,7 @@ use super::{ }; use crate::node::{HorizontalRule, HorizontalRuleMarker, MdxJsEsm, Node, Position}; use crate::parser::code::{Fence, fenced_code, indented_code}; -use crate::parser::line::Line; +use crate::parser::line::{Indent, Line}; use crate::parser::mdx_flow::{FlowOutcome, mdx_flow}; use crate::parser::table; use crate::parser::tree::Block; @@ -20,9 +20,8 @@ pub(super) struct Cx<'a, 'l> { pub(super) lines: &'l [Line<'a>], pub(super) index: usize, pub(super) line: &'l Line<'a>, - /// Columns and byte length of the indentation of `line`, and the text after it. - pub(super) columns: usize, - pub(super) indent: usize, + /// The indentation of `line`, and the text after it. + pub(super) indent: Indent, pub(super) rest: &'a str, pub(super) depth: usize, /// Whether containers can still be nested. @@ -106,7 +105,7 @@ pub(super) struct IndentedCode; impl BlockRule for IndentedCode { fn parse(cx: &Cx<'_, '_>, blocks: &mut Vec<Block>) -> Option<Step> { - if cx.columns < cx.line.code_indent() { + if cx.indent.columns < cx.line.code_indent() { return None; } Some(if cx.interrupting.code { @@ -127,7 +126,12 @@ impl BlockRule for FencedCode { && !(cx.separated && matches!(blocks.last(), Some(Block::List(l)) if l.items.last().is_some_and(|i| i.children.is_empty()))); Some(Step::to(fenced_code( - cx.lines, cx.index, cx.indent, &fence, own_end, blocks, + cx.lines, + cx.index, + cx.indent.bytes, + &fence, + own_end, + blocks, ))) } } @@ -137,7 +141,7 @@ pub(super) struct AtxHeading; impl BlockRule for AtxHeading { fn parse(cx: &Cx<'_, '_>, blocks: &mut Vec<Block>) -> Option<Step> { let depth = atx_depth(cx.rest)?; - blocks.push(atx_heading(cx.line, cx.indent, depth)); + blocks.push(atx_heading(cx.line, cx.indent.bytes, depth)); Some(Step::to(cx.index + 1)) } } @@ -146,7 +150,7 @@ pub(super) struct ThematicBreak; impl BlockRule for ThematicBreak { fn parse(cx: &Cx<'_, '_>, blocks: &mut Vec<Block>) -> Option<Step> { - if !is_thematic_break(cx.line, cx.indent) { + if !is_thematic_break(cx.line, cx.indent.bytes) { return None; } blocks.push(Block::Node(Node::HorizontalRule(HorizontalRule { @@ -261,7 +265,7 @@ impl BlockRule for Esm { let line = cx.line; if !(line.flavor.has_jsx() && cx.depth == 0 - && cx.indent == 0 + && cx.indent.bytes == 0 && (line.text.starts_with("import ") || line.text.starts_with("export "))) { return None; diff --git a/crates/mq-markdown/src/parser/code.rs b/crates/mq-markdown/src/parser/code.rs index 65fb21776..f52c3f1af 100644 --- a/crates/mq-markdown/src/parser/code.rs +++ b/crates/mq-markdown/src/parser/code.rs @@ -2,7 +2,7 @@ use super::block::CODE_INDENT; use super::inline; -use super::line::Line; +use super::line::{Indent, Line}; use super::tree::{Block, FencedBlock}; use crate::node::{Code, Math, Node, Point, Position}; @@ -55,7 +55,10 @@ pub(super) fn fenced_code( let mut end = None; while let Some(line) = lines.get(index) { - let (columns, line_indent) = line.indent(); + let Indent { + columns, + bytes: line_indent, + } = line.indent(); if columns < line.code_indent() && fence.is_closed_by(&line.text[line_indent..]) { end = Some(line.end()); index += 1; @@ -128,7 +131,7 @@ pub(super) fn indented_code(lines: &[Line<'_>], start: usize, blocks: &mut Vec<B while let Some(line) = lines.get(index) { // Whitespace-only lines are code when indented enough, otherwise they may just separate chunks. - if line.indent().0 >= CODE_INDENT { + if line.indent().columns >= CODE_INDENT { last_code = index; } else if !line.is_blank() { break; diff --git a/crates/mq-markdown/src/parser/definition.rs b/crates/mq-markdown/src/parser/definition.rs index 01de568e8..11de63390 100644 --- a/crates/mq-markdown/src/parser/definition.rs +++ b/crates/mq-markdown/src/parser/definition.rs @@ -3,7 +3,7 @@ use super::inline::{destination, normalize, remove_line_indent, title_at, unescape}; use super::scan::{eol_len, skip_blanks, skip_blanks_and_eol}; use super::tree::InlineSource; -use crate::node::{Definition, Node, Point, Position, Title, Url}; +use crate::node::{Definition, Node, Position, Title, Url}; /// Parses the label at `pos` (a `[`) followed by `:`, returning the raw label and the offset after. fn label(text: &str, pos: usize) -> Option<(&str, usize)> { @@ -84,9 +84,9 @@ fn title_start(bytes: &[u8], after_dest: usize) -> Option<usize> { (start > after_dest).then_some(start) } -/// Parses the definitions at the start of a paragraph. `lines` holds the start (including its -/// indentation) and the end of each line. Returns the definitions and how many lines they use. -pub(super) fn extract(source: &InlineSource, lines: &[(Point, Point)]) -> (Vec<Node>, usize) { +/// Parses the definitions at the start of a paragraph. `lines` holds the span of each line, from the +/// start of its indentation. Returns the definitions and how many lines they use. +pub(super) fn extract(source: &InlineSource, lines: &[Position]) -> (Vec<Node>, usize) { let text = source.text.as_str(); let mut nodes = Vec::new(); let mut pos = 0; @@ -96,13 +96,13 @@ pub(super) fn extract(source: &InlineSource, lines: &[(Point, Point)]) -> (Vec<N let Some(parsed) = parse_one(text, pos) else { break; }; - let first = source.lines.partition_point(|(start, _)| *start <= pos) - 1; + let first = source.line_index(pos); // The definition ends with its last line, trailing whitespace included. - let last = source.lines.partition_point(|(start, _)| *start <= parsed.end) - 1; + let last = source.line_index(parsed.end); nodes.push(Node::Definition(Definition { position: Some(Position { - start: lines[first].0.clone(), - end: lines[last].1.clone(), + start: lines[first].start.clone(), + end: lines[last].end.clone(), }), url: Url(parsed.url), title: parsed.title.map(Title), @@ -114,6 +114,6 @@ pub(super) fn extract(source: &InlineSource, lines: &[(Point, Point)]) -> (Vec<N } // Every line that starts before the end of the definitions is used. - let used = source.lines.partition_point(|(start, _)| *start < next_line); + let used = source.lines.partition_point(|line| line.offset < next_line); (nodes, used) } diff --git a/crates/mq-markdown/src/parser/inline.rs b/crates/mq-markdown/src/parser/inline.rs index e7a4e4ce5..64ba59c65 100644 --- a/crates/mq-markdown/src/parser/inline.rs +++ b/crates/mq-markdown/src/parser/inline.rs @@ -23,7 +23,7 @@ pub(super) use tail::{destination, title_at}; use super::flavor::Flavor; use super::mdx::{self, Fallback, Parsed, TagKind}; use super::resolve::References; -use super::scan::eol_len; +use super::scan::{Span, eol_len}; use super::tree::InlineSource; use crate::node::{ Break, CodeInline, MathInline, MdxAttributeContent, MdxJsxTextElement, MdxTextExpression, Node, Position, Text, @@ -35,7 +35,7 @@ use std::cell::{OnceCell, RefCell}; /// The value of a text item: a slice of the source, or a decoded string. pub(super) enum Value { - Slice(usize, usize), + Slice(Span), Owned(String), } @@ -65,7 +65,10 @@ pub(super) enum Item { value: Value, }, /// A finished node and how deeply it nests: a leaf is 1. - Node(Node, usize), + Node { + node: Node, + depth: usize, + }, Delim(Delim), Open(Opener), /// An email address in plain text. It is not linked inside link text. @@ -94,7 +97,7 @@ pub(super) const MAX_NESTING: usize = 128; /// How deeply the item nests: nodes carry it, and the rest are text. pub(super) fn item_depth(item: &Item) -> usize { match item { - Item::Node(_, depth) => *depth, + Item::Node { depth, .. } => *depth, _ => 0, } } @@ -202,7 +205,7 @@ pub(super) fn to_nodes(items: Vec<Item>, context: &Context<'_>) -> Vec<Node> { let (start, end, text): (usize, usize, Cow<'_, str>) = match item { Item::Text { start, end, value } => match value { // The backslash of an escape is not part of the position of the text it starts. - Value::Slice(from, to) => (from, end, Cow::Borrowed(&src[from..to])), + Value::Slice(span) => (span.start, end, Cow::Borrowed(span.of(src))), Value::Owned(value) => (start, end, Cow::Owned(value)), }, Item::Delim(delim) => { @@ -213,7 +216,7 @@ pub(super) fn to_nodes(items: Vec<Item>, context: &Context<'_>) -> Vec<Node> { let end = opener.start + if opener.image { 2 } else { 1 }; (opener.start, end, Cow::Borrowed(&src[opener.start..end])) } - Item::Node(node, _) => { + Item::Node { node, .. } => { finish(&mut current, &mut nodes); nodes.push(node); continue; @@ -478,7 +481,7 @@ impl Scanner<'_> { self.items.push(Item::Text { start: self.run, end: self.pos, - value: Value::Slice(self.run, self.pos), + value: Value::Slice(Span::new(self.run, self.pos)), }); } self.run = self.pos; @@ -487,7 +490,7 @@ impl Scanner<'_> { /// Pushes `item` after the pending text and continues scanning at `end`. fn push(&mut self, item: Item, end: usize) { self.flush(); - if let Item::Node(_, depth) = &item { + if let Item::Node { depth, .. } = &item { self.note_depth(*depth); } self.items.push(item); @@ -503,7 +506,7 @@ impl Scanner<'_> { } fn push_node(&mut self, node: Node, end: usize) { - self.push(Item::Node(node, 1), end); + self.push(Item::Node { node, depth: 1 }, end); } fn escape(&mut self) { @@ -515,7 +518,7 @@ impl Scanner<'_> { let item = Item::Text { start: pos, end: pos + 2, - value: Value::Slice(pos + 1, pos + 2), + value: Value::Slice(Span::new(pos + 1, pos + 2)), }; self.push(item, pos + 2); } @@ -574,12 +577,12 @@ impl Scanner<'_> { position: Some(self.context.position(start, pos + eol)), }); self.note_depth(1); - self.items.push(Item::Node(node, 1)); + self.items.push(Item::Node { node, depth: 1 }); } else { self.items.push(Item::Text { start: pos, end: pos + eol, - value: Value::Slice(pos, pos + eol), + value: Value::Slice(Span::new(pos, pos + eol)), }); } self.pos = after; @@ -687,8 +690,8 @@ impl Scanner<'_> { url: crate::node::Url(autolink.url), title: None, values: vec![Node::Text(Text { - value: src[autolink.text.0..autolink.text.1].to_string(), - position: position(autolink.text.0, autolink.text.1), + value: autolink.text.of(src).to_string(), + position: position(autolink.text.start, autolink.text.end), })], position: position(pos, autolink.end), }); diff --git a/crates/mq-markdown/src/parser/inline/emphasis.rs b/crates/mq-markdown/src/parser/inline/emphasis.rs index 634436f62..14ebdee73 100644 --- a/crates/mq-markdown/src/parser/inline/emphasis.rs +++ b/crates/mq-markdown/src/parser/inline/emphasis.rs @@ -325,7 +325,7 @@ pub(super) fn process(items: &mut Vec<Item>, context: &Context<'_>) { (_, 2) => Node::Strong(Strong { values, position }), _ => Node::Emphasis(Emphasis { values, position }), }; - slots.insert_between(opener, closer, Item::Node(node, depth + 1)); + slots.insert_between(opener, closer, Item::Node { node, depth: depth + 1 }); slots.delim_mut(opener).count -= used; let close = slots.delim_mut(closer); diff --git a/crates/mq-markdown/src/parser/inline/html.rs b/crates/mq-markdown/src/parser/inline/html.rs index 56d250baf..85f508a76 100644 --- a/crates/mq-markdown/src/parser/inline/html.rs +++ b/crates/mq-markdown/src/parser/inline/html.rs @@ -1,12 +1,12 @@ //! Autolinks (`<https://example.com>`) and raw inline HTML. -use crate::parser::scan::{is_whitespace, skip_whitespace}; +use crate::parser::scan::{Span, is_whitespace, skip_whitespace}; /// An autolink found at `<`. pub(super) struct Autolink { pub(super) url: String, - /// Start and end of the link text, inside the angle brackets. - pub(super) text: (usize, usize), + /// The link text, inside the angle brackets. + pub(super) text: Span, /// Offset after the closing `>`. pub(super) end: usize, } @@ -38,7 +38,7 @@ pub(super) fn autolink(src: &str, pos: usize) -> Option<Autolink> { .position(|&b| b == b'>' || b <= 0x1F || matches!(b, b' ' | b'<' | 0x7F))?; return (bytes[close] == b'>').then(|| Autolink { url: src[inner..close].to_string(), - text: (inner, close), + text: Span::new(inner, close), end: close + 1, }); } @@ -46,7 +46,7 @@ pub(super) fn autolink(src: &str, pos: usize) -> Option<Autolink> { let end = email(bytes, inner)?; Some(Autolink { url: format!("mailto:{}", &src[inner..end]), - text: (inner, end), + text: Span::new(inner, end), end: end + 1, }) } diff --git a/crates/mq-markdown/src/parser/inline/link.rs b/crates/mq-markdown/src/parser/inline/link.rs index 4b9142217..4fc579136 100644 --- a/crates/mq-markdown/src/parser/inline/link.rs +++ b/crates/mq-markdown/src/parser/inline/link.rs @@ -6,6 +6,7 @@ use super::{Context, Item, MAX_NESTING, Scanner, Value, emphasis, item_depth, to #[cfg(feature = "wikilink")] use crate::node::RenderOptions; use crate::node::{FootnoteRef, Image, ImageRef, Link, LinkRef, Node, Position, Text, Title, Url}; +use crate::parser::scan::Span; /// Normalizes a reference label: whitespace runs become one space, the ends are trimmed, and the case /// is folded. @@ -105,7 +106,7 @@ pub(super) fn close(scanner: &mut Scanner<'_>) -> bool { scanner.items.push(Item::Text { start: opener_start - 1, end: opener_start, - value: Value::Slice(opener_start - 1, opener_start), + value: Value::Slice(Span::new(opener_start - 1, opener_start)), }); } // Email addresses are not linked inside link text. @@ -125,7 +126,7 @@ pub(super) fn close(scanner: &mut Scanner<'_>) -> bool { } scanner.note_depth(depth); - scanner.items.push(Item::Node(node, depth)); + scanner.items.push(Item::Node { node, depth }); scanner.pos = end; scanner.run = end; true diff --git a/crates/mq-markdown/src/parser/inline/literal.rs b/crates/mq-markdown/src/parser/inline/literal.rs index 8a587e3a8..831ab7c1c 100644 --- a/crates/mq-markdown/src/parser/inline/literal.rs +++ b/crates/mq-markdown/src/parser/inline/literal.rs @@ -6,6 +6,7 @@ use super::punctuation::is_punctuation; use super::{Item, Scanner, Value}; use crate::node::{Link, Node, Text, Url}; +use crate::parser::scan::Span; #[derive(PartialEq)] enum Kind { @@ -249,7 +250,7 @@ fn split_emails(src: &str, start: usize, end: usize, out: &mut Vec<Item>) { out.push(Item::Text { start: emitted, end: from, - value: Value::Slice(emitted, from), + value: Value::Slice(Span::new(emitted, from)), }); } out.push(Item::Email { @@ -270,22 +271,22 @@ fn split_emails(src: &str, start: usize, end: usize, out: &mut Vec<Item>) { out.push(Item::Text { start: emitted, end, - value: Value::Slice(emitted, end), + value: Value::Slice(Span::new(emitted, end)), }); } } -/// The source range of an item that `markdown-rs` treats as plain data: text that is not decoded, -/// and delimiters and brackets that were not used. -fn data_range(item: &Item) -> Option<(usize, usize)> { +/// The source range of an item that is plain data: text that is not decoded, and delimiters and +/// brackets that were not used. +fn data_range(item: &Item) -> Option<Span> { match item { Item::Text { start, end, - value: Value::Slice(from, to), - } if from == start && to == end => Some((*start, *end)), - Item::Delim(delim) => Some((delim.start, delim.start + delim.count)), - Item::Open(opener) => Some((opener.start, opener.start + if opener.image { 2 } else { 1 })), + value: Value::Slice(span), + } if span.start == *start && span.end == *end => Some(*span), + Item::Delim(delim) => Some(Span::new(delim.start, delim.start + delim.count)), + Item::Open(opener) => Some(Span::new(opener.start, opener.start + if opener.image { 2 } else { 1 })), _ => None, } } @@ -294,28 +295,28 @@ fn data_range(item: &Item) -> Option<(usize, usize)> { pub(super) fn link_emails(src: &str, items: Vec<Item>) -> Vec<Item> { let mut out = Vec::with_capacity(items.len()); let mut group: Vec<Item> = Vec::new(); - let mut range = (0, 0); + let mut range = Span::new(0, 0); - let finish = |group: &mut Vec<Item>, range: (usize, usize), out: &mut Vec<Item>| { + let finish = |group: &mut Vec<Item>, range: Span, out: &mut Vec<Item>| { if group.is_empty() { return; } - if src[range.0..range.1].contains('@') { + if range.of(src).contains('@') { group.clear(); - split_emails(src, range.0, range.1, out); + split_emails(src, range.start, range.end, out); } else { out.append(group); } }; for item in items { - let data = data_range(&item).filter(|&(start, _)| !matches!(src.as_bytes()[start], b'\n' | b'\r')); + let data = data_range(&item).filter(|span| !matches!(src.as_bytes()[span.start], b'\n' | b'\r')); match data { - Some((start, end)) => { + Some(span) => { if group.is_empty() { - range.0 = start; + range.start = span.start; } - range.1 = end; + range.end = span.end; group.push(item); } None => { diff --git a/crates/mq-markdown/src/parser/line.rs b/crates/mq-markdown/src/parser/line.rs index efeac1fff..b036924c8 100644 --- a/crates/mq-markdown/src/parser/line.rs +++ b/crates/mq-markdown/src/parser/line.rs @@ -4,6 +4,14 @@ use super::block::CODE_INDENT; use super::flavor::Flavor; use crate::node::Point; +/// Leading whitespace: how many columns it spans, where tabs go to the next multiple of four, and how +/// many bytes it takes. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) struct Indent { + pub(super) columns: usize, + pub(super) bytes: usize, +} + #[derive(Clone, Copy)] pub(super) struct Line<'a> { pub(super) number: usize, @@ -52,8 +60,8 @@ impl<'a> Line<'a> { } } - /// Column count and byte length of the leading whitespace. - pub(super) fn indent(&self) -> (usize, usize) { + /// The leading whitespace. + pub(super) fn indent(&self) -> Indent { let mut columns = self.pad; let mut absolute = self.start_column(); for (index, byte) in self.text.bytes().enumerate() { @@ -67,10 +75,13 @@ impl<'a> Line<'a> { columns += width; absolute += width; } - _ => return (columns, index), + _ => return Indent { columns, bytes: index }, } } - (columns, self.text.len()) + Indent { + columns, + bytes: self.text.len(), + } } pub(super) fn is_blank(&self) -> bool { diff --git a/crates/mq-markdown/src/parser/mdx.rs b/crates/mq-markdown/src/parser/mdx.rs index 7408244c5..25c3cb043 100644 --- a/crates/mq-markdown/src/parser/mdx.rs +++ b/crates/mq-markdown/src/parser/mdx.rs @@ -1,6 +1,7 @@ //! MDX: JSX tags and expressions, ported from the rules of `markdown-rs` without a JavaScript parser, //! so expressions are only checked for balanced braces, outside of strings, template literals and comments. +use super::scan::Span; use crate::node::{MdxAttributeContent, MdxAttributeValue, MdxJsxAttribute}; use rustc_hash::FxHashMap; use smol_str::SmolStr; @@ -241,8 +242,8 @@ impl Cursor<'_> { Ok(name) } - /// A `{...}` expression with balanced braces, returning the offsets of its content. - fn expression(&mut self) -> Step<(usize, usize)> { + /// A `{...}` expression with balanced braces, returning the range of its content. + fn expression(&mut self) -> Step<Span> { let open = self.index; self.bump(); let start = self.index; @@ -255,15 +256,15 @@ impl Cursor<'_> { return Err(Stop::More(Fallback::Error(UNCLOSED_EXPRESSION.into()))); }; self.index = end + 1; - Ok((start, end)) + Ok(Span::new(start, end)) } } impl Cursor<'_> { /// The value of an expression: its content, without up to two columns of whitespace at the start /// of each line after the first. - fn expression_value(&self, start: usize, end: usize) -> SmolStr { - let content = &self.src[start..end]; + fn expression_value(&self, content: Span) -> SmolStr { + let content = content.of(self.src); if !content.contains(['\n', '\r']) { return SmolStr::new(content); } @@ -350,8 +351,8 @@ impl Cursor<'_> { ))) } Some('{') => { - let (start, end) = self.expression()?; - MdxAttributeValue::Expression(self.expression_value(start, end)) + let content = self.expression()?; + MdxAttributeValue::Expression(self.expression_value(content)) } _ => { return self.crash(format!("Unexpected {} before attribute value", self.describe())); @@ -415,8 +416,8 @@ impl Cursor<'_> { misplaced = Some("Unexpected attribute in closing tag, expected the end of the tag"); } let attribute = if char == '{' { - let (start, end) = self.expression()?; - MdxAttributeContent::Expression(self.expression_value(start, end)) + let content = self.expression()?; + MdxAttributeContent::Expression(self.expression_value(content)) } else { self.attribute()? }; @@ -481,6 +482,6 @@ pub(super) fn expression(src: &str, pos: usize, braces: Option<&Braces>) -> Pars parsed( cursor .expression() - .map(|(start, end)| (end + 1, cursor.expression_value(start, end))), + .map(|content| (content.end + 1, cursor.expression_value(content))), ) } diff --git a/crates/mq-markdown/src/parser/mdx_flow.rs b/crates/mq-markdown/src/parser/mdx_flow.rs index 58ccd3c04..67c5b8db3 100644 --- a/crates/mq-markdown/src/parser/mdx_flow.rs +++ b/crates/mq-markdown/src/parser/mdx_flow.rs @@ -110,13 +110,8 @@ pub(super) fn mdx_flow(lines: &[Line<'_>], start: usize, blocks: &mut Vec<Block> return FlowOutcome::Flow(lines.len()); } Flow::Nok(end) => { - let last_line = end.map(|end| { - start - + source - .lines - .partition_point(|(offset, _)| *offset < end) - .saturating_sub(1) - }); + let last_line = + end.map(|end| start + source.lines.partition_point(|line| line.offset < end).saturating_sub(1)); return FlowOutcome::Nok(last_line); } Flow::Error(message) => { @@ -127,7 +122,7 @@ pub(super) fn mdx_flow(lines: &[Line<'_>], start: usize, blocks: &mut Vec<Block> let last_line = start + source .lines - .partition_point(|(offset, _)| *offset < last) + .partition_point(|line| line.offset < last) .saturating_sub(1); if lines[start + 1..=last_line].iter().any(|line| line.lazy) { blocks.push(Block::Error( diff --git a/crates/mq-markdown/src/parser/render_html.rs b/crates/mq-markdown/src/parser/render_html.rs index a28d649dd..fd44e95ac 100644 --- a/crates/mq-markdown/src/parser/render_html.rs +++ b/crates/mq-markdown/src/parser/render_html.rs @@ -118,7 +118,7 @@ fn end_line(block: &Block) -> Option<usize> { kind: InlineKind::Heading { position, .. }, .. }) => Some(position.end.line), - Block::Inline(block) => block.source.lines.last().map(|(_, point)| point.line), + Block::Inline(block) => block.source.lines.last().map(|line| line.point.line), Block::Quote(quote) => Some(quote.position.end.line), // The position of an item includes the blank lines that follow it. Block::List(list) => list.items.last().and_then(|item| { @@ -144,7 +144,7 @@ fn start_line(block: &Block) -> Option<usize> { kind: InlineKind::Heading { position, .. }, .. }) => Some(position.start.line), - Block::Inline(block) => block.source.lines.first().map(|(_, point)| point.line), + Block::Inline(block) => block.source.lines.first().map(|line| line.point.line), Block::Quote(quote) => Some(quote.position.start.line), Block::List(list) => list.items.first().map(|item| item.position.start.line), Block::Footnote(footnote) => Some(footnote.position.start.line), diff --git a/crates/mq-markdown/src/parser/scan.rs b/crates/mq-markdown/src/parser/scan.rs index 0a66d0b56..32ba2bda9 100644 --- a/crates/mq-markdown/src/parser/scan.rs +++ b/crates/mq-markdown/src/parser/scan.rs @@ -1,5 +1,23 @@ //! Byte-level helpers shared by the block and inline parsers. +/// A byte range `start..end` of a source text. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) struct Span { + pub(super) start: usize, + pub(super) end: usize, +} + +impl Span { + pub(super) fn new(start: usize, end: usize) -> Self { + Self { start, end } + } + + /// The text of the range in `src`. + pub(super) fn of(self, src: &str) -> &str { + &src[self.start..self.end] + } +} + /// The length of the line ending at `index`, or 0 when there is none. pub(super) fn eol_len(bytes: &[u8], index: usize) -> usize { match bytes.get(index..) { diff --git a/crates/mq-markdown/src/parser/table.rs b/crates/mq-markdown/src/parser/table.rs index f4ae3c727..73262c11b 100644 --- a/crates/mq-markdown/src/parser/table.rs +++ b/crates/mq-markdown/src/parser/table.rs @@ -100,7 +100,11 @@ fn header<'a>( } let delimiter = lines.get(start + 1)?; // The delimiter row cannot start another block, such as a list item. - if head.indent().0 >= 4 || delimiter.indent().0 >= 4 || delimiter.lazy || starts_block(delimiter) { + if head.indent().columns >= head.code_indent() + || delimiter.indent().columns >= delimiter.code_indent() + || delimiter.lazy + || starts_block(delimiter) + { return None; } let text = delimiter.text.trim_matches([' ', '\t']); @@ -179,7 +183,7 @@ pub(super) fn parse( let mut index = start + 2; let mut row = 1; while let Some(line) = lines.get(index) { - if line.is_blank() || line.indent().0 >= 4 || ends_row(line) { + if line.is_blank() || line.indent().columns >= line.code_indent() || ends_row(line) { break; } for (column, cell) in split_row(line).iter().enumerate() { diff --git a/crates/mq-markdown/src/parser/tree.rs b/crates/mq-markdown/src/parser/tree.rs index ff04abd93..d10f5bf9f 100644 --- a/crates/mq-markdown/src/parser/tree.rs +++ b/crates/mq-markdown/src/parser/tree.rs @@ -8,12 +8,19 @@ use super::mdx::TagKind; use super::scan::eol_len; use crate::node::{ListMarker, MdxAttributeContent, Node, Point, Position, TableAlignKind}; +/// Where a line of an [`InlineSource`] starts: its offset in the text and its position in the document. +#[derive(Clone)] +pub(super) struct LineStart { + pub(super) offset: usize, + pub(super) point: Point, +} + /// Raw inline content of a paragraph, heading or table cell, with the source position of each line. #[derive(Clone)] pub(super) struct InlineSource { pub(super) text: String, - /// For each line, the offset in `text` where it starts and its position in the document. - pub(super) lines: Vec<(usize, Point)>, + /// Where each line starts, in `text` and in the document. + pub(super) lines: Vec<LineStart>, /// Whether this is the content of a table cell, where `\|` in code stands for `|`. pub(super) table: bool, /// Whether `text` contains a tab, which makes columns differ from byte offsets. @@ -27,7 +34,10 @@ impl InlineSource { let mut text = String::new(); let mut starts = Vec::with_capacity(count); for (index, (line, eol, start)) in lines.enumerate() { - starts.push((text.len(), start)); + starts.push(LineStart { + offset: text.len(), + point: start, + }); text.push_str(line); if index + 1 < count { text.push_str(eol); @@ -72,32 +82,35 @@ impl InlineSource { self.text.replace_range(..end, ""); self.tabs = self.text.contains('\t'); - let index = self.lines.partition_point(|(start, _)| *start <= end).saturating_sub(1); - let point = &self.lines[index].1; - let first = ( - 0, - Point { - line: point.line, + let index = self.line_index(end); + let first = LineStart { + offset: 0, + point: Point { + line: self.lines[index].point.line, column: new_point.column, }, - ); - let rest = self.lines[index + 1..] - .iter() - .map(|(start, point)| (start - end, point.clone())); + }; + let rest = self.lines[index + 1..].iter().map(|line| LineStart { + offset: line.offset - end, + point: line.point.clone(), + }); self.lines = std::iter::once(first).chain(rest).collect(); } + /// The index of the line that `offset` is on. + pub(super) fn line_index(&self, offset: usize) -> usize { + self.lines + .partition_point(|line| line.offset <= offset) + .saturating_sub(1) + } + /// The position where a node ends at `offset`. After a line ending that is the start of the next /// line in the document, not the start of its content inside containers. pub(super) fn end_point(&self, offset: usize) -> Point { - let index = self - .lines - .partition_point(|(start, _)| *start <= offset) - .saturating_sub(1); - let (start, point) = &self.lines[index]; - if *start == offset && offset > 0 { + let line = &self.lines[self.line_index(offset)]; + if line.offset == offset && offset > 0 { return Point { - line: point.line, + line: line.point.line, column: 1, }; } @@ -106,11 +119,7 @@ impl InlineSource { /// The position of the byte at `offset` in `text`. pub(super) fn point(&self, offset: usize) -> Point { - let index = self - .lines - .partition_point(|(start, _)| *start <= offset) - .saturating_sub(1); - let (start, point) = &self.lines[index]; + let LineStart { offset: start, point } = &self.lines[self.line_index(offset)]; let passed = &self.text[*start..offset]; let column = if self.tabs && passed.contains('\t') { visual_column(passed.chars(), point.column - 1) + 1 From 7368ed1375dd53a71d55050c6612ab42649e0560 Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Thu, 8 Oct 2026 13:09:23 +0900 Subject: [PATCH 44/88] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor(markdown):?= =?UTF-8?q?=20decide=20what=20a=20line=20starts=20in=20one=20place?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Block rules, the lazy continuation tracking of containers (LeafState::feed) and the end of paragraphs and tables each tested fences, ATX headings, thematic breaks, blockquotes, list items, HTML and footnotes on their own, in their own order. LineStart::of now classifies the line once, by its first character since those constructs never overlap, and every one of them matches on the result. The block rules read it from their context instead of parsing the line again, and blockquote_marker, which duplicated the blockquote check, is gone. Fence::open takes the flavor. --- crates/mq-markdown/src/parser/block.rs | 187 ++++++++++++-------- crates/mq-markdown/src/parser/block/rule.rs | 47 +++-- crates/mq-markdown/src/parser/code.rs | 6 +- 3 files changed, 145 insertions(+), 95 deletions(-) diff --git a/crates/mq-markdown/src/parser/block.rs b/crates/mq-markdown/src/parser/block.rs index a7d62c42c..81fe0ee6b 100644 --- a/crates/mq-markdown/src/parser/block.rs +++ b/crates/mq-markdown/src/parser/block.rs @@ -110,6 +110,7 @@ fn parse_blocks(lines: &[Line<'_>], depth: usize, interrupting: Interrupt) -> Ve index, line, indent, + start: LineStart::of(line, indent), rest: &line.text[indent.bytes..], depth, containers, @@ -126,6 +127,51 @@ fn parse_blocks(lines: &[Line<'_>], depth: usize, interrupting: Interrupt) -> Ve blocks } +/// A block that a line starts by its first characters, before the rest of the document is known. +/// +/// The constructs are told apart by their first character, so a line starts at most one of them. Block +/// rules, lazy continuation and the end of paragraphs and tables all decide from this one result. +pub(super) enum LineStart<'a> { + Fence(Fence<'a>), + /// An ATX heading of this depth. + Atx(u8), + ThematicBreak, + Blockquote, + Item(ItemMarker), + Html(HtmlKind), + Footnote(FootnoteMarker<'a>), +} + +impl LineStart<'_> { + /// What `line` starts, when it is indented less than code. Setext underlines, tables and MDX flow + /// content depend on the lines around them, so they are not among these. + pub(super) fn of<'a>(line: &Line<'a>, indent: Indent) -> Option<LineStart<'a>> { + if indent.columns >= line.code_indent() { + return None; + } + let rest = &line.text[indent.bytes..]; + match rest.as_bytes().first()? { + b'`' | b'~' | b'$' => Fence::open(rest, line.flavor).map(LineStart::Fence), + b'#' => atx_depth(rest).map(LineStart::Atx), + b'>' => Some(LineStart::Blockquote), + b'<' => html_start(line, rest).map(LineStart::Html), + b'[' => footnote_marker(line).map(LineStart::Footnote), + b'*' | b'-' | b'_' if is_thematic_break(line, indent.bytes) => Some(LineStart::ThematicBreak), + _ => ItemMarker::parse(line).map(LineStart::Item), + } + } + + /// Whether this start ends an open paragraph. A complete HTML tag, an empty item and an ordered + /// item that does not start at 1 cannot. + fn interrupts_paragraph(&self) -> bool { + match self { + LineStart::Html(kind) => *kind != HtmlKind::Complete, + LineStart::Item(marker) => marker.interrupts_paragraph(), + _ => true, + } + } +} + /// Tracks just enough of the block state of already collected lines to tell whether a paragraph /// is open, which decides whether a following line can be a lazy continuation. struct LeafState<'a> { @@ -173,7 +219,8 @@ impl<'a> LeafState<'a> { // Whether a paragraph was open when the line started, for markers deeper on the same line. let open = self.paragraph; loop { - let Indent { columns, bytes: indent } = line.indent(); + let line_indent = line.indent(); + let Indent { columns, bytes: indent } = line_indent; let rest = &line.text[indent..]; if let Some(fence) = &self.fence { @@ -227,7 +274,8 @@ impl<'a> LeafState<'a> { } return; } - if let Some(fence) = Fence::open(rest, line.flavor.has_math()) { + let start = LineStart::of(&line, line_indent); + if let Some(LineStart::Fence(fence)) = start { self.fence = Some(fence); self.paragraph = false; return; @@ -236,38 +284,38 @@ impl<'a> LeafState<'a> { self.paragraph = false; return; } - if atx_depth(rest).is_some() || is_thematic_break(&line, indent) { - self.paragraph = false; - return; - } - if let Some(kind) = html_start(&line, rest).filter(|&kind| kind != HtmlKind::Complete || !self.paragraph) { - let closed = match kind { - HtmlKind::Basic | HtmlKind::Complete => false, - _ => html_flow::ends_in(kind, &rest[html_flow::first_line_offset(kind)..]), - }; - self.html = (!closed).then_some(kind); - self.paragraph = false; - return; - } - if containers < self.budget - && let Some(stripped) = strip_blockquote(line) - { - line = stripped; - containers += 1; - continue; - } - if containers < self.budget - && let Some(marker) = ItemMarker::parse(&line).filter(|m| { - // A marker in or inside the container of the open paragraph has to be able to interrupt it. - let continues_paragraph = open && containers >= self.paragraph_containers; - !(restricted.list || continues_paragraph) || m.interrupts_paragraph() - }) - { - line = marker.content(line); - containers += 1; - // The rest of the line starts a new item. - self.paragraph = false; - continue; + match start { + Some(LineStart::Atx(_) | LineStart::ThematicBreak) => { + self.paragraph = false; + return; + } + Some(LineStart::Html(kind)) if kind != HtmlKind::Complete || !self.paragraph => { + let closed = match kind { + HtmlKind::Basic | HtmlKind::Complete => false, + _ => html_flow::ends_in(kind, &rest[html_flow::first_line_offset(kind)..]), + }; + self.html = (!closed).then_some(kind); + self.paragraph = false; + return; + } + Some(LineStart::Blockquote) if containers < self.budget => { + line = after_blockquote_marker(line, line_indent); + containers += 1; + continue; + } + // A marker in or inside the container of the open paragraph has to be able to interrupt it. + Some(LineStart::Item(marker)) + if containers < self.budget + && (!(restricted.list || (open && containers >= self.paragraph_containers)) + || marker.interrupts_paragraph()) => + { + line = marker.content(line); + containers += 1; + // The rest of the line starts a new item. + self.paragraph = false; + continue; + } + _ => {} } // MDX flow content is not part of a paragraph. @@ -286,30 +334,21 @@ impl<'a> LeafState<'a> { } } -/// Bytes to skip to get past a blockquote marker (`>` and one optional space), if `line` has one. -fn blockquote_marker(line: &Line<'_>) -> Option<usize> { - let Indent { columns, bytes: indent } = line.indent(); - let rest = &line.text[indent..]; - if columns >= line.code_indent() || !rest.starts_with('>') { - return None; - } - let optional_space = usize::from(rest[1..].starts_with([' ', '\t'])); - Some(indent + 1 + optional_space) -} - /// The line without its blockquote marker and the one space or column after it, if it has a marker. fn strip_blockquote(line: Line<'_>) -> Option<Line<'_>> { - let Indent { columns, bytes: indent } = line.indent(); - if columns >= line.code_indent() || !line.text[indent..].starts_with('>') { - return None; - } - let after = line.skip(indent + 1); - Some(match after.text.as_bytes().first() { + let indent = line.indent(); + matches!(LineStart::of(&line, indent), Some(LineStart::Blockquote)).then(|| after_blockquote_marker(line, indent)) +} + +/// The line after the blockquote marker that follows `indent`, and the one space or column after it. +fn after_blockquote_marker(line: Line<'_>, indent: Indent) -> Line<'_> { + let after = line.skip(indent.bytes + 1); + match after.text.as_bytes().first() { Some(b' ') => after.skip(1), // A tab is consumed by one column only. Some(b'\t') => after.skip_columns(1), _ => after, - }) + } } fn blockquote( @@ -355,7 +394,7 @@ fn blockquote( } #[derive(Clone)] -struct ItemMarker { +pub(super) struct ItemMarker { ordered: bool, kind: ListMarker, start: u32, @@ -632,8 +671,13 @@ fn list_item( // An unclosed fence in an item includes its line terminator when the next line is blank or starts // another container, but not when a plain paragraph or thematic break follows. let next = lines.get(index); - let item_end = next - .is_none_or(|line| line.is_blank() || blockquote_marker(line).is_some() || ItemMarker::parse(line).is_some()); + let item_end = next.is_none_or(|line| { + line.is_blank() + || matches!( + LineStart::of(line, line.indent()), + Some(LineStart::Blockquote | LineStart::Item(_)) + ) + }); if let Some(last) = inner.last_mut() { last.item_end = item_end; } @@ -789,35 +833,26 @@ fn setext_depth(line: &Line<'_>) -> Option<u8> { /// Whether a line without its container prefix can continue an open paragraph of that container. /// Any list marker ends the paragraph here, even one that could not interrupt it elsewhere. fn is_lazy_continuation(line: &Line<'_>) -> bool { - !interrupts_paragraph(line) && ItemMarker::parse(line).is_none() + let start = LineStart::of(line, line.indent()); + !ends_paragraph(line, start.as_ref()) && !matches!(start, Some(LineStart::Item(_))) } /// Whether `line` ends the paragraph that is being collected. fn interrupts_paragraph(line: &Line<'_>) -> bool { - if line.is_blank() { - return true; - } - // Already accepted as a continuation by an enclosing container. - if line.lazy { - return false; - } - let Indent { columns, bytes: indent } = line.indent(); - let rest = &line.text[indent..]; - columns < line.code_indent() - && (Fence::open(rest, line.flavor.has_math()).is_some() - || atx_depth(rest).is_some() - || is_thematic_break(line, indent) - || blockquote_marker(line).is_some() - || footnote_marker(line).is_some() - || html_start(line, rest).is_some_and(|kind| kind != HtmlKind::Complete) - || ItemMarker::parse(line).is_some_and(|marker| marker.interrupts_paragraph())) + ends_paragraph(line, LineStart::of(line, line.indent()).as_ref()) +} + +/// Whether `line`, which starts `start`, ends an open paragraph. +fn ends_paragraph(line: &Line<'_>, start: Option<&LineStart<'_>>) -> bool { + // A lazy line was already accepted as a continuation by an enclosing container. + line.is_blank() || (!line.lazy && start.is_some_and(LineStart::interrupts_paragraph)) } /// Whether `line` ends a table. Unlike a paragraph, a table is also ended by a list item that is empty /// or that is numbered from other than one. fn ends_table(line: &Line<'_>) -> bool { - interrupts_paragraph(line) - || (!line.lazy && line.indent().columns < line.code_indent() && ItemMarker::parse(line).is_some()) + let start = LineStart::of(line, line.indent()); + ends_paragraph(line, start.as_ref()) || (!line.lazy && matches!(start, Some(LineStart::Item(_)))) } /// Collects a paragraph or a setext heading starting at `lines[start]`, with the definitions at its @@ -939,7 +974,7 @@ fn paragraph_source(lines: &[Line<'_>]) -> InlineSource { } /// The start of a footnote definition: `[^label]:`. -struct FootnoteMarker<'a> { +pub(super) struct FootnoteMarker<'a> { label: &'a str, /// Bytes from the start of the line to the content. content: usize, diff --git a/crates/mq-markdown/src/parser/block/rule.rs b/crates/mq-markdown/src/parser/block/rule.rs index f0e3c9827..a3f0bdb74 100644 --- a/crates/mq-markdown/src/parser/block/rule.rs +++ b/crates/mq-markdown/src/parser/block/rule.rs @@ -4,12 +4,9 @@ //! types chained in [`Rules`], so the dispatch is static and the order is the precedence of the //! constructs. The paragraph is not a rule: it is what is left when no rule starts. -use super::{ - Interrupt, ItemMarker, atx_depth, atx_heading, blockquote, blockquote_marker, footnote, footnote_marker, - html_block, html_start, is_thematic_break, join_lines, list, paragraph, -}; +use super::{Interrupt, LineStart, atx_heading, blockquote, footnote, html_block, join_lines, list, paragraph}; use crate::node::{HorizontalRule, HorizontalRuleMarker, MdxJsEsm, Node, Position}; -use crate::parser::code::{Fence, fenced_code, indented_code}; +use crate::parser::code::{fenced_code, indented_code}; use crate::parser::line::{Indent, Line}; use crate::parser::mdx_flow::{FlowOutcome, mdx_flow}; use crate::parser::table; @@ -20,8 +17,9 @@ pub(super) struct Cx<'a, 'l> { pub(super) lines: &'l [Line<'a>], pub(super) index: usize, pub(super) line: &'l Line<'a>, - /// The indentation of `line`, and the text after it. + /// The indentation of `line`, what the line starts, and the text after the indentation. pub(super) indent: Indent, + pub(super) start: Option<LineStart<'a>>, pub(super) rest: &'a str, pub(super) depth: usize, /// Whether containers can still be nested. @@ -120,7 +118,9 @@ pub(super) struct FencedCode; impl BlockRule for FencedCode { fn parse(cx: &Cx<'_, '_>, blocks: &mut Vec<Block>) -> Option<Step> { - let fence = Fence::open(cx.rest, cx.line.flavor.has_math())?; + let Some(LineStart::Fence(fence)) = &cx.start else { + return None; + }; // The end of an unclosed fence without content is quirky right after a container. let own_end = cx.closed_container && !(cx.separated @@ -129,7 +129,7 @@ impl BlockRule for FencedCode { cx.lines, cx.index, cx.indent.bytes, - &fence, + fence, own_end, blocks, ))) @@ -140,7 +140,9 @@ pub(super) struct AtxHeading; impl BlockRule for AtxHeading { fn parse(cx: &Cx<'_, '_>, blocks: &mut Vec<Block>) -> Option<Step> { - let depth = atx_depth(cx.rest)?; + let Some(LineStart::Atx(depth)) = cx.start else { + return None; + }; blocks.push(atx_heading(cx.line, cx.indent.bytes, depth)); Some(Step::to(cx.index + 1)) } @@ -150,7 +152,7 @@ pub(super) struct ThematicBreak; impl BlockRule for ThematicBreak { fn parse(cx: &Cx<'_, '_>, blocks: &mut Vec<Block>) -> Option<Step> { - if !is_thematic_break(cx.line, cx.indent.bytes) { + if !matches!(cx.start, Some(LineStart::ThematicBreak)) { return None; } blocks.push(Block::Node(Node::HorizontalRule(HorizontalRule { @@ -168,7 +170,7 @@ pub(super) struct Blockquote; impl BlockRule for Blockquote { fn parse(cx: &Cx<'_, '_>, blocks: &mut Vec<Block>) -> Option<Step> { - if !cx.containers || blockquote_marker(cx.line).is_none() { + if !cx.containers || !matches!(cx.start, Some(LineStart::Blockquote)) { return None; } Some(Step::container(blockquote( @@ -185,12 +187,16 @@ pub(super) struct List; impl BlockRule for List { fn parse(cx: &Cx<'_, '_>, blocks: &mut Vec<Block>) -> Option<Step> { - let marker = ItemMarker::parse(cx.line) - .filter(|marker| cx.containers && (!cx.interrupting.list || marker.interrupts_paragraph()))?; + let Some(LineStart::Item(marker)) = &cx.start else { + return None; + }; + if !(cx.containers && (!cx.interrupting.list || marker.interrupts_paragraph())) { + return None; + } Some(Step::container(list( cx.lines, cx.index, - &marker, + marker, cx.depth, cx.interrupting, blocks, @@ -202,7 +208,9 @@ pub(super) struct Html; impl BlockRule for Html { fn parse(cx: &Cx<'_, '_>, blocks: &mut Vec<Block>) -> Option<Step> { - let kind = html_start(cx.line, cx.rest)?; + let Some(LineStart::Html(kind)) = cx.start else { + return None; + }; Some(Step::to(html_block(cx.lines, cx.index, kind, blocks))) } } @@ -211,11 +219,16 @@ pub(super) struct Footnote; impl BlockRule for Footnote { fn parse(cx: &Cx<'_, '_>, blocks: &mut Vec<Block>) -> Option<Step> { - let marker = footnote_marker(cx.line).filter(|_| cx.containers)?; + let Some(LineStart::Footnote(marker)) = &cx.start else { + return None; + }; + if !cx.containers { + return None; + } Some(Step::container(footnote( cx.lines, cx.index, - &marker, + marker, cx.depth, cx.interrupting, blocks, diff --git a/crates/mq-markdown/src/parser/code.rs b/crates/mq-markdown/src/parser/code.rs index f52c3f1af..47ab39075 100644 --- a/crates/mq-markdown/src/parser/code.rs +++ b/crates/mq-markdown/src/parser/code.rs @@ -1,6 +1,7 @@ //! Fenced and indented code, and math fences. use super::block::CODE_INDENT; +use super::flavor::Flavor; use super::inline; use super::line::{Indent, Line}; use super::tree::{Block, FencedBlock}; @@ -15,8 +16,9 @@ pub(super) struct Fence<'a> { } impl<'a> Fence<'a> { - /// Parses `rest` (indent already removed) as an opening fence. - pub(super) fn open(rest: &'a str, math: bool) -> Option<Self> { + /// Parses `rest` (indent already removed) as an opening fence, of math too when `flavor` has it. + pub(super) fn open(rest: &'a str, flavor: Flavor) -> Option<Self> { + let math = flavor.has_math(); let marker = *rest .as_bytes() .first() From 04546d91f8d2cee96f6b1db61fcd090aeb833a46 Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Thu, 8 Oct 2026 13:09:29 +0900 Subject: [PATCH 45/88] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor(markdown):?= =?UTF-8?q?=20report=20invalid=20MDX=20as=20typed=20errors=20with=20a=20po?= =?UTF-8?q?sition?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MDX errors were strings from the tag parser to resolve, with positions written into some messages by hand. They are now an MdxErrorKind that says what is wrong (an unexpected character and where in the tag, an unclosed expression or element, a mismatched closing tag, ...), and an MdxError that also records where it was found. The Parsed, Fallback and Stop results of the tag parser carry the kind and its offset, which inline and flow parsing turn into a position. At the API boundary the error becomes an MdxDiagnostic, a miette diagnostic that labels the position in the document, so the CLI shows the line that is wrong. The message text is unchanged. MdxDiagnostic, MdxError, MdxErrorKind, MdxFound and MdxPlace are exported so callers can match on the kind, and Point implements Eq. --- crates/mq-markdown/src/lib.rs | 1 + crates/mq-markdown/src/node.rs | 2 +- crates/mq-markdown/src/parser.rs | 6 +- crates/mq-markdown/src/parser/error.rs | 287 ++++++++++++++++++++ crates/mq-markdown/src/parser/inline.rs | 64 +++-- crates/mq-markdown/src/parser/mdx.rs | 86 +++--- crates/mq-markdown/src/parser/mdx_flow.rs | 28 +- crates/mq-markdown/src/parser/resolve.rs | 52 ++-- crates/mq-markdown/src/parser/tree.rs | 3 +- crates/mq-markdown/tests/mdx_error_tests.rs | 83 ++++++ 10 files changed, 504 insertions(+), 108 deletions(-) create mode 100644 crates/mq-markdown/src/parser/error.rs create mode 100644 crates/mq-markdown/tests/mdx_error_tests.rs diff --git a/crates/mq-markdown/src/lib.rs b/crates/mq-markdown/src/lib.rs index 153326c4b..f6ff14efd 100644 --- a/crates/mq-markdown/src/lib.rs +++ b/crates/mq-markdown/src/lib.rs @@ -98,6 +98,7 @@ pub use node::{ attr_value::AttrValue, }; pub use node_kind::{AttrSpec, AttrType, NodeKind}; +pub use parser::{MdxDiagnostic, MdxError, MdxErrorKind, MdxFound, MdxPlace}; #[cfg(feature = "wikilink")] pub use node::WikiLink; diff --git a/crates/mq-markdown/src/node.rs b/crates/mq-markdown/src/node.rs index 0a8fb5c23..612b83a52 100644 --- a/crates/mq-markdown/src/node.rs +++ b/crates/mq-markdown/src/node.rs @@ -975,7 +975,7 @@ pub struct Position { pub end: Point, } -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Eq)] #[cfg_attr( feature = "json", derive(serde::Serialize, serde::Deserialize), diff --git a/crates/mq-markdown/src/parser.rs b/crates/mq-markdown/src/parser.rs index 7bc24d042..f46985fb8 100644 --- a/crates/mq-markdown/src/parser.rs +++ b/crates/mq-markdown/src/parser.rs @@ -11,6 +11,7 @@ mod block; mod callout; mod code; mod definition; +pub(crate) mod error; mod flavor; mod html_flow; mod inline; @@ -26,6 +27,7 @@ mod tree; use crate::node::Node; use flavor::Flavor; +pub use error::{MdxDiagnostic, MdxError, MdxErrorKind, MdxFound, MdxPlace}; pub(crate) use inline::{is_autolink_email, normalize, unescape}; /// Parses `content` into a flat list of nodes. @@ -37,7 +39,7 @@ pub(crate) fn parse(content: &str) -> miette::Result<Vec<Node>> { /// Parses `content` into a flat list of nodes, with frontmatter at the start when `frontmatter` is set. pub(crate) fn parse_with(content: &str, frontmatter: bool) -> miette::Result<Vec<Node>> { resolve::resolve(block::parse(content, Flavor::Markdown, frontmatter), Flavor::Markdown) - .map_err(|message| miette::miette!(message)) + .map_err(|error| miette::Report::new(error.into_diagnostic(content))) } /// Renders `content` as HTML. @@ -54,7 +56,7 @@ pub(crate) fn parse_mdx(content: &str) -> miette::Result<Vec<Node>> { /// Parses `content` as MDX, with frontmatter at the start when `frontmatter` is set. pub(crate) fn parse_mdx_with(content: &str, frontmatter: bool) -> miette::Result<Vec<Node>> { resolve::resolve(block::parse(content, Flavor::Mdx, frontmatter), Flavor::Mdx) - .map_err(|message| miette::miette!(message)) + .map_err(|error| miette::Report::new(error.into_diagnostic(content))) } #[cfg(test)] diff --git a/crates/mq-markdown/src/parser/error.rs b/crates/mq-markdown/src/parser/error.rs new file mode 100644 index 000000000..79f717ca6 --- /dev/null +++ b/crates/mq-markdown/src/parser/error.rs @@ -0,0 +1,287 @@ +//! Errors in MDX, the only content that can be invalid. + +use super::tree::InlineSource; +use crate::node::Point; +use std::fmt; + +/// What a JSX tag has where something else must be. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum MdxFound { + Char(char), + EndOfFile, +} + +impl fmt::Display for MdxFound { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + MdxFound::Char(char) => write!(f, "character `{char}` (U+{:04X})", *char as u32), + MdxFound::EndOfFile => write!(f, "end of file"), + } + } +} + +/// The part of a JSX tag where an unexpected character is. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum MdxPlace { + BeforeName, + AfterName, + BeforeMemberName, + BeforeLocalName, + BeforeAttributeName, + InAttributeName, + BeforeLocalAttributeName, + InLocalAttributeName, + AfterAttributeName, + BeforeAttributeValue, + AfterSelfClosingSlash, +} + +impl fmt::Display for MdxPlace { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + MdxPlace::BeforeName => "before name", + MdxPlace::AfterName => "after name", + MdxPlace::BeforeMemberName => "before member name", + MdxPlace::BeforeLocalName => "before local name", + MdxPlace::BeforeAttributeName => "before attribute name", + MdxPlace::InAttributeName => "in attribute name", + MdxPlace::BeforeLocalAttributeName => "before local attribute name", + MdxPlace::InLocalAttributeName => "in local attribute name", + MdxPlace::AfterAttributeName => "after attribute name", + MdxPlace::BeforeAttributeValue => "before attribute value", + MdxPlace::AfterSelfClosingSlash => "after self-closing slash", + }) + } +} + +/// What is wrong with invalid MDX. +#[derive(Clone, Debug, PartialEq, Eq)] +#[non_exhaustive] +pub enum MdxErrorKind { + /// A character, or the end of the input, where a JSX tag cannot have it. + Unexpected { found: MdxFound, place: MdxPlace }, + /// An expression without the `}` that closes its `{`. + UnclosedExpression, + /// An attribute value without its closing quote. + UnclosedAttributeValue { quote: char }, + /// A closing tag that ends with `/>`. + SelfClosingSlashInClosingTag, + /// A closing tag with attributes. + AttributeInClosingTag, + /// A line of an expression or a tag inside a container that lacks the prefix of the container. + LazyLine, + /// Elements nested deeper than the parser follows. + TooDeep { limit: usize }, + /// A closing tag without an element open. + UnopenedClosingTag, + /// A closing tag whose name is not that of the open element. Elements on lines of their own say + /// where the open one starts. + MismatchedClosingTag { + closing: Option<String>, + opening: Option<String>, + opened_at: Option<Point>, + }, + /// An element on lines of its own that is not closed in its container. + UnclosedFlowElement { name: Option<String>, opened_at: Point }, + /// An element in text that is not closed in its paragraph. + UnclosedTextElement { name: Option<String> }, +} + +impl fmt::Display for MdxErrorKind { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let name = |name: &Option<String>| name.clone().unwrap_or_default(); + match self { + MdxErrorKind::Unexpected { found, place } => write!(f, "Unexpected {found} {place}"), + MdxErrorKind::UnclosedExpression => { + write!( + f, + "Unexpected end of file in expression, expected a corresponding closing brace for `{{`" + ) + } + MdxErrorKind::UnclosedAttributeValue { quote } => write!( + f, + "Unexpected end of file in attribute value, expected a corresponding closing quote `{quote}`" + ), + MdxErrorKind::SelfClosingSlashInClosingTag => write!( + f, + "Unexpected self-closing slash `/` in closing tag, expected the end of the tag" + ), + MdxErrorKind::AttributeInClosingTag => { + write!(f, "Unexpected attribute in closing tag, expected the end of the tag") + } + MdxErrorKind::LazyLine => write!( + f, + "Unexpected lazy line in expression in container, expected line to be prefixed with `>` when in a block quote, whitespace when in a list, etc" + ), + MdxErrorKind::TooDeep { limit } => write!(f, "Elements are nested deeper than {limit} levels"), + MdxErrorKind::UnopenedClosingTag => { + write!(f, "Unexpected closing slash `/` in tag, expected an open tag first") + } + MdxErrorKind::MismatchedClosingTag { + closing, + opening, + opened_at, + } => { + write!( + f, + "Unexpected closing tag `</{}>`, expected corresponding closing tag for `<{}>`", + name(closing), + name(opening) + )?; + match opened_at { + Some(at) => write!(f, " ({}:{})", at.line, at.column), + None => Ok(()), + } + } + MdxErrorKind::UnclosedFlowElement { + name: element, + opened_at, + } => write!( + f, + "Expected a closing tag for `<{}>` ({}:{}) before the end of its container", + name(element), + opened_at.line, + opened_at.column + ), + MdxErrorKind::UnclosedTextElement { name: element } => write!( + f, + "Expected a closing tag for `<{}>` before the end of the content", + name(element) + ), + } + } +} + +/// An error at an offset of the text that is being parsed, before the offset is known as a position. +#[derive(Clone, Debug)] +pub(super) struct Located { + pub(super) kind: MdxErrorKind, + pub(super) offset: usize, +} + +impl Located { + /// The error at its position in the document. + pub(super) fn in_source(self, source: &InlineSource) -> MdxError { + MdxError { + position: Some(source.point(self.offset)), + kind: self.kind, + } + } +} + +/// Invalid MDX, with where it is in the document when that is known. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct MdxError { + kind: MdxErrorKind, + position: Option<Point>, +} + +impl MdxError { + pub(super) fn new(kind: MdxErrorKind, position: Option<Point>) -> Self { + Self { kind, position } + } + + pub fn kind(&self) -> &MdxErrorKind { + &self.kind + } + + /// Where the error is, as a line and a column that count from 1. + pub fn position(&self) -> Option<&Point> { + self.position.as_ref() + } + + /// A diagnostic that points at the error in `content`, the document it was found in. + pub(super) fn into_diagnostic(self, content: &str) -> MdxDiagnostic { + let span = self + .position + .as_ref() + .and_then(|point| offset_of(content, point)) + .map(|offset| miette::SourceSpan::from((offset, 0))); + MdxDiagnostic { + error: self, + source_code: content.to_string(), + span, + } + } +} + +impl fmt::Display for MdxError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.kind.fmt(f) + } +} + +impl std::error::Error for MdxError {} + +/// An [`MdxError`] with the document, to show where in it the error is. +#[derive(Debug, miette::Diagnostic)] +#[diagnostic(code(mq_markdown::mdx))] +pub struct MdxDiagnostic { + error: MdxError, + #[source_code] + source_code: String, + #[label("here")] + span: Option<miette::SourceSpan>, +} + +impl MdxDiagnostic { + pub fn error(&self) -> &MdxError { + &self.error + } +} + +impl fmt::Display for MdxDiagnostic { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.error.fmt(f) + } +} + +impl std::error::Error for MdxDiagnostic {} + +/// The byte offset of `point` in `content`, where columns count bytes and a tab goes to the next +/// multiple of four, as the positions of nodes do. +fn offset_of(content: &str, point: &Point) -> Option<usize> { + let mut start = 0; + for _ in 1..point.line { + let rest = &content[start..]; + let eol = rest.find(['\n', '\r'])?; + start += eol + if rest[eol..].starts_with("\r\n") { 2 } else { 1 }; + } + let line = &content[start..]; + let line = &line[..line.find(['\n', '\r']).unwrap_or(line.len())]; + let mut column = 0; + for (index, char) in line.char_indices() { + if column + 1 >= point.column { + return Some(start + index); + } + column = if char == '\t' { + (column / 4 + 1) * 4 + } else { + column + char.len_utf8() + }; + } + Some(start + line.len()) +} + +#[cfg(test)] +mod tests { + use super::*; + use rstest::rstest; + + #[rstest] + #[case::first("abc", 1, 2, Some(1))] + #[case::second_line("a\nbc", 2, 2, Some(3))] + #[case::crlf("a\r\nbc", 2, 1, Some(3))] + #[case::tab("\tx", 1, 5, Some(1))] + #[case::multibyte("éx", 1, 3, Some(2))] + #[case::end_of_line("ab\ncd", 1, 3, Some(2))] + #[case::missing_line("ab", 3, 1, None)] + fn offset_of_point( + #[case] content: &str, + #[case] line: usize, + #[case] column: usize, + #[case] expected: Option<usize>, + ) { + assert_eq!(offset_of(content, &Point { line, column }), expected); + } +} diff --git a/crates/mq-markdown/src/parser/inline.rs b/crates/mq-markdown/src/parser/inline.rs index 64ba59c65..8fc166c33 100644 --- a/crates/mq-markdown/src/parser/inline.rs +++ b/crates/mq-markdown/src/parser/inline.rs @@ -20,6 +20,7 @@ pub(crate) use html::is_autolink_email; pub(crate) use link::normalize; pub(super) use tail::{destination, title_at}; +use super::error::{Located, MdxError, MdxErrorKind}; use super::flavor::Flavor; use super::mdx::{self, Fallback, Parsed, TagKind}; use super::resolve::References; @@ -119,13 +120,15 @@ pub(super) struct Context<'a> { /// in the text, which is looked for once. pub(super) emails: bool, /// The first error in the content, which only MDX has. - error: &'a RefCell<Option<String>>, + error: &'a RefCell<Option<MdxError>>, } impl Context<'_> { - /// Records an error, keeping the first one. - fn fail(&self, message: String) { - self.error.borrow_mut().get_or_insert(message); + /// Records an error at an offset of the source, keeping the first one. + fn fail(&self, error: Located) { + self.error + .borrow_mut() + .get_or_insert_with(|| error.in_source(self.source)); } pub(super) fn src(&self) -> &str { @@ -141,7 +144,7 @@ impl Context<'_> { } /// Parses raw inline content into nodes. -pub(super) fn parse(source: &InlineSource, doc: Document<'_>) -> Result<Vec<Node>, String> { +pub(super) fn parse(source: &InlineSource, doc: Document<'_>) -> Result<Vec<Node>, MdxError> { if source.text.is_empty() { return Ok(Vec::new()); } @@ -174,7 +177,7 @@ pub(super) fn parse(source: &InlineSource, doc: Document<'_>) -> Result<Vec<Node let nodes = to_nodes(items, &context); match error.into_inner() { - Some(message) => Err(message), + Some(error) => Err(error), None => Ok(nodes), } } @@ -225,7 +228,10 @@ pub(super) fn to_nodes(items: Vec<Item>, context: &Context<'_>) -> Vec<Node> { finish(&mut current, &mut nodes); match tag.kind { TagKind::Open if open.len() >= MAX_NESTING => { - context.fail(format!("Elements are nested deeper than {MAX_NESTING} levels")); + context.fail(Located { + kind: MdxErrorKind::TooDeep { limit: MAX_NESTING }, + offset: tag.start, + }); } TagKind::Open => open.push((tag, std::mem::take(&mut nodes))), TagKind::SelfClosing => { @@ -233,14 +239,20 @@ pub(super) fn to_nodes(items: Vec<Item>, context: &Context<'_>) -> Vec<Node> { nodes.push(text_element(context, tag, Vec::new(), end)); } TagKind::Close => match open.pop() { - None => context.fail("Unexpected closing slash `/` in tag, expected an open tag first".into()), + None => context.fail(Located { + kind: MdxErrorKind::UnopenedClosingTag, + offset: tag.start, + }), Some((opening, before)) => { if opening.name != tag.name { - context.fail(format!( - "Unexpected closing tag `</{}>`, expected corresponding closing tag for `<{}>`", - tag.name.as_deref().unwrap_or_default(), - opening.name.as_deref().unwrap_or_default() - )); + context.fail(Located { + kind: MdxErrorKind::MismatchedClosingTag { + closing: tag.name.clone(), + opening: opening.name.clone(), + opened_at: None, + }, + offset: tag.start, + }); } let children = std::mem::replace(&mut nodes, before); nodes.push(text_element(context, opening, children, tag.end)); @@ -281,10 +293,12 @@ pub(super) fn to_nodes(items: Vec<Item>, context: &Context<'_>) -> Vec<Node> { // Elements that never close are an error, and are closed where the content ends. while let Some((opening, before)) = open.pop() { - context.fail(format!( - "Expected a closing tag for `<{}>` before the end of the content", - opening.name.as_deref().unwrap_or_default() - )); + context.fail(Located { + kind: MdxErrorKind::UnclosedTextElement { + name: opening.name.clone(), + }, + offset: opening.start, + }); let end = opening.end; let children = std::mem::replace(&mut nodes, before); nodes.push(text_element(context, opening, children, end)); @@ -733,12 +747,8 @@ impl Scanner<'_> { }); self.push(item, end); } - Parsed::Error(message) => { - self.context.fail(message); - self.pos += 1; - } - Parsed::More(Fallback::Error(message)) => { - self.context.fail(message); + Parsed::Error(error) | Parsed::More(Fallback::Error(error)) => { + self.context.fail(error); self.pos += 1; } Parsed::Nok | Parsed::More(Fallback::Nok) => self.pos += 1, @@ -755,12 +765,8 @@ impl Scanner<'_> { }); self.push_node(node, end); } - Parsed::Error(message) => { - self.context.fail(message); - self.pos += 1; - } - Parsed::More(Fallback::Error(message)) => { - self.context.fail(message); + Parsed::Error(error) | Parsed::More(Fallback::Error(error)) => { + self.context.fail(error); self.pos += 1; } Parsed::Nok | Parsed::More(Fallback::Nok) => self.pos += 1, diff --git a/crates/mq-markdown/src/parser/mdx.rs b/crates/mq-markdown/src/parser/mdx.rs index 25c3cb043..584fe997f 100644 --- a/crates/mq-markdown/src/parser/mdx.rs +++ b/crates/mq-markdown/src/parser/mdx.rs @@ -1,6 +1,7 @@ //! MDX: JSX tags and expressions, ported from the rules of `markdown-rs` without a JavaScript parser, //! so expressions are only checked for balanced braces, outside of strings, template literals and comments. +use super::error::{Located, MdxErrorKind, MdxFound, MdxPlace}; use super::scan::Span; use crate::node::{MdxAttributeContent, MdxAttributeValue, MdxJsxAttribute}; use rustc_hash::FxHashMap; @@ -10,7 +11,7 @@ use std::cell::OnceCell; /// What the end of the input means for a construct it ends inside of. pub(super) enum Fallback { Nok, - Error(String), + Error(Located), } /// The outcome of parsing at a position. @@ -22,7 +23,7 @@ pub(super) enum Parsed<T> { /// is what the end of the input means for it. More(Fallback), /// The construct is invalid. - Error(String), + Error(Located), } #[derive(Clone, Copy, PartialEq)] @@ -55,14 +56,11 @@ fn id_continue(char: char) -> bool { enum Stop { Nok, More(Fallback), - Error(String), + Error(Located), } type Step<T> = Result<T, Stop>; -const UNCLOSED_EXPRESSION: &str = - "Unexpected end of file in expression, expected a corresponding closing brace for `{`"; - /// Where each `{` of a source closes, found in one pass the first time one is asked for, so that /// many unclosed braces do not each scan to the end of the input. #[derive(Default)] @@ -179,8 +177,18 @@ impl Cursor<'_> { Err(Stop::More(Fallback::Nok)) } - fn crash<T>(&self, message: String) -> Step<T> { - Err(Stop::Error(message)) + /// The construct is invalid at the current position. + fn crash<T>(&self, kind: MdxErrorKind) -> Step<T> { + Err(Stop::Error(Located { + kind, + offset: self.index, + })) + } + + /// The current character, or the end of the input, cannot be at `place`. + fn unexpected<T>(&self, place: MdxPlace) -> Step<T> { + let found = self.peek().map_or(MdxFound::EndOfFile, MdxFound::Char); + self.crash(MdxErrorKind::Unexpected { found, place }) } /// Skips whitespace, including line endings. Ends the tag at the end of the input. @@ -203,13 +211,6 @@ impl Cursor<'_> { Ok(&self.src[start..self.index]) } - fn describe(&self) -> String { - match self.peek() { - Some(char) => format!("character `{char}` (U+{:04X})", char as u32), - None => "end of file".to_string(), - } - } - /// The name of a tag, with its member (`.`) or namespace (`:`) parts, cleaned of whitespace. fn tag_name(&mut self) -> Step<String> { let mut name = self.name_part()?.to_string(); @@ -220,7 +221,7 @@ impl Cursor<'_> { self.bump(); self.skip_whitespace()?; if !self.peek().is_some_and(id_start) { - return self.crash(format!("Unexpected {} before member name", self.describe())); + return self.unexpected(MdxPlace::BeforeMemberName); } name.push('.'); name.push_str(self.name_part()?); @@ -231,7 +232,7 @@ impl Cursor<'_> { self.bump(); self.skip_whitespace()?; if !self.peek().is_some_and(id_start) { - return self.crash(format!("Unexpected {} before local name", self.describe())); + return self.unexpected(MdxPlace::BeforeLocalName); } name.push(':'); name.push_str(self.name_part()?); @@ -253,7 +254,10 @@ impl Cursor<'_> { }; let Some(end) = end else { self.index = self.src.len(); - return Err(Stop::More(Fallback::Error(UNCLOSED_EXPRESSION.into()))); + return Err(Stop::More(Fallback::Error(Located { + kind: MdxErrorKind::UnclosedExpression, + offset: open, + }))); }; self.index = end + 1; Ok(Span::new(start, end)) @@ -301,35 +305,35 @@ impl Cursor<'_> { } /// Checks the character after a name: whitespace or one of `allowed`. - fn end_of_name(&self, allowed: &[char], what: &str) -> Step<()> { + fn end_of_name(&self, allowed: &[char], place: MdxPlace) -> Step<()> { match self.peek() { None => self.end_of_input(), Some(char) if char.is_whitespace() || allowed.contains(&char) => Ok(()), - Some(_) => self.crash(format!("Unexpected {} in {what}", self.describe())), + Some(_) => self.unexpected(place), } } fn attribute(&mut self) -> Step<MdxAttributeContent> { let mut name = self.name_part()?.to_string(); - self.end_of_name(&['/', ':', '=', '>', '{'], "attribute name")?; + self.end_of_name(&['/', ':', '=', '>', '{'], MdxPlace::InAttributeName)?; self.skip_whitespace()?; if self.peek() == Some(':') { self.bump(); self.skip_whitespace()?; if !self.peek().is_some_and(id_start) { - return self.crash(format!("Unexpected {} before local attribute name", self.describe())); + return self.unexpected(MdxPlace::BeforeLocalAttributeName); } name.push(':'); name.push_str(self.name_part()?); - self.end_of_name(&['/', '=', '>', '{'], "local attribute name")?; + self.end_of_name(&['/', '=', '>', '{'], MdxPlace::InLocalAttributeName)?; self.skip_whitespace()?; } let name = SmolStr::new(name); if self.peek() != Some('=') { if !matches!(self.peek(), Some('/' | '>' | '{')) && !self.peek().is_some_and(id_start) { - return self.crash(format!("Unexpected {} after attribute name", self.describe())); + return self.unexpected(MdxPlace::AfterAttributeName); } return Ok(MdxAttributeContent::Property(MdxJsxAttribute { name, value: None })); } @@ -341,9 +345,10 @@ impl Cursor<'_> { self.bump(); let start = self.index; let Some(length) = self.src[start..].find(quote) else { - return Err(Stop::More(Fallback::Error(format!( - "Unexpected end of file in attribute value, expected a corresponding closing quote `{quote}`" - )))); + return Err(Stop::More(Fallback::Error(Located { + kind: MdxErrorKind::UnclosedAttributeValue { quote }, + offset: start - 1, + }))); }; self.index = start + length + 1; MdxAttributeValue::Literal(SmolStr::new(super::inline::decode_references( @@ -355,7 +360,7 @@ impl Cursor<'_> { MdxAttributeValue::Expression(self.expression_value(content)) } _ => { - return self.crash(format!("Unexpected {} before attribute value", self.describe())); + return self.unexpected(MdxPlace::BeforeAttributeValue); } }; Ok(MdxAttributeContent::Property(MdxJsxAttribute { @@ -383,7 +388,7 @@ impl Cursor<'_> { let name = match self.peek() { Some('>') => None, Some(char) if id_start(char) => Some(self.tag_name()?), - _ => return self.crash(format!("Unexpected {} before name", self.describe())), + _ => return self.unexpected(MdxPlace::BeforeName), }; if name.is_some() { self.end_of_tag_name()?; @@ -398,12 +403,10 @@ impl Cursor<'_> { self.bump(); self.skip_whitespace()?; if self.peek() != Some('>') { - return self.crash(format!("Unexpected {} after self-closing slash", self.describe())); + return self.unexpected(MdxPlace::AfterSelfClosingSlash); } if kind == TagKind::Close { - return self.crash( - "Unexpected self-closing slash `/` in closing tag, expected the end of the tag".into(), - ); + return self.crash(MdxErrorKind::SelfClosingSlashInClosingTag); } kind = TagKind::SelfClosing; } @@ -412,8 +415,11 @@ impl Cursor<'_> { break; } Some(char) if char == '{' || id_start(char) => { - if kind == TagKind::Close { - misplaced = Some("Unexpected attribute in closing tag, expected the end of the tag"); + if kind == TagKind::Close && misplaced.is_none() { + misplaced = Some(Located { + kind: MdxErrorKind::AttributeInClosingTag, + offset: self.index, + }); } let attribute = if char == '{' { let content = self.expression()?; @@ -425,12 +431,12 @@ impl Cursor<'_> { self.skip_whitespace()?; } None => return self.end_of_input(), - Some(_) => return self.crash(format!("Unexpected {} before attribute name", self.describe())), + Some(_) => return self.unexpected(MdxPlace::BeforeAttributeName), } } - if let Some(message) = misplaced { - return self.crash(message.into()); + if let Some(error) = misplaced { + return Err(Stop::Error(error)); } Ok(Tag { @@ -448,7 +454,7 @@ impl Cursor<'_> { Some('/' | '>' | '{') => Ok(()), // An attribute needs whitespace before it. Some(char) if id_start(char) && self.src[..self.index].ends_with(char::is_whitespace) => Ok(()), - Some(_) => self.crash(format!("Unexpected {} after name", self.describe())), + Some(_) => self.unexpected(MdxPlace::AfterName), } } } @@ -458,7 +464,7 @@ fn parsed<T>(step: Step<T>) -> Parsed<T> { Ok(value) => Parsed::Ok(value), Err(Stop::Nok) => Parsed::Nok, Err(Stop::More(fallback)) => Parsed::More(fallback), - Err(Stop::Error(message)) => Parsed::Error(message), + Err(Stop::Error(error)) => Parsed::Error(error), } } diff --git a/crates/mq-markdown/src/parser/mdx_flow.rs b/crates/mq-markdown/src/parser/mdx_flow.rs index 67c5b8db3..92f033f17 100644 --- a/crates/mq-markdown/src/parser/mdx_flow.rs +++ b/crates/mq-markdown/src/parser/mdx_flow.rs @@ -1,5 +1,6 @@ //! MDX flow content: JSX tags and expressions that make up whole lines. +use super::error::{Located, MdxError, MdxErrorKind}; use super::line::Line; use super::mdx::{self, Fallback, Parsed}; use super::tree::{Block, InlineSource, JsxTag}; @@ -10,7 +11,7 @@ pub(super) enum Flow { /// Not flow content. When some items were parsed, the offset where the last one ends. Nok(Option<usize>), More(Fallback), - Error(String), + Error(Located), /// The items, and the offset where the last one ends. Done(Vec<FlowItem>, usize), } @@ -37,7 +38,7 @@ fn flow_items(source: &InlineSource) -> Flow { } Parsed::Nok => return Flow::Nok(items.last().map(item_end)), Parsed::More(fallback) => return Flow::More(fallback), - Parsed::Error(message) => return Flow::Error(message), + Parsed::Error(error) => return Flow::Error(error), }, Some('{') => match mdx::expression(text, index, None) { Parsed::Ok((end, value)) => { @@ -47,7 +48,7 @@ fn flow_items(source: &InlineSource) -> Flow { } Parsed::Nok => return Flow::Nok(items.last().map(item_end)), Parsed::More(fallback) => return Flow::More(fallback), - Parsed::Error(message) => return Flow::Error(message), + Parsed::Error(error) => return Flow::Error(error), }, _ => return Flow::Nok(items.last().map(item_end)), }; @@ -99,14 +100,15 @@ pub(super) fn mdx_flow(lines: &[Line<'_>], start: usize, blocks: &mut Vec<Block> // The construct goes on: read more lines, unless the container has none left. Flow::More(_) if end < lines.len() => count *= 2, Flow::More(_) if container_end => { - blocks.push(Block::Error( - "Unexpected lazy line in expression in container, expected line to be prefixed with `>` when in a block quote, whitespace when in a list, etc".into(), - )); + blocks.push(Block::Error(MdxError::new( + MdxErrorKind::LazyLine, + Some(lines[start].point(0)), + ))); return FlowOutcome::Flow(lines.len()); } Flow::More(Fallback::Nok) => return FlowOutcome::Pending, - Flow::More(Fallback::Error(message)) => { - blocks.push(Block::Error(message)); + Flow::More(Fallback::Error(error)) => { + blocks.push(Block::Error(error.in_source(&source))); return FlowOutcome::Flow(lines.len()); } Flow::Nok(end) => { @@ -114,8 +116,8 @@ pub(super) fn mdx_flow(lines: &[Line<'_>], start: usize, blocks: &mut Vec<Block> end.map(|end| start + source.lines.partition_point(|line| line.offset < end).saturating_sub(1)); return FlowOutcome::Nok(last_line); } - Flow::Error(message) => { - blocks.push(Block::Error(message)); + Flow::Error(error) => { + blocks.push(Block::Error(error.in_source(&source))); return FlowOutcome::Flow(lines.len()); } Flow::Done(items, last) => { @@ -124,10 +126,8 @@ pub(super) fn mdx_flow(lines: &[Line<'_>], start: usize, blocks: &mut Vec<Block> .lines .partition_point(|line| line.offset < last) .saturating_sub(1); - if lines[start + 1..=last_line].iter().any(|line| line.lazy) { - blocks.push(Block::Error( - "Unexpected lazy line in expression in container, expected line to be prefixed with `>` when in a block quote, whitespace when in a list, etc".into(), - )); + if let Some(lazy) = lines[start + 1..=last_line].iter().find(|line| line.lazy) { + blocks.push(Block::Error(MdxError::new(MdxErrorKind::LazyLine, Some(lazy.point(0))))); return FlowOutcome::Flow(lines.len()); } for item in items { diff --git a/crates/mq-markdown/src/parser/resolve.rs b/crates/mq-markdown/src/parser/resolve.rs index 91216c21d..a11b13672 100644 --- a/crates/mq-markdown/src/parser/resolve.rs +++ b/crates/mq-markdown/src/parser/resolve.rs @@ -1,6 +1,7 @@ //! Turns the block tree into nodes: parses the inline content, pairs JSX tags and flattens lists and //! tables. +use super::error::{MdxError, MdxErrorKind}; use super::flavor::Flavor; use super::inline::{self, Document, MAX_NESTING}; use super::mdx::TagKind; @@ -19,7 +20,7 @@ pub(super) struct References { pub(super) footnotes: FxHashSet<String>, } -pub(super) fn resolve(blocks: Vec<Block>, flavor: Flavor) -> Result<Vec<Node>, String> { +pub(super) fn resolve(blocks: Vec<Block>, flavor: Flavor) -> Result<Vec<Node>, MdxError> { let mut references = References::default(); collect(&blocks, &mut references); @@ -67,24 +68,31 @@ impl Frames { } } - fn tag(&mut self, tag: JsxTag) -> Result<(), String> { + fn tag(&mut self, tag: JsxTag) -> Result<(), MdxError> { match tag.kind { TagKind::Open if self.open.len() >= MAX_NESTING => { - return Err(format!("Elements are nested deeper than {MAX_NESTING} levels")); + return Err(MdxError::new( + MdxErrorKind::TooDeep { limit: MAX_NESTING }, + Some(tag.position.start), + )); } TagKind::Open => self.open.push((tag, Vec::new())), TagKind::SelfClosing => self.push(element(tag, Vec::new(), None)), TagKind::Close => { let Some((open, children)) = self.open.pop() else { - return Err("Unexpected closing slash `/` in tag, expected an open tag first".into()); + return Err(MdxError::new( + MdxErrorKind::UnopenedClosingTag, + Some(tag.position.start), + )); }; if open.name != tag.name { - return Err(format!( - "Unexpected closing tag `</{}>`, expected corresponding closing tag for `<{}>` ({}:{})", - tag.name.as_deref().unwrap_or_default(), - open.name.as_deref().unwrap_or_default(), - open.position.start.line, - open.position.start.column + return Err(MdxError::new( + MdxErrorKind::MismatchedClosingTag { + closing: tag.name, + opening: open.name, + opened_at: Some(open.position.start), + }, + Some(tag.position.start), )); } let end = tag.position.end; @@ -107,7 +115,7 @@ fn element(tag: JsxTag, children: Vec<Node>, end: Option<crate::node::Point>) -> }) } -fn flatten(blocks: Vec<Block>, doc: Document<'_>, nodes: &mut Vec<Node>) -> Result<(), String> { +fn flatten(blocks: Vec<Block>, doc: Document<'_>, nodes: &mut Vec<Node>) -> Result<(), MdxError> { let mut frames = Frames { open: Vec::new(), root: Vec::new(), @@ -163,16 +171,18 @@ fn flatten(blocks: Vec<Block>, doc: Document<'_>, nodes: &mut Vec<Node>) -> Resu } } Block::Jsx(tag) => frames.tag(tag)?, - Block::Error(message) => return Err(message), + Block::Error(error) => return Err(error), } } - if let Some((open, _)) = frames.open.last() { - return Err(format!( - "Expected a closing tag for `<{}>` ({}:{}) before the end of its container", - open.name.as_deref().unwrap_or_default(), - open.position.start.line, - open.position.start.column + if let Some((open, _)) = frames.open.pop() { + let opened_at = open.position.start; + return Err(MdxError::new( + MdxErrorKind::UnclosedFlowElement { + name: open.name, + opened_at: opened_at.clone(), + }, + Some(opened_at), )); } nodes.append(&mut frames.root); @@ -181,7 +191,7 @@ fn flatten(blocks: Vec<Block>, doc: Document<'_>, nodes: &mut Vec<Node>) -> Resu } /// A block quote, or a callout when the `callout` feature is enabled and its first line is a callout header. -fn quote_node(quote: QuoteBlock, doc: Document<'_>) -> Result<Node, String> { +fn quote_node(quote: QuoteBlock, doc: Document<'_>) -> Result<Node, MdxError> { #[cfg(feature = "callout")] let (header, children) = { let mut children = quote.children; @@ -207,7 +217,7 @@ fn quote_node(quote: QuoteBlock, doc: Document<'_>) -> Result<Node, String> { Ok(Node::Blockquote(Blockquote { values, position })) } -fn inline_block(block: InlineBlock, doc: Document<'_>, nodes: &mut Vec<Node>) -> Result<(), String> { +fn inline_block(block: InlineBlock, doc: Document<'_>, nodes: &mut Vec<Node>) -> Result<(), MdxError> { let values = inline::parse(&block.source, doc)?; match block.kind { InlineKind::Paragraph => nodes.extend(values), @@ -221,7 +231,7 @@ fn inline_block(block: InlineBlock, doc: Document<'_>, nodes: &mut Vec<Node>) -> } /// Emits one flat `Node::List` per item, followed by the items of its nested lists one level deeper. -fn list_nodes(list: ListBlock, level: Level, doc: Document<'_>, nodes: &mut Vec<Node>) -> Result<(), String> { +fn list_nodes(list: ListBlock, level: Level, doc: Document<'_>, nodes: &mut Vec<Node>) -> Result<(), MdxError> { for (index, item) in list.items.into_iter().enumerate() { let (nested, others): (Vec<_>, Vec<_>) = item.children.into_iter().partition(|b| matches!(b, Block::List(_))); let mut values = Vec::new(); diff --git a/crates/mq-markdown/src/parser/tree.rs b/crates/mq-markdown/src/parser/tree.rs index d10f5bf9f..ef0050aab 100644 --- a/crates/mq-markdown/src/parser/tree.rs +++ b/crates/mq-markdown/src/parser/tree.rs @@ -3,6 +3,7 @@ //! Inline parsing needs to know every definition of the document, including those that come later, so //! it runs in [`super::resolve`] once all blocks are known. +use super::error::MdxError; use super::line::visual_column; use super::mdx::TagKind; use super::scan::eol_len; @@ -146,7 +147,7 @@ pub(super) enum Block { /// A JSX tag on a line of its own in MDX. Tags are paired when the blocks become nodes. Jsx(JsxTag), /// Invalid MDX, reported when the blocks become nodes. - Error(String), + Error(MdxError), } pub(super) struct FencedBlock { diff --git a/crates/mq-markdown/tests/mdx_error_tests.rs b/crates/mq-markdown/tests/mdx_error_tests.rs new file mode 100644 index 000000000..59a2c49b0 --- /dev/null +++ b/crates/mq-markdown/tests/mdx_error_tests.rs @@ -0,0 +1,83 @@ +//! Invalid MDX is reported as an `MdxDiagnostic` that says what is wrong and where. + +use miette::Diagnostic; +use mq_markdown::{Markdown, MdxDiagnostic, MdxErrorKind, MdxFound, MdxPlace, Point}; +use rstest::rstest; + +fn diagnostic(input: &str) -> MdxDiagnostic { + let report = Markdown::from_mdx_str(input).expect_err("the input is invalid MDX"); + let message = report.to_string(); + let diagnostic = report.downcast::<MdxDiagnostic>().expect("an MDX diagnostic"); + assert_eq!(diagnostic.to_string(), message); + diagnostic +} + +#[rstest] +#[case::unclosed_expression("a {b", MdxErrorKind::UnclosedExpression, 1, 3, 2)] +#[case::unclosed_attribute_value( + "a <b c=\"d", + MdxErrorKind::UnclosedAttributeValue { quote: '"' }, + 1, + 8, + 7 +)] +#[case::unexpected_after_name( + "<a!>", + MdxErrorKind::Unexpected { found: MdxFound::Char('!'), place: MdxPlace::AfterName }, + 1, + 3, + 2 +)] +#[case::unopened_closing_tag("</a>", MdxErrorKind::UnopenedClosingTag, 1, 1, 0)] +#[case::mismatched_flow_tag( + "<a>\n\nb\n\n</c>", + MdxErrorKind::MismatchedClosingTag { + closing: Some("c".into()), + opening: Some("a".into()), + opened_at: Some(Point { line: 1, column: 1 }), + }, + 5, + 1, + 8 +)] +#[case::mismatched_text_tag( + "x <a>b</c> y", + MdxErrorKind::MismatchedClosingTag { + closing: Some("c".into()), + opening: Some("a".into()), + opened_at: None, + }, + 1, + 7, + 6 +)] +#[case::unclosed_text_element("x <a>b", MdxErrorKind::UnclosedTextElement { name: Some("a".into()) }, 1, 3, 2)] +#[case::unclosed_flow_element( + "- <a>\n\nb", + MdxErrorKind::UnclosedFlowElement { name: Some("a".into()), opened_at: Point { line: 1, column: 3 } }, + 1, + 3, + 2 +)] +fn reports_kind_and_position( + #[case] input: &str, + #[case] kind: MdxErrorKind, + #[case] line: usize, + #[case] column: usize, + #[case] offset: usize, +) { + let diagnostic = diagnostic(input); + assert_eq!(diagnostic.error().kind(), &kind); + assert_eq!(diagnostic.error().position(), Some(&Point { line, column })); + let labels = diagnostic.labels().expect("a label").collect::<Vec<_>>(); + assert_eq!(labels.len(), 1); + assert_eq!(labels[0].offset(), offset); +} + +#[test] +fn keeps_the_message() { + assert_eq!( + diagnostic("<a!>").to_string(), + "Unexpected character `!` (U+0021) after name" + ); +} From bf7a90a57cb31220202b520519fa174953b30702 Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Thu, 8 Oct 2026 13:09:36 +0900 Subject: [PATCH 46/88] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor(markdown):?= =?UTF-8?q?=20split=20node.rs=20and=20share=20its=20child=20accessors?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit node.rs held the node types, their accessors, Markdown rendering, attribute access, link expansion and their tests in 7,000 lines. Rendering now lives in node/render.rs (with the tests of its escaping and fence helpers), attr/set_attr in node/attr.rs, the wikilink, embed and inline link expansion in node/expand.rs, and the remaining tests in node/tests.rs. set_children, into_fragment, apply_fragment, into_with_children_value and clear_text_position_at each matched every variant with children on their own. They now go through children_mut, fragment_values_mut and editable_values_mut, which keep the variant set each of them had, so the duplicated callout arm of apply_fragment and the per-variant rebuilds of into_with_children_value are gone. --- crates/mq-markdown/src/node.rs | 5342 +------------------------ crates/mq-markdown/src/node/attr.rs | 464 +++ crates/mq-markdown/src/node/expand.rs | 339 ++ crates/mq-markdown/src/node/render.rs | 1273 ++++++ crates/mq-markdown/src/node/tests.rs | 3032 ++++++++++++++ 5 files changed, 5200 insertions(+), 5250 deletions(-) create mode 100644 crates/mq-markdown/src/node/attr.rs create mode 100644 crates/mq-markdown/src/node/expand.rs create mode 100644 crates/mq-markdown/src/node/render.rs create mode 100644 crates/mq-markdown/src/node/tests.rs diff --git a/crates/mq-markdown/src/node.rs b/crates/mq-markdown/src/node.rs index 612b83a52..3fb5634bf 100644 --- a/crates/mq-markdown/src/node.rs +++ b/crates/mq-markdown/src/node.rs @@ -11,7 +11,17 @@ use std::{ type ColorPair<'a> = (Cow<'a, str>, Cow<'a, str>); +mod attr; pub mod attr_value; +mod expand; +mod render; +pub(crate) use render::{ + indent_lines, list_own_prefix_width, reindent_all_lines, render_before, render_cell_values, render_values_block, + values_to_string, +}; +use render::{render_link_destination, render_link_title}; +#[cfg(test)] +mod tests; /// Color theme for rendering markdown nodes with optional ANSI escape codes. /// @@ -1098,21 +1108,13 @@ impl Node { /// /// Leaf nodes become [`Node::Empty`]. This is the consuming counterpart to /// [`Self::to_fragment`]. - pub fn into_fragment(self) -> Node { + pub fn into_fragment(mut self) -> Node { + if let Some(values) = self.fragment_values_mut() { + return Self::Fragment(Fragment { + values: std::mem::take(values), + }); + } match self { - Node::List(List { values, .. }) - | Node::TableCell(TableCell { values, .. }) - | Node::TableRow(TableRow { values, .. }) - | Node::Link(Link { values, .. }) - | Node::Footnote(Footnote { values, .. }) - | Node::LinkRef(LinkRef { values, .. }) - | Node::Heading(Heading { values, .. }) - | Node::Blockquote(Blockquote { values, .. }) - | Node::Delete(Delete { values, .. }) - | Node::Emphasis(Emphasis { values, .. }) - | Node::Strong(Strong { values, .. }) => Self::Fragment(Fragment { values }), - #[cfg(feature = "callout")] - Node::Callout(Callout { values, .. }) => Self::Fragment(Fragment { values }), node @ Node::Fragment(_) => node, _ => Self::Empty, } @@ -1123,7 +1125,53 @@ impl Node { } fn _apply_fragment(node: &mut Node, fragment: Node) { - match node { + let (Some(values), Node::Fragment(Fragment { values: new_values })) = (node.fragment_values_mut(), fragment) + else { + return; + }; + let new_values = values + .iter() + .zip(new_values) + .map(|(current_value, new_value)| { + if new_value.is_empty() { + current_value.clone() + } else if new_value.is_fragment() { + let mut current_value = current_value.clone(); + Self::_apply_fragment(&mut current_value, new_value); + current_value + } else { + new_value + } + }) + .collect::<Vec<_>>(); + *values = new_values; + } + + /// The children of a node that has them, which [`Self::set_children`] replaces. + fn children_mut(&mut self) -> Option<&mut Vec<Node>> { + match self { + Node::Footnote(Footnote { values, .. }) + | Node::Link(Link { values, .. }) + | Node::Heading(Heading { values, .. }) + | Node::List(List { values, .. }) + | Node::TableCell(TableCell { values, .. }) + | Node::TableRow(TableRow { values, .. }) + | Node::Strong(Strong { values, .. }) + | Node::Blockquote(Blockquote { values, .. }) + | Node::Delete(Delete { values, .. }) + | Node::Emphasis(Emphasis { values, .. }) + | Node::Fragment(Fragment { values }) => Some(values), + #[cfg(feature = "callout")] + Node::Callout(Callout { values, .. }) => Some(values), + Node::MdxJsxFlowElement(MdxJsxFlowElement { children, .. }) + | Node::MdxJsxTextElement(MdxJsxTextElement { children, .. }) => Some(children), + _ => None, + } + } + + /// The children that a fragment made from this node holds, and that applying a fragment maps onto. + fn fragment_values_mut(&mut self) -> Option<&mut Vec<Node>> { + match self { Node::List(List { values, .. }) | Node::TableCell(TableCell { values, .. }) | Node::TableRow(TableRow { values, .. }) @@ -1134,512 +1182,30 @@ impl Node { | Node::Blockquote(Blockquote { values, .. }) | Node::Delete(Delete { values, .. }) | Node::Emphasis(Emphasis { values, .. }) - | Node::Strong(Strong { values, .. }) => { - if let Node::Fragment(Fragment { values: new_values }) = fragment { - let new_values = values - .iter() - .zip(new_values) - .map(|(current_value, new_value)| { - if new_value.is_empty() { - current_value.clone() - } else if new_value.is_fragment() { - let mut current_value = current_value.clone(); - Self::_apply_fragment(&mut current_value, new_value); - current_value - } else { - new_value - } - }) - .collect::<Vec<_>>(); - *values = new_values; - } - } + | Node::Strong(Strong { values, .. }) => Some(values), #[cfg(feature = "callout")] - Node::Callout(Callout { values, .. }) => { - if let Node::Fragment(Fragment { values: new_values }) = fragment { - let new_values = values - .iter() - .zip(new_values) - .map(|(current_value, new_value)| { - if new_value.is_empty() { - current_value.clone() - } else if new_value.is_fragment() { - let mut current_value = current_value.clone(); - Self::_apply_fragment(&mut current_value, new_value); - current_value - } else { - new_value - } - }) - .collect::<Vec<_>>(); - *values = new_values; - } - } - _ => {} + Node::Callout(Callout { values, .. }) => Some(values), + _ => None, } } - pub fn to_string_with(&self, options: &RenderOptions) -> String { - self.render_with_theme(options, &ColorTheme::PLAIN) - } - - /// Returns a colored string representation of this node using ANSI escape codes. - #[cfg(feature = "color")] - pub fn to_colored_string_with(&self, options: &RenderOptions) -> String { - self.render_with_theme(options, &ColorTheme::COLORED) - } - - pub(crate) fn render_with_theme(&self, options: &RenderOptions, theme: &ColorTheme<'_>) -> String { + /// The children whose value [`Self::into_with_children_value`] replaces and whose text position + /// [`Self::clear_text_position_at`] clears. + fn editable_values_mut(&mut self) -> Option<&mut Vec<Node>> { match self { - Self::List(List { - level, - checked, - values, - ordered, - index, - start, - marker, - .. - }) => { - let marker = if *ordered { - let delimiter = if *marker == Some(ListMarker::Paren) { ')' } else { '.' }; - format!("{}{}", start.unwrap_or(1) as usize + *index, delimiter) - } else if let Some(style) = &options.list_style { - style.to_string() - } else { - marker.and_then(ListMarker::style).unwrap_or_default().to_string() - }; - let checkbox = (*checked).map(|it| if it { "[x] " } else { "[ ] " }).unwrap_or(""); - let prefix_width = *level as usize * 2 + list_own_prefix_width(*ordered, *index, *start); - // Lines after the first are indented to the content of the item, whatever column they - // had in the source, which is not reliable after an escape. - let content = render_values_block(values, options, theme); - let content = indent_continuation(&content, prefix_width); - let (ms, me) = &theme.list_marker; - format!( - "{}{}{}{} {}{}", - " ".repeat(*level as usize), - ms, - marker, - me, - checkbox, - content - ) - } - Self::TableRow(TableRow { values, .. }) => { - let (ts, te) = &theme.table_separator; - let cells = values - .iter() - .map(|cell| cell.render_with_theme(options, theme)) - .collect::<Vec<_>>() - .join("|"); - format!("{}|{}{}|", ts, te, cells) - } - Self::TableCell(TableCell { values, .. }) => render_cell_values(values, options, theme), - Self::TableAlign(TableAlign { align, .. }) => { - let (ts, te) = &theme.table_separator; - format!("{}|{}|{}", ts, align.iter().map(|a| a.to_string()).join("|"), te) - } - Self::Blockquote(Blockquote { values, .. }) => { - let (bs, be) = &theme.blockquote_marker; - render_values_block(values, options, theme) - .split('\n') - .map(|line| format!("{}> {}{}", bs, be, line)) - .join("\n") - } + Node::Blockquote(Blockquote { values, .. }) + | Node::Delete(Delete { values, .. }) + | Node::Emphasis(Emphasis { values, .. }) + | Node::List(List { values, .. }) + | Node::TableCell(TableCell { values, .. }) + | Node::Strong(Strong { values, .. }) + | Node::LinkRef(LinkRef { values, .. }) + | Node::Heading(Heading { values, .. }) => Some(values), #[cfg(feature = "callout")] - Self::Callout(Callout { - kind, - fold, - title, - values, - .. - }) => { - let (bs, be) = &theme.blockquote_marker; - let fold = fold.map(String::from).unwrap_or_default(); - let header = match title.as_deref() { - Some(t) if !t.is_empty() => format!("[!{}]{} {}", kind, fold, t), - _ => format!("[!{}]{}", kind, fold), - }; - let header_line = format!("{}> {}{}", bs, be, header); - if values.is_empty() { - return header_line; - } - let body = render_values_block(values, options, theme); - if body.trim().is_empty() { - header_line - } else { - let body_lines = body - .split('\n') - .map(|line| format!("{}> {}{}", bs, be, line)) - .join("\n"); - format!("{}\n{}", header_line, body_lines) - } - } - #[cfg(feature = "embed")] - Self::Embed(Embed { target, display, .. }) => match display.as_deref() { - Some(d) if !d.is_empty() => format!("![[{}|{}]]", target, d), - _ => format!("![[{}]]", target), - }, - Self::Code(Code { - value, - lang, - fence, - meta, - position, - }) => { - let (cs, ce) = &theme.code; - if lang.is_some() || *fence { - let meta = meta.as_deref().map(|meta| format!(" {}", meta)).unwrap_or_default(); - let info = format!("{}{}", lang.as_deref().unwrap_or(""), meta); - // Empty body skips the content line so it doesn't gain a blank one. - let fence_str = code_fence(value, &info); - if value.is_empty() { - // Blank lines are not in the value, but the lines of the block tell of them. - let blank = position - .as_ref() - .map_or(0, |p| p.end.line.saturating_sub(p.start.line).saturating_sub(1)); - format!("{}{}{}\n{}{}{}", cs, fence_str, info, "\n".repeat(blank), fence_str, ce) - } else { - format!("{}{}{}\n{}\n{}{}", cs, fence_str, info, value, fence_str, ce) - } - } else { - value.lines().map(|line| format!("{} {}{}", cs, line, ce)).join("\n") - } - } - Self::Definition(Definition { - ident, - label, - url, - title, - .. - }) => { - let (us, ue) = &theme.link_url; - format!( - "[{}]: {}{}{}{}", - definition_label(label.as_deref(), ident), - us, - url.to_string_with(options), - ue, - title - .as_ref() - .map(|title| format!(" {}", title.to_string_with(options))) - .unwrap_or_default() - ) - } - Self::Delete(Delete { values, .. }) => { - let (ds, de) = &theme.delete; - format!("{}~~{}~~{}", ds, render_values(values, options, theme), de) - } - Self::Emphasis(Emphasis { values, .. }) => { - // A lone nested Emphasis child needs the other delimiter, or adjacent - // `*` `*` pairs fuse into `**` and reparse as Strong instead. - let delim = if matches!(values.as_slice(), [Self::Emphasis(_)]) { - "_" - } else { - "*" - }; - render_emphasis(values, delim, options, theme) - } - Self::Footnote(Footnote { values, ident, .. }) => { - format!( - "[^{}]: {}", - escape_brackets(ident), - render_values(values, options, theme) - ) - } - Self::FootnoteRef(FootnoteRef { label, ident, .. }) => { - format!("[^{}]", reference_label(label.as_deref(), ident)) - } - Self::Heading(Heading { depth, values, .. }) => { - let (hs, he) = &theme.heading; - let text = render_values(values, options, theme); - // A line ending at either end is not part of the heading text, so it is written as a - // character reference. - let text = { - let inner = text.trim_matches('\n'); - let leading = text.len() - text.trim_start_matches('\n').len(); - let trailing = text.len() - text.trim_end_matches('\n').len(); - if inner.is_empty() || (leading == 0 && trailing == 0) { - text.clone() - } else { - format!("{}{}{}", "&#10;".repeat(leading), inner, "&#10;".repeat(trailing)) - } - }; - // Multi-line content must stay setext for depths 1-2; ATX has no setext form. - if text.contains('\n') && matches!(depth, 1 | 2) { - let underline = if *depth == 1 { "===" } else { "---" }; - format!("{}{}\n{}{}", hs, text, underline, he) - } else { - // A trailing `#` run reads back as an ATX closing sequence and gets - // stripped as syntax rather than kept as heading text. - let text = text.replace('\n', " "); - let text = match text.rfind(|c: char| c != '#') { - Some(i) => { - let hash_start = i + text[i..].chars().next().map_or(1, char::len_utf8); - if hash_start < text.len() { - format!("{}\\{}", &text[..hash_start], &text[hash_start..]) - } else { - text - } - } - None if !text.is_empty() => format!("\\{text}"), - _ => text, - }; - format!("{}{} {}{}", hs, "#".repeat(*depth as usize), text, he) - } - } - Self::Html(Html { value, .. }) => { - let (hs, he) = &theme.html; - format!("{}{}{}", hs, value, he) - } - Self::Image(Image { alt, url, title, .. }) => { - let (is, ie) = &theme.image; - format!( - "{}![{}]({}{}){}", - is, - escape_label(alt), - render_link_destination(url, &options.link_url_style), - title - .as_deref() - .map(|it| format!(" {}", render_link_title(it, &options.link_title_style))) - .unwrap_or_default(), - ie - ) - } - // Shortcut is safe only if `alt` normalizes back to `ident`; `ident` is raw - // (already correctly escaped), so escape_label would double it up. - Self::ImageRef(ImageRef { alt, ident, label, .. }) => { - let (is, ie) = &theme.image; - let mismatched = crate::parser::normalize(alt) != ident.as_str(); - if mismatched || needs_broad_escaping(alt) { - format!( - "{}![{}][{}]{}", - is, - escape_label(alt), - reference_label(label.as_deref(), ident), - ie - ) - } else { - format!("{}![{}]{}", is, escape_label(alt), ie) - } - } - Self::CodeInline(CodeInline { value, .. }) => { - let (cs, ce) = &theme.code_inline; - let fence = code_span_fence(value); - // Padding avoids fusing with an edge backtick and protects a genuine - // leading+trailing space from the parser's own space-stripping rule. - let all_spaces = value.chars().all(|c| c == ' '); - if value.starts_with('`') - || value.ends_with('`') - || (!all_spaces && value.starts_with(' ') && value.ends_with(' ')) - { - format!("{}{} {} {}{}", cs, fence, value, fence, ce) - } else { - format!("{}{}{}{}{}", cs, fence, value, fence, ce) - } - } - Self::MathInline(MathInline { value, .. }) => { - let (ms, me) = &theme.math; - let fence = math_span_fence(value); - // Padding keeps a dollar sign at an end from fusing with the fence, and a genuine space at - // both ends from being removed when it is read. - let all_spaces = value.chars().all(|c| c == ' '); - if value.starts_with('$') - || value.ends_with('$') - || (!all_spaces && value.starts_with(' ') && value.ends_with(' ')) - { - format!("{}{} {} {}{}", ms, fence, value, fence, me) - } else { - format!("{}{}{}{}{}", ms, fence, value, fence, me) - } - } - Self::Link(Link { url, title, values, .. }) => { - let (ls, le) = &theme.link; - if title.is_none() - && let Some(target) = autolink_target(url.as_str(), values) - { - return format!("{ls}<{target}>{le}"); - } - format!( - "{}[{}]({}{}){}", - ls, - render_values(values, options, theme), - url.to_string_with(options), - title - .as_ref() - .map(|title| format!(" {}", title.to_string_with(options))) - .unwrap_or_default(), - le - ) - } - #[cfg(feature = "wikilink")] - Self::WikiLink(WikiLink { target, text, .. }) => { - let (ls, le) = &theme.link; - match text.as_deref() { - Some(t) if t != target.as_str() => format!("{}[[{}|{}]]{}", ls, target, t, le), - _ => format!("{}[[{}]]{}", ls, target, le), - } - } - // Same reasoning as ImageRef, plus the same broad-escaping fallback. - Self::LinkRef(LinkRef { - values, ident, label, .. - }) => { - let (ls, le) = &theme.link; - let rendered = render_values(values, options, theme); - let plain = values_to_value(values); - // The label of a shortcut reference is its text as written, markup included. - let written = render_values(values, options, &ColorTheme::PLAIN); - let mismatched = crate::parser::normalize(&written) != ident.as_str(); - - if mismatched || needs_broad_escaping(&plain) { - format!( - "{}[{}][{}]{}", - ls, - rendered, - reference_label(label.as_deref(), ident), - le - ) - } else { - format!("{}[{}]{}", ls, rendered, le) - } - } - Self::Math(Math { value, .. }) => { - let (ms, me) = &theme.math; - let fence = "$".repeat((longest_run(value, '$') + 1).max(2)); - format!("{}{}\n{}\n{}{}", ms, fence, value, fence, me) - } - // Only escape text parsed from real markdown (has a position); programmatic - // values like JSON output or attr lookups must stay untouched. - Self::Text(Text { value, position }) => { - if position.is_some() { - escape_text_with(value.clone(), options.mdx) - } else { - value.clone() - } - } - Self::MdxFlowExpression(mdx_flow_expression) => { - format!("{{{}}}", indent_expression(&mdx_flow_expression.value)) - } - Self::MdxJsxFlowElement(mdx_jsx_flow_element) => { - let name = mdx_jsx_flow_element.name.as_deref().unwrap_or_default(); - let attributes = if mdx_jsx_flow_element.attributes.is_empty() { - "".to_string() - } else { - format!( - " {}", - mdx_jsx_flow_element - .attributes - .iter() - .map(Self::mdx_attribute_content_to_string) - .join(" ") - ) - }; - - if mdx_jsx_flow_element.children.is_empty() { - // A fragment has no self-closing form. - if name.is_empty() { - format!("<{}></>", attributes) - } else { - format!("<{}{} />", name, attributes,) - } - } else { - // Each tag and the children sit on their own lines, or the children would read as - // inline content of the tag. - let children = render_values_block(&mdx_jsx_flow_element.children, options, theme); - // The lines of an expression are indented as they are written, which indenting them - // again would add to each time they are read. - let children = if mdx_jsx_flow_element.children.iter().any(has_multiline_expression) { - children - } else { - indent_continuation(&format!(" {}", children), 2) - }; - format!("<{}{}>\n{}\n</{}>", name, attributes, children, name) - } - } - Self::MdxJsxTextElement(mdx_jsx_text_element) => { - let name = mdx_jsx_text_element.name.as_deref().unwrap_or_default(); - let attributes = if mdx_jsx_text_element.attributes.is_empty() { - "".to_string() - } else { - format!( - " {}", - mdx_jsx_text_element - .attributes - .iter() - .map(Self::mdx_attribute_content_to_string) - .join(" ") - ) - }; - - if mdx_jsx_text_element.children.is_empty() { - // A fragment has no self-closing form. - if name.is_empty() { - format!("<{}></>", attributes) - } else { - format!("<{}{} />", name, attributes,) - } - } else { - format!( - "<{}{}>{}</{}>", - name, - attributes, - render_values(&mdx_jsx_text_element.children, options, theme), - name - ) - } - } - Self::MdxTextExpression(mdx_text_expression) => { - format!("{{{}}}", indent_expression(&mdx_text_expression.value)) - } - Self::MdxJsEsm(mdxjs_esm) => mdxjs_esm.value.to_string(), - Self::Strong(Strong { values, .. }) => { - let (ss, se) = &theme.strong; - let last = values.len().saturating_sub(1); - let content = values - .iter() - .enumerate() - .map(|(index, value)| match value { - // Emphasis at an end uses the other delimiter, or `*` and `**` fuse into `***`. - // An underscore does not open or close next to a letter or digit, so those keep `*`. - Self::Emphasis(Emphasis { values: inner, .. }) - if (index == 0 || index == last) - && !index - .checked_sub(1) - .is_some_and(|before| ends_with_word_char(&values[before])) - && !values.get(index + 1).is_some_and(starts_with_word_char) => - { - render_emphasis(inner, "_", options, theme) - } - value => value.render_with_theme(options, theme), - }) - .collect::<String>(); - format!("{}**{}**{}", ss, content, se) - } - Self::Yaml(Yaml { value, .. }) => { - let (fs, fe) = &theme.frontmatter; - format!("{}---\n{}\n---{}", fs, value, fe) - } - Self::Toml(Toml { value, .. }) => { - let (fs, fe) = &theme.frontmatter; - format!("{}+++\n{}\n+++{}", fs, value, fe) - } - Self::Break(_) => "\\\n".to_string(), - Self::HorizontalRule(HorizontalRule { marker, .. }) => { - let (hs, he) = &theme.horizontal_rule; - let marker = match marker { - Some(HorizontalRuleMarker::Dash) => '-', - Some(HorizontalRuleMarker::Underscore) => '_', - _ => '*', - }; - format!("{hs}{marker}{marker}{marker}{he}") - } - Self::Fragment(Fragment { values }) => values - .iter() - .filter(|value| !value.is_empty() && !value.is_empty_fragment()) - .map(|value| value.render_with_theme(options, theme)) - .join("\n"), - Self::Empty => String::new(), + Node::Callout(Callout { values, .. }) => Some(values), + Node::MdxJsxFlowElement(MdxJsxFlowElement { children, .. }) + | Node::MdxJsxTextElement(MdxJsxTextElement { children, .. }) => Some(children), + _ => None, } } @@ -1779,23 +1345,8 @@ impl Node { /// Sets the children nodes of the current node, if the node supports children. /// Nodes without children (e.g. `Text`, `Code`, `Image`) are left unchanged. pub fn set_children(&mut self, children: Vec<Node>) { - match self { - Node::Footnote(f) => f.values = children, - Node::Link(l) => l.values = children, - #[cfg(feature = "callout")] - Node::Callout(c) => c.values = children, - Node::Heading(h) => h.values = children, - Node::List(l) => l.values = children, - Node::TableCell(c) => c.values = children, - Node::TableRow(r) => r.values = children, - Node::Strong(s) => s.values = children, - Node::Blockquote(b) => b.values = children, - Node::Delete(d) => d.values = children, - Node::Emphasis(e) => e.values = children, - Node::Fragment(f) => f.values = children, - Node::MdxJsxFlowElement(m) => m.children = children, - Node::MdxJsxTextElement(m) => m.children = children, - _ => (), + if let Some(values) = self.children_mut() { + *values = children; } } @@ -1920,21 +1471,10 @@ impl Node { /// `into_with_children_value` would write into at `index`, leaving this node and other /// descendants untouched. Used to stop the renderer re-escaping an already-escaped leaf. pub fn clear_text_position_at(&mut self, index: usize) { - match self { - Self::Blockquote(v) => Self::clear_position_at(&mut v.values, index), - Self::Delete(v) => Self::clear_position_at(&mut v.values, index), - Self::Emphasis(v) => Self::clear_position_at(&mut v.values, index), - Self::List(v) => Self::clear_position_at(&mut v.values, index), - Self::TableCell(v) => Self::clear_position_at(&mut v.values, index), - Self::Strong(v) => Self::clear_position_at(&mut v.values, index), - Self::Heading(v) => Self::clear_position_at(&mut v.values, index), - Self::LinkRef(v) => Self::clear_position_at(&mut v.values, index), - Self::MdxJsxFlowElement(v) => Self::clear_position_at(&mut v.children, index), - Self::MdxJsxTextElement(v) => Self::clear_position_at(&mut v.children, index), - #[cfg(feature = "callout")] - Self::Callout(v) => Self::clear_position_at(&mut v.values, index), - Self::Text(text) => text.position = None, - _ => {} + if let Self::Text(text) = self { + text.position = None; + } else if let Some(values) = self.editable_values_mut() { + Self::clear_position_at(values, index); } } @@ -2353,4712 +1893,14 @@ impl Node { /// /// Prefer this over [`Self::with_children_value`] when the caller owns the node and does not /// need an unchanged copy. - pub fn into_with_children_value(self, value: &str, index: usize) -> Self { - match self { - Self::Blockquote(mut v) => { - Self::replace_value_at(&mut v.values, index, value); - - Self::Blockquote(v) - } - Self::Delete(mut v) => { - Self::replace_value_at(&mut v.values, index, value); - - Self::Delete(v) - } - Self::Emphasis(mut v) => { - Self::replace_value_at(&mut v.values, index, value); - - Self::Emphasis(v) - } - Self::List(mut v) => { - Self::replace_value_at(&mut v.values, index, value); - - Self::List(v) - } - Self::TableCell(mut v) => { - Self::replace_value_at(&mut v.values, index, value); - - Self::TableCell(v) - } - Self::Strong(mut v) => { - Self::replace_value_at(&mut v.values, index, value); - - Self::Strong(v) - } - Self::LinkRef(mut v) => { - Self::replace_value_at(&mut v.values, index, value); - - Self::LinkRef(v) - } - Self::Heading(mut v) => { - Self::replace_value_at(&mut v.values, index, value); - - Self::Heading(v) - } - Self::MdxJsxFlowElement(mut mdx) => { - Self::replace_value_at(&mut mdx.children, index, value); - - Self::MdxJsxFlowElement(MdxJsxFlowElement { - name: mdx.name, - attributes: mdx.attributes, - children: mdx.children, - ..mdx - }) - } - Self::MdxJsxTextElement(mut mdx) => { - Self::replace_value_at(&mut mdx.children, index, value); - - Self::MdxJsxTextElement(MdxJsxTextElement { - name: mdx.name, - attributes: mdx.attributes, - children: mdx.children, - ..mdx - }) - } - #[cfg(feature = "wikilink")] - a @ Self::WikiLink(_) => a, - #[cfg(feature = "callout")] - Self::Callout(mut c) => { - Self::replace_value_at(&mut c.values, index, value); - Self::Callout(c) - } - #[cfg(feature = "embed")] - a @ Self::Embed(_) => a, - a => a, - } - } - - /// Returns the value of the specified attribute, if present. - /// - /// `line`/`end_line` are handled here, not per-variant below, since `position` is common - /// to every node. - pub fn attr(&self, attr: &str) -> Option<AttrValue> { - match attr { - attr_keys::LINE => return self.position().map(|p| AttrValue::Integer(p.start.line as i64)), - attr_keys::END_LINE => return self.position().map(|p| AttrValue::Integer(p.end.line as i64)), - _ => {} - } - - match self { - Node::Footnote(Footnote { ident, values, .. }) => match attr { - attr_keys::IDENT => Some(AttrValue::String(ident.clone())), - attr_keys::VALUE => Some(AttrValue::String(values_to_string(values, &RenderOptions::default()))), - attr_keys::VALUES | attr_keys::CHILDREN => Some(AttrValue::Array(values.clone())), - _ => None, - }, - Node::Html(Html { value, .. }) => match attr { - attr_keys::VALUE => Some(AttrValue::String(value.clone())), - _ => None, - }, - Node::Text(Text { value, .. }) => match attr { - attr_keys::VALUE => Some(AttrValue::String(value.clone())), - _ => None, - }, - Node::Code(Code { - value, - lang, - meta, - fence, - .. - }) => match attr { - attr_keys::VALUE => Some(AttrValue::String(value.clone())), - attr_keys::LANG => lang.clone().map(AttrValue::String), - attr_keys::META => meta.clone().map(AttrValue::String), - attr_keys::FENCE => Some(AttrValue::Boolean(*fence)), - _ => None, - }, - Node::CodeInline(CodeInline { value, .. }) => match attr { - attr_keys::VALUE => Some(AttrValue::String(value.to_string())), - _ => None, - }, - Node::MathInline(MathInline { value, .. }) => match attr { - attr_keys::VALUE => Some(AttrValue::String(value.to_string())), - _ => None, - }, - Node::Math(Math { value, .. }) => match attr { - attr_keys::VALUE => Some(AttrValue::String(value.clone())), - _ => None, - }, - Node::Yaml(Yaml { value, .. }) => match attr { - attr_keys::VALUE => Some(AttrValue::String(value.clone())), - _ => None, - }, - Node::Toml(Toml { value, .. }) => match attr { - attr_keys::VALUE => Some(AttrValue::String(value.clone())), - _ => None, - }, - Node::Image(Image { alt, url, title, .. }) => match attr { - attr_keys::ALT => Some(AttrValue::String(alt.clone())), - attr_keys::URL => Some(AttrValue::String(url.clone())), - attr_keys::TITLE => title.clone().map(AttrValue::String), - _ => None, - }, - Node::ImageRef(ImageRef { alt, ident, label, .. }) => match attr { - attr_keys::ALT => Some(AttrValue::String(alt.clone())), - attr_keys::IDENT => Some(AttrValue::String(ident.clone())), - attr_keys::LABEL => label.clone().map(AttrValue::String), - _ => None, - }, - Node::Link(Link { url, title, values, .. }) => match attr { - attr_keys::URL => Some(AttrValue::String(url.as_str().to_string())), - attr_keys::TITLE => title.as_ref().map(|t| AttrValue::String(t.to_value())), - attr_keys::VALUE => Some(AttrValue::String(values_to_string(values, &RenderOptions::default()))), - attr_keys::VALUES | attr_keys::CHILDREN => Some(AttrValue::Array(values.clone())), - _ => None, - }, - #[cfg(feature = "wikilink")] - Node::WikiLink(WikiLink { target, text, .. }) => match attr { - attr_keys::URL => Some(AttrValue::String(target.clone())), - attr_keys::VALUE => Some(AttrValue::String(text.clone().unwrap_or_else(|| target.clone()))), - _ => None, - }, - #[cfg(feature = "callout")] - Node::Callout(Callout { - kind, - fold, - title, - values, - .. - }) => match attr { - attr_keys::KIND => Some(AttrValue::String(kind.clone())), - attr_keys::FOLD => fold.map(|fold| AttrValue::String(fold.to_string())), - attr_keys::TITLE => title.clone().map(AttrValue::String), - attr_keys::VALUE => Some(AttrValue::String(values_to_string(values, &RenderOptions::default()))), - attr_keys::VALUES | attr_keys::CHILDREN => Some(AttrValue::Array(values.clone())), - _ => None, - }, - #[cfg(feature = "embed")] - Node::Embed(Embed { target, display, .. }) => match attr { - attr_keys::URL => Some(AttrValue::String(target.clone())), - attr_keys::VALUE => Some(AttrValue::String(display.clone().unwrap_or_else(|| target.clone()))), - _ => None, - }, - Node::LinkRef(LinkRef { ident, label, .. }) => match attr { - attr_keys::IDENT => Some(AttrValue::String(ident.clone())), - attr_keys::LABEL => label.clone().map(AttrValue::String), - _ => None, - }, - Node::FootnoteRef(FootnoteRef { ident, label, .. }) => match attr { - attr_keys::IDENT => Some(AttrValue::String(ident.clone())), - attr_keys::LABEL => label.clone().map(AttrValue::String), - _ => None, - }, - Node::Definition(Definition { - ident, - url, - title, - label, - .. - }) => match attr { - attr_keys::IDENT => Some(AttrValue::String(ident.clone())), - attr_keys::URL => Some(AttrValue::String(url.as_str().to_string())), - attr_keys::TITLE => title.as_ref().map(|t| AttrValue::String(t.to_value())), - attr_keys::LABEL => label.clone().map(AttrValue::String), - _ => None, - }, - Node::Heading(Heading { depth, values, .. }) => match attr { - attr_keys::DEPTH | attr_keys::LEVEL => Some(AttrValue::Integer(*depth as i64)), - attr_keys::VALUE => Some(AttrValue::String(values_to_string(values, &RenderOptions::default()))), - attr_keys::VALUES | attr_keys::CHILDREN => Some(AttrValue::Array(values.clone())), - _ => None, - }, - Node::List(List { - index, - level, - ordered, - checked, - values, - .. - }) => match attr { - attr_keys::INDEX => Some(AttrValue::Integer(*index as i64)), - attr_keys::LEVEL => Some(AttrValue::Integer(*level as i64)), - attr_keys::ORDERED => Some(AttrValue::Boolean(*ordered)), - attr_keys::CHECKED => checked.map(AttrValue::Boolean), - attr_keys::VALUE => Some(AttrValue::String(values_to_string(values, &RenderOptions::default()))), - attr_keys::VALUES | attr_keys::CHILDREN => Some(AttrValue::Array(values.clone())), - _ => None, - }, - Node::TableCell(TableCell { - column, row, values, .. - }) => match attr { - attr_keys::COLUMN => Some(AttrValue::Integer(*column as i64)), - attr_keys::ROW => Some(AttrValue::Integer(*row as i64)), - attr_keys::VALUE => Some(AttrValue::String(values_to_string(values, &RenderOptions::default()))), - attr_keys::VALUES | attr_keys::CHILDREN => Some(AttrValue::Array(values.clone())), - _ => None, - }, - Node::TableAlign(TableAlign { align, .. }) => match attr { - attr_keys::ALIGN => Some(AttrValue::String( - align.iter().map(|a| a.to_string()).collect::<Vec<_>>().join(","), - )), - _ => None, - }, - Node::MdxFlowExpression(MdxFlowExpression { value, .. }) => match attr { - attr_keys::VALUE => Some(AttrValue::String(value.to_string())), - _ => None, - }, - Node::MdxTextExpression(MdxTextExpression { value, .. }) => match attr { - attr_keys::VALUE => Some(AttrValue::String(value.to_string())), - _ => None, - }, - Node::MdxJsEsm(MdxJsEsm { value, .. }) => match attr { - attr_keys::VALUE => Some(AttrValue::String(value.to_string())), - _ => None, - }, - Node::MdxJsxFlowElement(MdxJsxFlowElement { name, children, .. }) => match attr { - attr_keys::NAME => name.clone().map(AttrValue::String), - attr_keys::VALUES | attr_keys::CHILDREN => Some(AttrValue::Array(children.clone())), - _ => None, - }, - Node::MdxJsxTextElement(MdxJsxTextElement { name, children, .. }) => match attr { - attr_keys::NAME => name.as_ref().map(|n| AttrValue::String(n.to_string())), - attr_keys::VALUES | attr_keys::CHILDREN => Some(AttrValue::Array(children.clone())), - _ => None, - }, - Node::Strong(Strong { values, .. }) - | Node::Blockquote(Blockquote { values, .. }) - | Node::Delete(Delete { values, .. }) - | Node::Emphasis(Emphasis { values, .. }) - | Node::TableRow(TableRow { values, .. }) - | Node::Fragment(Fragment { values, .. }) => match attr { - attr_keys::VALUE => Some(AttrValue::String(values_to_string(values, &RenderOptions::default()))), - attr_keys::VALUES | attr_keys::CHILDREN => Some(AttrValue::Array(values.clone())), - _ => None, - }, - Node::Break(_) | Node::HorizontalRule(_) | Node::Empty => None, + pub fn into_with_children_value(mut self, value: &str, index: usize) -> Self { + if let Some(values) = self.editable_values_mut() { + Self::replace_value_at(values, index, value); } + self } - - /// Sets the value of the specified attribute for the node, if supported. - pub fn set_attr(&mut self, attr: &str, value: impl Into<AttrValue>) { - let value = value.into(); - let value_str = value.as_string(); - - match self { - Node::Footnote(f) => { - if attr == attr_keys::IDENT { - f.ident = value_str; - } - } - Node::Html(h) => { - if attr == attr_keys::VALUE { - h.value = value_str; - } - } - Node::Text(t) => { - if attr == attr_keys::VALUE { - t.value = value_str; - } - } - Node::Code(c) => match attr { - attr_keys::VALUE => { - c.value = value_str; - } - attr_keys::LANG | "language" => { - c.lang = if value_str.is_empty() { None } else { Some(value_str) }; - } - attr_keys::META => { - c.meta = if value_str.is_empty() { None } else { Some(value_str) }; - } - attr_keys::FENCE => { - c.fence = match value { - AttrValue::Boolean(b) => b, - _ => value_str == "true", - }; - } - _ => (), - }, - Node::CodeInline(ci) => { - if attr == attr_keys::VALUE { - ci.value = value_str.into(); - } - } - Node::MathInline(mi) => { - if attr == attr_keys::VALUE { - mi.value = value_str.into(); - } - } - Node::Math(m) => { - if attr == attr_keys::VALUE { - m.value = value_str; - } - } - Node::Yaml(y) => { - if attr == attr_keys::VALUE { - y.value = value_str; - } - } - Node::Toml(t) => { - if attr == attr_keys::VALUE { - t.value = value_str; - } - } - Node::Image(i) => match attr { - attr_keys::ALT => { - i.alt = value_str; - } - attr_keys::URL => { - i.url = value_str; - } - attr_keys::TITLE => { - i.title = if value_str.is_empty() { None } else { Some(value_str) }; - } - _ => (), - }, - Node::ImageRef(i) => match attr { - attr_keys::ALT => { - i.alt = value_str; - } - attr_keys::IDENT => { - i.ident = value_str; - } - attr_keys::LABEL => { - i.label = if value_str.is_empty() { None } else { Some(value_str) }; - } - _ => (), - }, - Node::Link(l) => match attr { - attr_keys::URL => { - l.url = Url::new(value_str); - } - attr_keys::TITLE => { - l.title = if value_str.is_empty() { - None - } else { - Some(Title::new(value_str)) - }; - } - _ => (), - }, - Node::LinkRef(l) => match attr { - attr_keys::IDENT => { - l.ident = value_str; - } - attr_keys::LABEL => { - l.label = if value_str.is_empty() { None } else { Some(value_str) }; - } - _ => (), - }, - Node::FootnoteRef(f) => match attr { - attr_keys::IDENT => { - f.ident = value_str; - } - attr_keys::LABEL => { - f.label = if value_str.is_empty() { None } else { Some(value_str) }; - } - _ => (), - }, - Node::Definition(d) => match attr { - attr_keys::IDENT => { - d.ident = value_str; - } - attr_keys::URL => { - d.url = Url::new(value_str); - } - attr_keys::TITLE => { - d.title = if value_str.is_empty() { - None - } else { - Some(Title::new(value_str)) - }; - } - attr_keys::LABEL => { - d.label = if value_str.is_empty() { None } else { Some(value_str) }; - } - _ => (), - }, - Node::Heading(h) => match attr { - attr_keys::DEPTH | attr_keys::LEVEL => { - h.depth = match value { - AttrValue::Integer(i) => i as u8, - _ => value_str.parse::<u8>().unwrap_or(h.depth), - }; - } - _ => (), - }, - Node::List(l) => match attr { - attr_keys::INDEX => { - l.index = match value { - AttrValue::Integer(i) => i as usize, - _ => value_str.parse::<usize>().unwrap_or(l.index), - }; - } - attr_keys::LEVEL => { - l.level = match value { - AttrValue::Integer(i) => i as u8, - _ => value_str.parse::<u8>().unwrap_or(l.level), - }; - } - attr_keys::ORDERED => { - l.ordered = match value { - AttrValue::Boolean(b) => b, - _ => value_str == "true", - }; - } - attr_keys::CHECKED => { - l.checked = if value_str.is_empty() { - None - } else { - Some(match value { - AttrValue::Boolean(b) => b, - _ => value_str == "true", - }) - }; - } - _ => (), - }, - Node::TableCell(c) => match attr { - attr_keys::COLUMN => { - c.column = match value { - AttrValue::Integer(i) => i as usize, - _ => value_str.parse::<usize>().unwrap_or(c.column), - }; - } - attr_keys::ROW => { - c.row = match value { - AttrValue::Integer(i) => i as usize, - _ => value_str.parse::<usize>().unwrap_or(c.row), - }; - } - _ => (), - }, - Node::TableAlign(th) => { - if attr == attr_keys::ALIGN { - th.align = value_str.split(',').map(|s| s.trim().into()).collect(); - } - } - Node::MdxFlowExpression(m) => { - if attr == attr_keys::VALUE { - m.value = value_str.into(); - } - } - Node::MdxTextExpression(m) => { - if attr == attr_keys::VALUE { - m.value = value_str.into(); - } - } - Node::MdxJsEsm(m) => { - if attr == attr_keys::VALUE { - m.value = value_str.into(); - } - } - Node::MdxJsxFlowElement(m) => { - if attr == attr_keys::NAME { - m.name = if value_str.is_empty() { None } else { Some(value_str) }; - } - } - Node::MdxJsxTextElement(m) => { - if attr == attr_keys::NAME { - m.name = if value_str.is_empty() { - None - } else { - Some(value_str.into()) - }; - } - } - Node::Delete(_) - | Node::Blockquote(_) - | Node::Emphasis(_) - | Node::Strong(_) - | Node::TableRow(_) - | Node::Break(_) - | Node::HorizontalRule(_) - | Node::Fragment(_) - | Node::Empty => (), - #[cfg(feature = "wikilink")] - Node::WikiLink(w) => match attr { - attr_keys::URL => w.target = value_str, - attr_keys::VALUE => w.text = if value_str.is_empty() { None } else { Some(value_str) }, - _ => (), - }, - #[cfg(feature = "callout")] - Node::Callout(c) => match attr { - attr_keys::KIND => c.kind = value_str, - attr_keys::FOLD => c.fold = value_str.chars().next().filter(|c| matches!(c, '+' | '-')), - attr_keys::TITLE => c.title = if value_str.is_empty() { None } else { Some(value_str) }, - _ => (), - }, - #[cfg(feature = "embed")] - Node::Embed(e) => match attr { - attr_keys::URL => e.target = value_str, - attr_keys::VALUE => e.display = if value_str.is_empty() { None } else { Some(value_str) }, - _ => (), - }, - } - } - - /// Returns true if any `Text` descendant contains `[[`, covering both - /// embed (`![[`) and wikilink (`[[`) patterns. - #[cfg(any(feature = "embed", feature = "wikilink"))] - fn values_contain_links(values: &[Node]) -> bool { - values.iter().any(|n| match n { - Node::Text(t) => t.value.contains("[["), - Node::Heading(h) => Self::values_contain_links(&h.values), - Node::Blockquote(b) => Self::values_contain_links(&b.values), - #[cfg(feature = "callout")] - Node::Callout(c) => Self::values_contain_links(&c.values), - Node::List(l) => Self::values_contain_links(&l.values), - Node::Strong(s) => Self::values_contain_links(&s.values), - Node::Emphasis(e) => Self::values_contain_links(&e.values), - Node::Delete(d) => Self::values_contain_links(&d.values), - Node::TableCell(tc) => Self::values_contain_links(&tc.values), - Node::TableRow(tr) => Self::values_contain_links(&tr.values), - Node::Footnote(f) => Self::values_contain_links(&f.values), - _ => false, - }) - } - - /// Shared tree walker used by all expand functions. Recurses into container - /// nodes and delegates `Text` node processing to `parse_into`. - #[cfg(any(feature = "embed", feature = "wikilink"))] - fn expand_with(nodes: Vec<Node>, parse_into: fn(&str, Option<Position>, &mut Vec<Node>)) -> Vec<Node> { - let mut result = Vec::with_capacity(nodes.len()); - for node in nodes { - Self::expand_with_into(node, &mut result, parse_into); - } - result - } - - #[cfg(any(feature = "embed", feature = "wikilink"))] - fn expand_with_into(node: Node, out: &mut Vec<Node>, parse_into: fn(&str, Option<Position>, &mut Vec<Node>)) { - match node { - Node::Text(Text { ref value, .. }) if !value.contains("[[") => out.push(node), - Node::Text(Text { value, position }) => parse_into(&value, position, out), - Node::Heading(mut h) => { - if Self::values_contain_links(&h.values) { - h.values = Self::expand_with(h.values, parse_into); - } - out.push(Node::Heading(h)); - } - Node::Blockquote(mut b) => { - if Self::values_contain_links(&b.values) { - b.values = Self::expand_with(b.values, parse_into); - } - out.push(Node::Blockquote(b)); - } - #[cfg(feature = "callout")] - Node::Callout(mut c) => { - if Self::values_contain_links(&c.values) { - c.values = Self::expand_with(c.values, parse_into); - } - out.push(Node::Callout(c)); - } - Node::List(mut l) => { - if Self::values_contain_links(&l.values) { - l.values = Self::expand_with(l.values, parse_into); - } - out.push(Node::List(l)); - } - Node::Strong(mut s) => { - if Self::values_contain_links(&s.values) { - s.values = Self::expand_with(s.values, parse_into); - } - out.push(Node::Strong(s)); - } - Node::Emphasis(mut e) => { - if Self::values_contain_links(&e.values) { - e.values = Self::expand_with(e.values, parse_into); - } - out.push(Node::Emphasis(e)); - } - Node::Delete(mut d) => { - if Self::values_contain_links(&d.values) { - d.values = Self::expand_with(d.values, parse_into); - } - out.push(Node::Delete(d)); - } - Node::TableCell(mut tc) => { - if Self::values_contain_links(&tc.values) { - tc.values = Self::expand_with(tc.values, parse_into); - } - out.push(Node::TableCell(tc)); - } - Node::TableRow(mut tr) => { - if Self::values_contain_links(&tr.values) { - tr.values = Self::expand_with(tr.values, parse_into); - } - out.push(Node::TableRow(tr)); - } - Node::Footnote(mut f) => { - if Self::values_contain_links(&f.values) { - f.values = Self::expand_with(f.values, parse_into); - } - out.push(Node::Footnote(f)); - } - other => out.push(other), - } - } - - /// Recursively scans a node list and converts `![[target]]` / `![[target|display]]` - /// patterns inside `Text` nodes into `Embed` nodes. - #[cfg(feature = "embed")] - pub fn expand_embeds(nodes: Vec<Node>) -> Vec<Node> { - Self::expand_with(nodes, Self::parse_embeds_into) - } - - /// Splits a text string on `![[...]]` patterns, pushing a mix of `Text` - /// and `Embed` nodes into `out`. - #[cfg(feature = "embed")] - fn parse_embeds_into(text: &str, position: Option<Position>, out: &mut Vec<Node>) { - let mut last_end = 0; - let mut search_from = 0; - let mut found_any = false; - - while let Some(bang_rel) = text[search_from..].find("![[") { - let bang = search_from + bang_rel; - let inner_start = bang + 3; - - let Some(close_rel) = text[inner_start..].find("]]") else { - search_from = bang + 1; - continue; - }; - let close = inner_start + close_rel; - let content = &text[inner_start..close]; - - if content.contains('[') || content.contains(']') { - search_from = bang + 1; - continue; - } - - found_any = true; - if last_end < bang { - out.push(Node::Text(Text { - value: text[last_end..bang].to_string(), - position: position.clone(), - })); - } - let (target, display) = if let Some(pipe) = content.find('|') { - ( - content[..pipe].trim().to_string(), - Some(content[pipe + 1..].trim().to_string()), - ) - } else { - (content.trim().to_string(), None) - }; - out.push(Node::Embed(Embed { - target, - display, - position: position.clone(), - })); - last_end = close + 2; - search_from = close + 2; - } - - if !found_any { - out.push(Node::Text(Text { - value: text.to_string(), - position, - })); - return; - } - - if last_end < text.len() { - out.push(Node::Text(Text { - value: text[last_end..].to_string(), - position, - })); - } - } - - /// Recursively scans a node list and converts `[[target]]` / `[[target|text]]` - /// patterns inside `Text` nodes into `WikiLink` nodes. - #[cfg(feature = "wikilink")] - pub fn expand_wikilinks(nodes: Vec<Node>) -> Vec<Node> { - Self::expand_with(nodes, Self::parse_wikilinks_into) - } - - /// Splits a text string on `[[...]]` patterns, pushing a mix of `Text` - /// and `WikiLink` nodes into `out`. - #[cfg(feature = "wikilink")] - fn parse_wikilinks_into(text: &str, position: Option<Position>, out: &mut Vec<Node>) { - let mut last_end = 0; - let mut search_from = 0; - let mut found_any = false; - - while let Some(open_rel) = text[search_from..].find("[[") { - let open = search_from + open_rel; - let inner_start = open + 2; - - let Some(close_rel) = text[inner_start..].find("]]") else { - break; - }; - let close = inner_start + close_rel; - let content = &text[inner_start..close]; - - if content.contains('[') || content.contains(']') { - // nested brackets — skip past this '[' and retry - search_from = open + 1; - continue; - } - - found_any = true; - if last_end < open { - out.push(Node::Text(Text { - value: text[last_end..open].to_string(), - position: position.clone(), - })); - } - let (target, text_part) = if let Some(pipe) = content.find('|') { - (content[..pipe].trim(), Some(content[pipe + 1..].trim())) - } else { - (content.trim(), None) - }; - out.push(Node::WikiLink(WikiLink { - target: target.to_string(), - text: text_part.map(|t| t.to_string()), - position: position.clone(), - })); - last_end = close + 2; - search_from = close + 2; - } - - if !found_any { - out.push(Node::Text(Text { - value: text.to_string(), - position, - })); - return; - } - - if last_end < text.len() { - out.push(Node::Text(Text { - value: text[last_end..].to_string(), - position, - })); - } - } - - /// Splits a text string on `[[...]]` patterns, returning a mix of `Text` - /// and `WikiLink` nodes. Used only in tests. - #[cfg(all(feature = "wikilink", test))] - fn parse_wikilinks_in_text(text: &str, position: Option<Position>) -> Vec<Node> { - let mut result = Vec::new(); - Self::parse_wikilinks_into(text, position, &mut result); - result - } - - /// Recursively scans a node list and converts both `![[target]]` (embed) and - /// `[[target]]` (wikilink) patterns in a single tree traversal. More efficient - /// than running `expand_embeds` followed by `expand_wikilinks` separately, which - /// traverses the tree and scans each `Text` node twice. - #[cfg(all(feature = "embed", feature = "wikilink"))] - pub fn expand_inline_links(nodes: Vec<Node>) -> Vec<Node> { - Self::expand_with(nodes, Self::parse_inline_links_into) - } - - /// Splits a text string on `![[...]]` and `[[...]]` patterns in a single pass, - /// pushing a mix of `Text`, `Embed`, and `WikiLink` nodes into `out`. - /// A `[[` immediately preceded by `!` is treated as an embed; otherwise as a wikilink. - #[cfg(all(feature = "embed", feature = "wikilink"))] - fn parse_inline_links_into(text: &str, position: Option<Position>, out: &mut Vec<Node>) { - let mut last_end = 0; - let mut search_from = 0; - let mut found_any = false; - - while let Some(open_rel) = text[search_from..].find("[[") { - let open = search_from + open_rel; - let inner_start = open + 2; - - let Some(close_rel) = text[inner_start..].find("]]") else { - break; - }; - let close = inner_start + close_rel; - let content = &text[inner_start..close]; - - if content.contains('[') || content.contains(']') { - search_from = open + 1; - continue; - } - - let is_embed = text.as_bytes()[..open].last() == Some(&b'!'); - let token_start = if is_embed { open - 1 } else { open }; - - found_any = true; - if last_end < token_start { - out.push(Node::Text(Text { - value: text[last_end..token_start].to_string(), - position: position.clone(), - })); - } - - let (target, opt_part) = if let Some(pipe) = content.find('|') { - (content[..pipe].trim(), Some(content[pipe + 1..].trim())) - } else { - (content.trim(), None) - }; - - if is_embed { - out.push(Node::Embed(Embed { - target: target.to_string(), - display: opt_part.map(|s| s.to_string()), - position: position.clone(), - })); - } else { - out.push(Node::WikiLink(WikiLink { - target: target.to_string(), - text: opt_part.map(|s| s.to_string()), - position: position.clone(), - })); - } - - last_end = close + 2; - search_from = close + 2; - } - - if !found_any { - out.push(Node::Text(Text { - value: text.to_string(), - position, - })); - return; - } - - if last_end < text.len() { - out.push(Node::Text(Text { - value: text[last_end..].to_string(), - position, - })); - } - } - - fn mdx_attribute_content_to_string(attr: &MdxAttributeContent) -> SmolStr { - match attr { - MdxAttributeContent::Expression(value) => format!("{{{}}}", indent_expression(value)).into(), - MdxAttributeContent::Property(property) => match &property.value { - Some(value) => match value { - MdxAttributeValue::Expression(value) => { - format!("{}={{{}}}", property.name, indent_expression(value)).into() - } - MdxAttributeValue::Literal(literal) => { - format!("{}={}", property.name, quote_attribute_value(literal)).into() - } - }, - None => property.name.clone(), - }, - } - } -} - -/// Whether `node` or a node in it is an expression of more than one line. -fn has_multiline_expression(node: &Node) -> bool { - match node { - Node::MdxFlowExpression(expression) => expression.value.contains('\n'), - Node::MdxTextExpression(expression) => expression.value.contains('\n'), - Node::MdxJsxTextElement(MdxJsxTextElement { attributes, .. }) - | Node::MdxJsxFlowElement(MdxJsxFlowElement { attributes, .. }) - if attributes.iter().any(|attribute| match attribute { - MdxAttributeContent::Expression(value) => value.contains('\n'), - MdxAttributeContent::Property(MdxJsxAttribute { - value: Some(MdxAttributeValue::Expression(value)), - .. - }) => value.contains('\n'), - _ => false, - }) => - { - true - } - other => other.children().iter().any(has_multiline_expression), - } -} - -/// Quotes the value of a JSX attribute with the quote it does not contain. Character references that the -/// value holds as text are written with their `&` as a reference. -fn quote_attribute_value(value: &str) -> String { - let mut escaped = String::with_capacity(value.len()); - for (index, c) in value.char_indices() { - if c == '&' && starts_reference(&value[index + 1..]) { - escaped.push_str("&amp;"); - } else { - escaped.push(c); - } - } - if !escaped.contains('"') { - format!("\"{escaped}\"") - } else if !escaped.contains('\'') { - format!("'{escaped}'") - } else { - format!("\"{}\"", escaped.replace('"', "&quot;")) - } -} - -/// The content of an expression as it is written: the lines after the first are indented, as the -/// indentation of up to two characters is removed from them when they are read. -fn indent_expression(value: &str) -> String { - value - .split('\n') - .enumerate() - .map(|(index, line)| { - if index == 0 || line.is_empty() || line == "\r" { - line.to_string() - } else { - format!(" {line}") - } - }) - .collect::<Vec<_>>() - .join("\n") -} - -pub(crate) fn values_to_string(values: &[Node], options: &RenderOptions) -> String { - render_values(values, options, &ColorTheme::PLAIN) -} - -/// Indents every line, the first included, by `width` spaces, except empty ones. -pub(crate) fn indent_lines(content: &str, width: usize) -> String { - let pad = " ".repeat(width); - content - .split('\n') - .map(|line| { - if line.is_empty() { - line.to_string() - } else { - format!("{pad}{line}") - } - }) - .collect::<Vec<_>>() - .join("\n") -} - -/// Indents every line after the first by `width` spaces, except empty ones. -fn indent_continuation(content: &str, width: usize) -> String { - let Some((first, rest)) = content.split_once('\n') else { - return content.to_string(); - }; - let pad = " ".repeat(width); - let mut result = String::with_capacity(content.len() + width * 2); - result.push_str(first); - for line in rest.split('\n') { - result.push('\n'); - if !line.is_empty() { - result.push_str(&pad); - result.push_str(line); - } - } - result -} - -fn render_emphasis(values: &[Node], delim: &str, options: &RenderOptions, theme: &ColorTheme<'_>) -> String { - let (es, ee) = &theme.emphasis; - format!("{es}{delim}{}{delim}{ee}", render_values(values, options, theme)) -} - -/// Renders the content of a table cell. A pipe that is not escaped yet, as in code, would end the cell. -pub(crate) fn render_cell_values(values: &[Node], options: &RenderOptions, theme: &ColorTheme<'_>) -> String { - let rendered = render_values(values, options, theme); - if !rendered.contains('|') { - return rendered; - } - let mut result = String::with_capacity(rendered.len() + 4); - let mut escaped = false; - for c in rendered.chars() { - if c == '|' && !escaped { - result.push('\\'); - } - escaped = c == '\\' && !escaped; - result.push(c); - } - result -} - -fn ends_with_word_char(node: &Node) -> bool { - matches!(node, Node::Text(Text { value, .. }) if value.ends_with(|c: char| c.is_alphanumeric())) -} - -fn starts_with_word_char(node: &Node) -> bool { - matches!(node, Node::Text(Text { value, .. }) if value.starts_with(|c: char| c.is_alphanumeric())) -} - -/// Renders `value`, escaping a `!` that ends its text when `next` starts a link, which would make an image. -pub(crate) fn render_before( - value: &Node, - next: Option<&Node>, - options: &RenderOptions, - theme: &ColorTheme<'_>, -) -> String { - let mut rendered = value.render_with_theme(options, theme); - let starts_link = match next { - Some(Node::Link(_) | Node::LinkRef(_)) => true, - #[cfg(feature = "wikilink")] - Some(Node::WikiLink(_)) => true, - _ => false, - }; - if starts_link && matches!(value, Node::Text(Text { position: Some(_), .. })) && rendered.ends_with('!') { - rendered.insert(rendered.len() - 1, '\\'); - } - rendered -} - -pub(crate) fn render_values(values: &[Node], options: &RenderOptions, theme: &ColorTheme<'_>) -> String { - let mut pre_position: Option<Position> = None; - values - .iter() - .enumerate() - .map(|(index, value)| { - let rendered = || render_before(value, values.get(index + 1), options, theme); - if let Some(pos) = value.position() { - let new_line_count = pre_position - .as_ref() - .map(|p: &Position| pos.start.line.saturating_sub(p.end.line)) - .unwrap_or_default(); - - let space = if new_line_count > 0 - && pre_position - .as_ref() - .map(|p| pos.start.line > p.end.line) - .unwrap_or_default() - { - " ".repeat(pos.start.column.saturating_sub(1)) - } else { - "".to_string() - }; - - pre_position = Some(pos); - - if space.is_empty() { - format!("{}{}", "\n".repeat(new_line_count), rendered()) - } else { - format!( - "{}{}", - "\n".repeat(new_line_count), - rendered().lines().map(|line| format!("{}{}", space, line)).join("\n") - ) - } - } else { - pre_position = None; - rendered() - } - }) - .collect::<String>() -} - -/// Like `render_values` but without column-offset–based indentation. -/// -/// Inside blockquotes and callouts, node positions include the `> ` prefix offset, -/// so applying column-based spacing causes double-indentation. This variant preserves -/// blank lines between values (via newline counting) but ignores the column position. -pub(crate) fn render_values_block(values: &[Node], options: &RenderOptions, theme: &ColorTheme<'_>) -> String { - let is_table_part = |node: &Node| matches!(node, Node::TableCell(_) | Node::TableAlign(_)); - let mut result = String::new(); - let mut pre_position: Option<Position> = None; - let mut index = 0; - - while index < values.len() { - // The cells of a table are laid out together, as at the top level. - let (rendered, position, next) = if is_table_part(&values[index]) { - let end = values[index..] - .iter() - .position(|node| !is_table_part(node)) - .map_or(values.len(), |offset| index + offset); - let run = &values[index..end]; - let table = crate::Markdown { - nodes: run.to_vec(), - options: options.clone(), - }; - let position = match (run[0].position(), run[run.len() - 1].position()) { - (Some(first), Some(last)) => Some(Position { - start: first.start, - end: last.end, - }), - _ => None, - }; - ( - table.render_with_theme(theme).trim_end_matches('\n').to_string(), - position, - end, - ) - } else { - let value = &values[index]; - ( - render_before(value, values.get(index + 1), options, theme), - value.position(), - index + 1, - ) - }; - index = next; - - if let Some(pos) = position { - let new_line_count = pre_position - .as_ref() - .map(|p: &Position| pos.start.line.saturating_sub(p.end.line)) - .unwrap_or_default(); - pre_position = Some(pos); - result.push_str(&"\n".repeat(new_line_count)); - } else { - pre_position = None; - } - result.push_str(&rendered); - } - result -} +} fn values_to_value(values: &[Node]) -> String { values.iter().map(|value| value.value()).collect::<String>() } - -/// Renders a link/image destination, auto-upgrading to `<...>` (escaping `\ < >`) -/// when the bare form would be unsafe (empty, or contains whitespace/control/`< >`); -/// otherwise bare, escaping `\ ( )` since unescaped parens must balance. -fn render_link_destination(url: &str, style: &UrlSurroundStyle) -> String { - let needs_angle = url.is_empty() - || url - .chars() - .any(|c| c.is_whitespace() || c.is_control() || c == '<' || c == '>'); - - if matches!(style, UrlSurroundStyle::Angle) || needs_angle { - let mut result = String::with_capacity(url.len() + 2); - result.push('<'); - for c in url.chars() { - if matches!(c, '\\' | '<' | '>') { - result.push('\\'); - } - result.push(c); - } - result.push('>'); - result - } else { - let mut result = String::with_capacity(url.len()); - for c in url.chars() { - if matches!(c, '\\' | '(' | ')') { - result.push('\\'); - } - result.push(c); - } - result - } -} - -/// Renders a link/image title, escaping `\` and the style's delimiter(s) so the -/// title's own quote/paren characters can't end it early. -fn render_link_title(title: &str, style: &TitleSurroundStyle) -> String { - let (open, close, delims): (char, char, &[char]) = match style { - TitleSurroundStyle::Double => ('"', '"', &['"']), - TitleSurroundStyle::Single => ('\'', '\'', &['\'']), - TitleSurroundStyle::Paren => ('(', ')', &['(', ')']), - }; - let mut result = String::with_capacity(title.len() + 2); - result.push(open); - for c in title.chars() { - if c == '\\' || delims.contains(&c) { - result.push('\\'); - } - result.push(c); - } - result.push(close); - result -} - -/// Escapes `\ [ ]` in reference labels/idents and image alt text: plain strings -/// (not walked as inline nodes) that still sit inside `[...]` syntax. -/// True if `s` needs escaping beyond `escape_label`'s narrow `\ [ ]` set — unsafe -/// for a reference's implicit shortcut form, since the definition's label won't match. -fn needs_broad_escaping(s: &str) -> bool { - s.bytes().any(|b| { - matches!( - b, - b'`' | b'*' | b'_' | b'|' | b'~' | b'$' | b'#' | b'-' | b'+' | b'>' | b'=' | b'.' | b')' | b'!' - ) - }) -} - -/// Shifts every line's leading indentation by `delta` spaces (never below zero), -/// leaving each line's own content untouched. -pub(crate) fn reindent_all_lines(content: &str, delta: isize) -> String { - if delta == 0 { - return content.to_string(); - } - content - .split('\n') - .map(|line| { - if line.is_empty() { - line.to_string() - } else { - let leading = line.bytes().take_while(|&b| b == b' ').count(); - let new_leading = (leading as isize + delta).max(0) as usize; - format!("{}{}", " ".repeat(new_leading), &line[leading..]) - } - }) - .collect::<Vec<_>>() - .join("\n") -} - -/// The width of a list item's own marker and trailing space (not including ancestor indentation), -/// e.g. 2 for `"- "`, 3 for `"1. "`, 4 for `"10. "`. A checkbox is not part of it, as the content of -/// the item starts at the checkbox, so children are indented to the marker alone. -pub(crate) fn list_own_prefix_width(ordered: bool, index: usize, start: Option<u32>) -> usize { - let marker_len = if ordered { - (start.unwrap_or(1) as usize + index).to_string().chars().count() + 1 - } else { - 1 - }; - marker_len + 1 -} - -/// The length of the longest run of `ch` in `s`. -fn longest_run(s: &str, ch: char) -> usize { - s.split('\n') - .flat_map(|line| line.split(|c| c != ch).map(str::len)) - .max() - .unwrap_or(0) -} - -/// Longest fence-char run in `value` + 1 (min 3); switches to tilde when `info` -/// has a backtick, since a backtick-fenced info string can't contain one. -fn code_fence(value: &str, info: &str) -> String { - if info.contains('`') { - let len = (longest_run(value, '~') + 1).max(3); - "~".repeat(len) - } else { - let len = (longest_run(value, '`') + 1).max(3); - "`".repeat(len) - } -} - -/// The label to write in a definition: the identifier as it was written when the label is only that -/// without its escapes and character references, which would not read back as the same identifier. -fn definition_label(label: Option<&str>, ident: &str) -> String { - match label { - Some(label) if crate::parser::normalize(label) == ident => escape_label(label), - Some(label) if crate::parser::normalize(label) == crate::parser::normalize(&crate::parser::unescape(ident)) => { - ident.to_string() - } - Some(label) => escape_label(label), - None => ident.to_string(), - } -} - -/// The label to write in the second brackets of a full reference. The label as written is kept when it -/// still resolves to `ident`. -fn reference_label(label: Option<&str>, ident: &str) -> String { - match label { - Some(label) if crate::parser::normalize(label) == ident => escape_label(label), - _ => ident.to_string(), - } -} - -/// Picks an inline code span delimiter one backtick longer than the longest -/// backtick run in `value`, so no run inside it can be mistaken for the closing. -fn code_span_fence(value: &str) -> String { - let mut max_run = 0; - let mut cur = 0; - for c in value.chars() { - if c == '`' { - cur += 1; - max_run = max_run.max(cur); - } else { - cur = 0; - } - } - "`".repeat(max_run + 1) -} - -/// The shortest run of dollar signs, from one, that is not the length of a run in `value`, so that no -/// run inside a math span can be mistaken for the closing. -fn math_span_fence(value: &str) -> String { - let mut lengths = std::collections::HashSet::new(); - let mut current = 0; - for c in value.chars() { - if c == '$' { - current += 1; - } else if current > 0 { - lengths.insert(current); - current = 0; - } - } - if current > 0 { - lengths.insert(current); - } - "$".repeat((1..).find(|length| !lengths.contains(length)).unwrap_or(1)) -} - -/// Escapes the brackets that are not escaped yet, for an identifier that keeps the escapes it was written with. -fn escape_brackets(s: &str) -> String { - let mut result = String::with_capacity(s.len()); - let mut chars = s.chars().peekable(); - while let Some(c) = chars.next() { - match c { - '\\' if chars.peek().is_some_and(char::is_ascii_punctuation) => { - result.push(c); - result.extend(chars.next()); - } - '[' | ']' => { - result.push('\\'); - result.push(c); - } - _ => result.push(c), - } - } - result -} - -fn escape_label(s: &str) -> String { - if !s.bytes().any(|b| matches!(b, b'\\' | b'[' | b']')) { - return s.to_string(); - } - let mut result = String::with_capacity(s.len()); - for c in s.chars() { - if matches!(c, '\\' | '[' | ']') { - result.push('\\'); - } - result.push(c); - } - result -} - -/// The text of an autolink for a link whose only content is its own destination, when writing that -/// text as a link label would need escapes. A URL in a label is read as it is, so `\_` would stay. -fn autolink_target(url: &str, values: &[Node]) -> Option<String> { - let [Node::Text(Text { value, .. })] = values else { - return None; - }; - let target = if value == url { - url - } else if url.strip_prefix("mailto:") == Some(value.as_str()) && crate::parser::is_autolink_email(value) { - value.as_str() - } else { - return None; - }; - let plain = !target - .chars() - .any(|c| c.is_whitespace() || c.is_control() || matches!(c, '<' | '>')); - let scheme = target.split_once(':').is_some_and(|(scheme, _)| { - (2..=32).contains(&scheme.len()) - && scheme.starts_with(|c: char| c.is_ascii_alphabetic()) - && scheme - .chars() - .all(|c| c.is_ascii_alphanumeric() || matches!(c, '+' | '.' | '-')) - }); - let email = value != url; - (plain && (scheme || email) && escape_text(target.to_string()) != target).then(|| target.to_string()) -} - -/// Whether `rest`, the text after an `&`, reads as the rest of a character reference. -fn starts_reference(rest: &str) -> bool { - let name = match rest.strip_prefix('#') { - Some(number) => match number.strip_prefix(['x', 'X']) { - Some(hex) => hex - .split_once(';') - .map(|(digits, _)| (digits, char::is_ascii_hexdigit as fn(&char) -> bool)), - None => number - .split_once(';') - .map(|(digits, _)| (digits, char::is_ascii_digit as fn(&char) -> bool)), - }, - None => rest - .split_once(';') - .map(|(name, _)| (name, char::is_ascii_alphanumeric as fn(&char) -> bool)), - }; - name.is_some_and(|(name, is_valid)| !name.is_empty() && name.chars().all(|c| is_valid(&c))) -} - -fn is_email_local(c: char) -> bool { - c.is_ascii_alphanumeric() || matches!(c, '.' | '-' | '_' | '+') -} - -/// Whether `rest`, the text after an `@`, starts with a domain that makes an email address of the text -/// before it. -fn is_email_domain(rest: &str) -> bool { - let domain = rest - .split(|c: char| !(c.is_ascii_alphanumeric() || matches!(c, '.' | '-' | '_'))) - .next() - .unwrap_or_default(); - domain - .split_once('.') - .is_some_and(|(label, after)| !label.is_empty() && after.starts_with(|c: char| c.is_ascii_alphanumeric())) -} - -/// Re-escapes markdown-significant characters so plain text can't be reinterpreted as -/// syntax on re-parse. `# - + > =` only unsafe at line start, `. )` only after leading digits; -/// a leading `\t` and runs of 2+ `\n` (only possible via decoded entities) use `&#N;` instead. -fn escape_text(value: String) -> String { - escape_text_with(value, false) -} - -/// Whether `text` starts with the `import ` or `export ` that makes a block ESM in MDX. -fn starts_esm(text: &str) -> bool { - text.starts_with("import ") || text.starts_with("export ") -} - -/// `escape_text`, and for MDX also the braces and the `import` or `export` that starts a line. -fn escape_text_with(value: String, mdx: bool) -> String { - // Skip the copy when nothing needs escaping; byte scan avoids UTF-8 decoding. - let needs_escaping = value.bytes().any(|b| { - matches!( - b, - b'\\' - | b'`' - | b'*' - | b'_' - | b'[' - | b']' - | b'|' - | b'~' - | b'$' - | b'#' - | b'-' - | b'+' - | b'>' - | b'<' - | b'=' - | b'.' - | b')' - | b'\n' - | b'\t' - | b'&' - | b'@' - ) - }); - let mdx_special = mdx && (value.contains('{') || value.contains("import ") || value.contains("export ")); - if !needs_escaping && !mdx_special { - return value; - } - - let mut result = String::with_capacity(value.len()); - let mut at_line_start = true; - // True while in an unbroken digit run since line start (ordered-list marker number). - let mut leading_digits = false; - let mut chars = value - .char_indices() - .map(|(index, c)| (c, &value[index + c.len_utf8()..])) - .peekable(); - let mut previous = None; - - while let Some((c, rest)) = chars.next() { - match c { - '\n' if chars.peek().is_some_and(|(next, _)| *next == '\n') => { - result.push_str("&#10;"); - at_line_start = true; - leading_digits = false; - continue; - } - '\t' if at_line_start => { - result.push_str("&#9;"); - at_line_start = false; - leading_digits = false; - continue; - } - '\\' | '`' | '*' | '_' | '[' | ']' | '|' | '~' | '$' | '<' => result.push('\\'), - '{' if mdx => result.push('\\'), - // The first letter as a character reference keeps the line from starting an ESM block. - 'i' | 'e' if mdx && at_line_start && starts_esm(&value[value.len() - rest.len() - 1..]) => { - result.push_str(if c == 'i' { "&#105;" } else { "&#101;" }); - at_line_start = false; - leading_digits = false; - previous = Some(c); - continue; - } - '#' | '-' | '+' | '>' | '=' if at_line_start => result.push('\\'), - '.' | ')' if leading_digits => result.push('\\'), - '&' if starts_reference(rest) => result.push('\\'), - '@' if previous.is_some_and(is_email_local) && is_email_domain(rest) => result.push('\\'), - _ => {} - } - result.push(c); - previous = Some(c); - - let is_digit = c.is_ascii_digit(); - leading_digits = if at_line_start { - is_digit - } else { - leading_digits && is_digit - }; - at_line_start = c == '\n'; - } - - result -} - -#[cfg(test)] -mod tests { - use super::*; - use proptest::prelude::*; - use rstest::rstest; - - #[rstest] - #[case::dash('-', Some(ListMarker::Dash))] - #[case::plus('+', Some(ListMarker::Plus))] - #[case::star('*', Some(ListMarker::Star))] - #[case::period('.', Some(ListMarker::Period))] - #[case::paren(')', Some(ListMarker::Paren))] - #[case::underscore('_', None)] - #[case::digit('1', None)] - fn list_marker_from_char(#[case] char: char, #[case] expected: Option<ListMarker>) { - assert_eq!(ListMarker::from_char(char), expected); - assert_eq!(expected.and_then(ListMarker::as_char), expected.map(|_| char)); - } - - #[rstest] - #[case::dash(ListMarker::Dash, Some(ListStyle::Dash))] - #[case::plus(ListMarker::Plus, Some(ListStyle::Plus))] - #[case::star(ListMarker::Star, Some(ListStyle::Star))] - #[case::period(ListMarker::Period, None)] - #[case::paren(ListMarker::Paren, None)] - #[case::continuation(ListMarker::Continuation, None)] - fn list_marker_style(#[case] marker: ListMarker, #[case] expected: Option<ListStyle>) { - assert_eq!(marker.style(), expected); - } - - #[rstest] - #[case::dash(ListStyle::Dash, ListMarker::Dash)] - #[case::plus(ListStyle::Plus, ListMarker::Plus)] - #[case::star(ListStyle::Star, ListMarker::Star)] - fn list_marker_from_style(#[case] style: ListStyle, #[case] expected: ListMarker) { - assert_eq!(ListMarker::from(style), expected); - } - - #[rstest] - #[case::star('*', Some(HorizontalRuleMarker::Star))] - #[case::dash('-', Some(HorizontalRuleMarker::Dash))] - #[case::underscore('_', Some(HorizontalRuleMarker::Underscore))] - #[case::plus('+', None)] - fn horizontal_rule_marker_from_char(#[case] char: char, #[case] expected: Option<HorizontalRuleMarker>) { - assert_eq!(HorizontalRuleMarker::from_char(char), expected); - assert_eq!(expected.map(HorizontalRuleMarker::as_char), expected.map(|_| char)); - } - - #[rstest] - #[case::bullet_dash(false, Some(ListMarker::Dash), "- a")] - #[case::bullet_plus(false, Some(ListMarker::Plus), "+ a")] - #[case::bullet_star(false, Some(ListMarker::Star), "* a")] - #[case::bullet_unknown(false, None, "- a")] - #[case::bullet_with_an_ordered_marker(false, Some(ListMarker::Paren), "- a")] - #[case::ordered_period(true, Some(ListMarker::Period), "1. a")] - #[case::ordered_paren(true, Some(ListMarker::Paren), "1) a")] - #[case::ordered_unknown(true, None, "1. a")] - #[case::ordered_with_a_bullet_marker(true, Some(ListMarker::Star), "1. a")] - fn list_renders_its_marker(#[case] ordered: bool, #[case] marker: Option<ListMarker>, #[case] expected: &str) { - let node = Node::List(List { - index: 0, - level: 0, - checked: None, - values: vec![Node::Text(Text { - value: "a".into(), - position: None, - })], - ordered, - start: None, - spread: false, - marker, - position: None, - }); - assert_eq!( - node.render_with_theme(&RenderOptions::default(), &ColorTheme::PLAIN), - expected - ); - } - - #[rstest] - #[case::star(Some(HorizontalRuleMarker::Star), "***")] - #[case::dash(Some(HorizontalRuleMarker::Dash), "---")] - #[case::underscore(Some(HorizontalRuleMarker::Underscore), "___")] - #[case::unknown(None, "***")] - fn horizontal_rule_renders_its_marker(#[case] marker: Option<HorizontalRuleMarker>, #[case] expected: &str) { - let node = Node::HorizontalRule(HorizontalRule { marker, position: None }); - assert_eq!( - node.render_with_theme(&RenderOptions::default(), &ColorTheme::PLAIN), - expected - ); - } - - /// The JSON of the markers is the character they are written with, as it was when they were chars. - #[cfg(feature = "json")] - #[rstest] - #[case::dash(ListMarker::Dash, "\"-\"")] - #[case::plus(ListMarker::Plus, "\"+\"")] - #[case::star(ListMarker::Star, "\"*\"")] - #[case::period(ListMarker::Period, "\".\"")] - #[case::paren(ListMarker::Paren, "\")\"")] - fn list_marker_json(#[case] marker: ListMarker, #[case] expected: &str) { - assert_eq!(serde_json::to_string(&marker).unwrap(), expected); - assert_eq!(serde_json::from_str::<ListMarker>(expected).unwrap(), marker); - } - - #[cfg(feature = "json")] - #[rstest] - #[case::star(HorizontalRuleMarker::Star, "\"*\"")] - #[case::dash(HorizontalRuleMarker::Dash, "\"-\"")] - #[case::underscore(HorizontalRuleMarker::Underscore, "\"_\"")] - fn horizontal_rule_marker_json(#[case] marker: HorizontalRuleMarker, #[case] expected: &str) { - assert_eq!(serde_json::to_string(&marker).unwrap(), expected); - assert_eq!(serde_json::from_str::<HorizontalRuleMarker>(expected).unwrap(), marker); - } - - #[cfg(feature = "json")] - #[test] - fn list_json_keeps_the_marker_as_a_string() { - let json = r#"{"type":"list","values":[],"index":0,"level":0,"ordered":false,"checked":null,"spread":false,"marker":"+"}"#; - let list = serde_json::from_str::<List>(json).unwrap(); - assert_eq!(list.marker, Some(ListMarker::Plus)); - assert!(serde_json::to_string(&list).unwrap().contains(r#""marker":"+""#)); - } - - #[test] - fn map_values_into_transforms_owned_fragments() { - let node = Node::Fragment(Fragment { - values: vec![Node::Text(Text { - value: "before".to_string(), - position: None, - })], - }); - - let mapped = node - .map_values_into(&mut |node| -> Result<Node, std::io::Error> { - Ok(match node { - Node::Text(text) => Node::Text(Text { - value: text.value.to_uppercase(), - position: text.position.clone(), - }), - node => node.clone(), - }) - }) - .unwrap(); - - assert_eq!(mapped.to_string(), "BEFORE"); - } - - #[test] - fn map_values_into_owned_moves_nodes_through_the_callback() { - let node = Node::Fragment(Fragment { - values: vec![Node::Text(Text { - value: "before".to_string(), - position: None, - })], - }); - - let mapped = node - .map_values_into_owned(&mut |node| -> Result<Node, std::io::Error> { - Ok(match node { - Node::Text(mut text) => { - text.value.make_ascii_uppercase(); - Node::Text(text) - } - node => node, - }) - }) - .unwrap(); - - assert_eq!(mapped.to_string(), "BEFORE"); - } - - #[rstest] - #[case::text(Node::Text(Text{value: "".to_string(), position: None}), - "test".to_string(), - Node::Text(Text{value: "test".to_string(), position: None }))] - #[case::blockquote(Node::Blockquote(Blockquote{values: vec!["test".to_string().into()], position: None }), - "test".to_string(), - Node::Blockquote(Blockquote{values: vec!["test".to_string().into()], position: None }))] - #[case::delete(Node::Delete(Delete{values: vec!["test".to_string().into()], position: None }), - "test".to_string(), - Node::Delete(Delete{values: vec!["test".to_string().into()], position: None }))] - #[case::emphasis(Node::Emphasis(Emphasis{values: vec!["test".to_string().into()], position: None }), - "test".to_string(), - Node::Emphasis(Emphasis{values: vec!["test".to_string().into()], position: None }))] - #[case::strong(Node::Strong(Strong{values: vec!["test".to_string().into()], position: None }), - "test".to_string(), - Node::Strong(Strong{values: vec!["test".to_string().into()], position: None }))] - #[case::heading(Node::Heading(Heading {depth: 1, values: vec!["test".to_string().into()], position: None }), - "test".to_string(), - Node::Heading(Heading{depth: 1, values: vec!["test".to_string().into()], position: None }))] - #[case::link(Node::Link(Link {url: Url::new("test".to_string()), values: Vec::new(), title: None, position: None }), - "test".to_string(), - Node::Link(Link{url: Url::new("test".to_string()), values: Vec::new(), title: None, position: None }))] - #[case::image(Node::Image(Image {alt: "test".to_string(), url: "test".to_string(), title: None, position: None }), - "test".to_string(), - Node::Image(Image{alt: "test".to_string(), url: "test".to_string(), title: None, position: None }))] - #[case::code(Node::Code(Code {value: "test".to_string(), lang: None, fence: true, meta: None, position: None }), - "test".to_string(), - Node::Code(Code{value: "test".to_string(), lang: None, fence: true, meta: None, position: None }))] - #[case::footnote_ref(Node::FootnoteRef(FootnoteRef {ident: "test".to_string(), label: None, position: None }), - "test".to_string(), - Node::FootnoteRef(FootnoteRef{ident: "test".to_string(), label: Some("test".to_string()), position: None }))] - #[case::footnote(Node::Footnote(Footnote {ident: "test".to_string(), values: Vec::new(), position: None }), - "test".to_string(), - Node::Footnote(Footnote{ident: "test".to_string(), values: Vec::new(), position: None }))] - #[case::list(Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: vec!["test".to_string().into()], position: None }), - "test".to_string(), - Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: vec!["test".to_string().into()], position: None }))] - #[case::list(Node::List(List{ marker: None,start: None, spread: false, index: 1, level: 1, checked: Some(true), ordered: false, values: vec!["test".to_string().into()], position: None }), - "test".to_string(), - Node::List(List{ marker: None,start: None, spread: false, index: 1, level: 1, checked: Some(true), ordered: false, values: vec!["test".to_string().into()], position: None }))] - #[case::list(Node::List(List{ marker: None,start: None, spread: false, index: 2, level: 2, checked: Some(false), ordered: false, values: vec!["test".to_string().into()], position: None }), - "test".to_string(), - Node::List(List{ marker: None,start: None, spread: false, index: 2, level: 2, checked: Some(false), ordered: false, values: vec!["test".to_string().into()], position: None }))] - #[case::code_inline(Node::CodeInline(CodeInline{ value: "t".into(), position: None }), - "test".to_string(), - Node::CodeInline(CodeInline{ value: "test".into(), position: None }))] - #[case::math_inline(Node::MathInline(MathInline{ value: "t".into(), position: None }), - "test".to_string(), - Node::MathInline(MathInline{ value: "test".into(), position: None }))] - #[case::toml(Node::Toml(Toml{ value: "t".to_string(), position: None }), - "test".to_string(), - Node::Toml(Toml{ value: "test".to_string(), position: None }))] - #[case::yaml(Node::Yaml(Yaml{ value: "t".to_string(), position: None }), - "test".to_string(), - Node::Yaml(Yaml{ value: "test".to_string(), position: None }))] - #[case::html(Node::Html(Html{ value: "t".to_string(), position: None }), - "test".to_string(), - Node::Html(Html{ value: "test".to_string(), position: None }))] - #[case::table_row(Node::TableRow(TableRow{ values: vec![ - Node::TableCell(TableCell{values: vec!["test1".to_string().into()], row:0, column:1, position: None}), - Node::TableCell(TableCell{values: vec!["test2".to_string().into()], row:0, column:2, position: None}) - ] - , position: None }), - "test3,test4".to_string(), - Node::TableRow(TableRow{ values: vec![ - Node::TableCell(TableCell{values: vec!["test3".to_string().into()], row:0, column:1, position: None}), - Node::TableCell(TableCell{values: vec!["test4".to_string().into()], row:0, column:2, position: None}) - ] - , position: None }))] - #[case::table_cell(Node::TableCell(TableCell{values: vec!["test1".to_string().into()], row:0, column:1, position: None}), - "test2".to_string(), - Node::TableCell(TableCell{values: vec!["test2".to_string().into()], row:0, column:1, position: None}),)] - #[case::link_ref(Node::LinkRef(LinkRef{ident: "test2".to_string(), values: vec![attr_keys::VALUE.to_string().into()], label: Some("test2".to_string()), position: None}), - "test2".to_string(), - Node::LinkRef(LinkRef{ident: "test2".to_string(), values: vec![attr_keys::VALUE.to_string().into()], label: Some("test2".to_string()), position: None}),)] - #[case::image_ref(Node::ImageRef(ImageRef{alt: attr_keys::ALT.to_string(), ident: "test1".to_string(), label: None, position: None}), - "test2".to_string(), - Node::ImageRef(ImageRef{alt: attr_keys::ALT.to_string(), ident: "test2".to_string(), label: Some("test2".to_string()), position: None}),)] - #[case::definition(Node::Definition(Definition{ url: Url::new(attr_keys::URL.to_string()), title: None, ident: "test1".to_string(), label: None, position: None}), - "test2".to_string(), - Node::Definition(Definition{url: Url::new("test2".to_string()), title: None, ident: "test1".to_string(), label: None, position: None}),)] - #[case::break_(Node::Break(Break{ position: None}), - "test".to_string(), - Node::Break(Break{position: None}))] - #[case::horizontal_rule(Node::HorizontalRule(HorizontalRule{ marker: None, position: None}), - "test".to_string(), - Node::HorizontalRule(HorizontalRule{ marker: None,position: None}))] - #[case::mdx_flow_expression(Node::MdxFlowExpression(MdxFlowExpression{value: "test".into(), position: None}), - "updated".to_string(), - Node::MdxFlowExpression(MdxFlowExpression{value: "updated".into(), position: None}))] - #[case::mdx_text_expression(Node::MdxTextExpression(MdxTextExpression{value: "test".into(), position: None}), - "updated".to_string(), - Node::MdxTextExpression(MdxTextExpression{value: "updated".into(), position: None}))] - #[case::mdx_js_esm(Node::MdxJsEsm(MdxJsEsm{value: "test".into(), position: None}), - "updated".to_string(), - Node::MdxJsEsm(MdxJsEsm{value: "updated".into(), position: None}))] - #[case(Node::MdxJsxFlowElement(MdxJsxFlowElement{ - name: Some("div".to_string()), - attributes: Vec::new(), - children: vec!["test".to_string().into()], - position: None - }), - "updated".to_string(), - Node::MdxJsxFlowElement(MdxJsxFlowElement{ - name: Some("div".to_string()), - attributes: Vec::new(), - children: vec!["updated".to_string().into()], - position: None - }))] - #[case::mdx_jsx_text_element(Node::MdxJsxTextElement(MdxJsxTextElement{ - name: Some("span".into()), - attributes: Vec::new(), - children: vec!["test".to_string().into()], - position: None - }), - "updated".to_string(), - Node::MdxJsxTextElement(MdxJsxTextElement{ - name: Some("span".into()), - attributes: Vec::new(), - children: vec!["updated".to_string().into()], - position: None - }))] - #[case(Node::Math(Math{ value: "x^2".to_string(), position: None }), - "test".to_string(), - Node::Math(Math{ value: "test".to_string(), position: None }))] - fn test_with_value(#[case] node: Node, #[case] input: String, #[case] expected: Node) { - assert_eq!(node.clone().with_value(input.as_str()), expected); - assert_eq!(node.into_with_value(input.as_str()), expected); - } - - #[rstest] - #[case(Node::Blockquote(Blockquote{values: vec![ - Node::Text(Text{value: "first".to_string(), position: None}), - Node::Text(Text{value: "second".to_string(), position: None}) - ], position: None}), - "new", - 0, - Node::Blockquote(Blockquote{values: vec![ - Node::Text(Text{value: "new".to_string(), position: None}), - Node::Text(Text{value: "second".to_string(), position: None}) - ], position: None}))] - #[case(Node::Blockquote(Blockquote{values: vec![ - Node::Text(Text{value: "first".to_string(), position: None}), - Node::Text(Text{value: "second".to_string(), position: None}) - ], position: None}), - "new", - 1, - Node::Blockquote(Blockquote{values: vec![ - Node::Text(Text{value: "first".to_string(), position: None}), - Node::Text(Text{value: "new".to_string(), position: None}) - ], position: None}))] - #[case(Node::Delete(Delete{values: vec![ - Node::Text(Text{value: "first".to_string(), position: None}), - Node::Text(Text{value: "second".to_string(), position: None}) - ], position: None}), - "new", - 0, - Node::Delete(Delete{values: vec![ - Node::Text(Text{value: "new".to_string(), position: None}), - Node::Text(Text{value: "second".to_string(), position: None}) - ], position: None}))] - #[case(Node::Emphasis(Emphasis{values: vec![ - Node::Text(Text{value: "first".to_string(), position: None}), - Node::Text(Text{value: "second".to_string(), position: None}) - ], position: None}), - "new", - 1, - Node::Emphasis(Emphasis{values: vec![ - Node::Text(Text{value: "first".to_string(), position: None}), - Node::Text(Text{value: "new".to_string(), position: None}) - ], position: None}))] - #[case(Node::Strong(Strong{values: vec![ - Node::Text(Text{value: "first".to_string(), position: None}), - Node::Text(Text{value: "second".to_string(), position: None}) - ], position: None}), - "new", - 0, - Node::Strong(Strong{values: vec![ - Node::Text(Text{value: "new".to_string(), position: None}), - Node::Text(Text{value: "second".to_string(), position: None}) - ], position: None}))] - #[case(Node::Heading(Heading{depth: 1, values: vec![ - Node::Text(Text{value: "first".to_string(), position: None}), - Node::Text(Text{value: "second".to_string(), position: None}) - ], position: None}), - "new", - 1, - Node::Heading(Heading{depth: 1, values: vec![ - Node::Text(Text{value: "first".to_string(), position: None}), - Node::Text(Text{value: "new".to_string(), position: None}) - ], position: None}))] - #[case(Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: vec![ - Node::Text(Text{value: "first".to_string(), position: None}), - Node::Text(Text{value: "second".to_string(), position: None}) - ], position: None}), - "new", - 0, - Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: vec![ - Node::Text(Text{value: "new".to_string(), position: None}), - Node::Text(Text{value: "second".to_string(), position: None}) - ], position: None}))] - #[case(Node::TableCell(TableCell{column: 0, row: 0, values: vec![ - Node::Text(Text{value: "first".to_string(), position: None}), - Node::Text(Text{value: "second".to_string(), position: None}) - ], position: None}), - "new", - 1, - Node::TableCell(TableCell{column: 0, row: 0, values: vec![ - Node::Text(Text{value: "first".to_string(), position: None}), - Node::Text(Text{value: "new".to_string(), position: None}) - ], position: None}))] - #[case(Node::Text(Text{value: "plain text".to_string(), position: None}), - "new", - 0, - Node::Text(Text{value: "plain text".to_string(), position: None}))] - #[case(Node::Code(Code{value: "code".to_string(), lang: None, fence: true, meta: None, position: None}), - "new", - 0, - Node::Code(Code{value: "code".to_string(), lang: None, fence: true, meta: None, position: None}))] - #[case(Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 1, checked: Some(true), ordered: false, values: vec![ - Node::Text(Text{value: "first".to_string(), position: None}) - ], position: None}), - "new", - 0, - Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 1, checked: Some(true), ordered: false, values: vec![ - Node::Text(Text{value: "new".to_string(), position: None}) - ], position: None}))] - #[case(Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 1, checked: None, ordered: false, values: vec![ - Node::Text(Text{value: "first".to_string(), position: None}) - ], position: None}), - "new", - 2, - Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 1, checked: None, ordered: false, values: vec![ - Node::Text(Text{value: "first".to_string(), position: None}) - ], position: None}))] - #[case::link_ref(Node::LinkRef(LinkRef{ident: "id".to_string(), values: vec![ - Node::Text(Text{value: "first".to_string(), position: None}), - Node::Text(Text{value: "second".to_string(), position: None}) - ], label: None, position: None}), "new", 0, Node::LinkRef(LinkRef{ident: "id".to_string(), values: vec![ - Node::Text(Text{value: "new".to_string(), position: None}), - Node::Text(Text{value: "second".to_string(), position: None}) - ], label: None, position: None}))] - #[case(Node::MdxJsxFlowElement(MdxJsxFlowElement{ - name: Some("div".to_string()), - attributes: Vec::new(), - children: vec![ - Node::Text(Text{value: "first".to_string(), position: None}), - Node::Text(Text{value: "second".to_string(), position: None}) - ], - position: None - }), - "new", - 0, - Node::MdxJsxFlowElement(MdxJsxFlowElement{ - name: Some("div".to_string()), - attributes: Vec::new(), - children: vec![ - Node::Text(Text{value: "new".to_string(), position: None}), - Node::Text(Text{value: "second".to_string(), position: None}) - ], - position: None - }))] - #[case(Node::MdxJsxFlowElement(MdxJsxFlowElement{ - name: Some("div".to_string()), - attributes: Vec::new(), - children: vec![ - Node::Text(Text{value: "first".to_string(), position: None}), - Node::Text(Text{value: "second".to_string(), position: None}) - ], - position: None - }), - "new", - 1, - Node::MdxJsxFlowElement(MdxJsxFlowElement{ - name: Some("div".to_string()), - attributes: Vec::new(), - children: vec![ - Node::Text(Text{value: "first".to_string(), position: None}), - Node::Text(Text{value: "new".to_string(), position: None}) - ], - position: None - }))] - #[case(Node::MdxJsxTextElement(MdxJsxTextElement{ - name: Some("span".into()), - attributes: Vec::new(), - children: vec![ - Node::Text(Text{value: "first".to_string(), position: None}), - Node::Text(Text{value: "second".to_string(), position: None}) - ], - position: None - }), - "new", - 0, - Node::MdxJsxTextElement(MdxJsxTextElement{ - name: Some("span".into()), - attributes: Vec::new(), - children: vec![ - Node::Text(Text{value: "new".to_string(), position: None}), - Node::Text(Text{value: "second".to_string(), position: None}) - ], - position: None - }))] - #[case(Node::MdxJsxTextElement(MdxJsxTextElement{ - name: Some("span".into()), - attributes: Vec::new(), - children: vec![ - Node::Text(Text{value: "first".to_string(), position: None}), - Node::Text(Text{value: "second".to_string(), position: None}) - ], - position: None - }), - "new", - 1, - Node::MdxJsxTextElement(MdxJsxTextElement{ - name: Some("span".into()), - attributes: Vec::new(), - children: vec![ - Node::Text(Text{value: "first".to_string(), position: None}), - Node::Text(Text{value: "new".to_string(), position: None}) - ], - position: None - }))] - fn test_with_children_value(#[case] node: Node, #[case] value: &str, #[case] index: usize, #[case] expected: Node) { - assert_eq!(node.clone().with_children_value(value, index), expected); - assert_eq!(node.into_with_children_value(value, index), expected); - } - - #[rstest] - #[case(Node::Text(Text{value: "test".to_string(), position: None }), - "test".to_string())] - #[case(Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 2, checked: None, ordered: false, values: vec!["test".to_string().into()], position: None}), - " - test".to_string())] - fn test_display(#[case] node: Node, #[case] expected: String) { - assert_eq!(node.to_string_with(&RenderOptions::default()), expected); - } - - #[rstest] - #[case(Node::Text(Text{value: "test".to_string(), position: None}), true)] - #[case(Node::CodeInline(CodeInline{value: "test".into(), position: None}), false)] - #[case(Node::MathInline(MathInline{value: "test".into(), position: None}), false)] - fn test_is_text(#[case] node: Node, #[case] expected: bool) { - assert_eq!(node.is_text(), expected); - } - - #[rstest] - #[case(Node::CodeInline(CodeInline{value: "test".into(), position: None}), true)] - #[case(Node::Text(Text{value: "test".to_string(), position: None}), false)] - fn test_is_inline_code(#[case] node: Node, #[case] expected: bool) { - assert_eq!(node.is_inline_code(), expected); - } - - #[rstest] - #[case(Node::MathInline(MathInline{value: "test".into(), position: None}), true)] - #[case(Node::Text(Text{value: "test".to_string(), position: None}), false)] - fn test_is_inline_math(#[case] node: Node, #[case] expected: bool) { - assert_eq!(node.is_inline_math(), expected); - } - - #[rstest] - #[case(Node::Strong(Strong{values: vec!["test".to_string().into()], position: None}), true)] - #[case(Node::Text(Text{value: "test".to_string(), position: None}), false)] - fn test_is_strong(#[case] node: Node, #[case] expected: bool) { - assert_eq!(node.is_strong(), expected); - } - - #[rstest] - #[case(Node::Delete(Delete{values: vec!["test".to_string().into()], position: None}), true)] - #[case(Node::Text(Text{value: "test".to_string(), position: None}), false)] - fn test_is_delete(#[case] node: Node, #[case] expected: bool) { - assert_eq!(node.is_delete(), expected); - } - - #[rstest] - #[case(Node::Link(Link{url: Url::new("test".to_string()), values: Vec::new(), title: None, position: None}), true)] - #[case(Node::Text(Text{value: "test".to_string(), position: None}), false)] - fn test_is_link(#[case] node: Node, #[case] expected: bool) { - assert_eq!(node.is_link(), expected); - } - - #[rstest] - #[case(Node::LinkRef(LinkRef{ident: "test".to_string(), values: Vec::new(), label: None, position: None}), true)] - #[case(Node::Text(Text{value: "test".to_string(), position: None}), false)] - fn test_is_link_ref(#[case] node: Node, #[case] expected: bool) { - assert_eq!(node.is_link_ref(), expected); - } - - #[rstest] - #[case(Node::Image(Image{alt: attr_keys::ALT.to_string(), url: "test".to_string(), title: None, position: None}), true)] - #[case(Node::Text(Text{value: "test".to_string(), position: None}), false)] - fn test_is_image(#[case] node: Node, #[case] expected: bool) { - assert_eq!(node.is_image(), expected); - } - - #[rstest] - #[case(Node::ImageRef(ImageRef{alt: attr_keys::ALT.to_string(), ident: "test".to_string(), label: None, position: None}), true)] - #[case(Node::Text(Text{value: "test".to_string(), position: None}), false)] - fn test_is_image_ref(#[case] node: Node, #[case] expected: bool) { - assert_eq!(node.is_image_ref(), expected); - } - - #[rstest] - #[case(Node::Code(Code{value: "code".to_string(), lang: Some("rust".to_string()), fence: true, meta: None, position: None}), true, Some("rust".into()))] - #[case(Node::Code(Code{value: "code".to_string(), lang: Some("rust".to_string()), fence: true, meta: None, position: None}), false, Some("python".into()))] - #[case(Node::Code(Code{value: "code".to_string(), lang: None, fence: true, meta: None, position: None}), true, None)] - #[case(Node::Code(Code{value: "code".to_string(), lang: None, fence: false, meta: None, position: None}), true, None)] - #[case(Node::Text(Text{value: "test".to_string(), position: None}), false, None)] - fn test_is_code(#[case] node: Node, #[case] expected: bool, #[case] lang: Option<SmolStr>) { - assert_eq!(node.is_code(lang), expected); - } - - #[rstest] - #[case(Node::Heading(Heading{depth: 1, values: vec!["test".to_string().into()], position: None}), true, Some(1))] - #[case(Node::Heading(Heading{depth: 2, values: vec!["test".to_string().into()], position: None}), false, Some(1))] - #[case(Node::Heading(Heading{depth: 1, values: vec!["test".to_string().into()], position: None}), true, None)] - #[case(Node::Text(Text{value: "test".to_string(), position: None}), false, None)] - fn test_is_heading(#[case] node: Node, #[case] expected: bool, #[case] depth: Option<u8>) { - assert_eq!(node.is_heading(depth), expected); - } - - #[rstest] - #[case(Node::HorizontalRule(HorizontalRule{ marker: None,position: None}), true)] - #[case(Node::Text(Text{value: "test".to_string(), position: None}), false)] - fn test_is_horizontal_rule(#[case] node: Node, #[case] expected: bool) { - assert_eq!(node.is_horizontal_rule(), expected); - } - - #[rstest] - #[case(Node::Blockquote(Blockquote{values: vec!["test".to_string().into()], position: None}), true)] - #[case(Node::Text(Text{value: "test".to_string(), position: None}), false)] - fn test_is_blockquote(#[case] node: Node, #[case] expected: bool) { - assert_eq!(node.is_blockquote(), expected); - } - - #[cfg(feature = "wikilink")] - #[rstest] - #[case(Node::WikiLink(WikiLink{target: "target".to_string(), text: None, position: None}), true)] - #[case(Node::WikiLink(WikiLink{target: "Three laws of motion".to_string(), text: Some("Newton".to_string()), position: None}), true)] - #[case(Node::Link(Link{url: Url::new("https://example.com".to_string()), values: Vec::new(), title: None, position: None}), false)] - #[case(Node::Link(Link{url: Url::new("relative.md".to_string()), values: Vec::new(), title: None, position: None}), false)] - #[case(Node::Text(Text{value: "test".to_string(), position: None}), false)] - #[case(Node::Text(Text{value: "[[target]]".to_string(), position: None}), false)] - fn test_is_wikilink(#[case] node: Node, #[case] expected: bool) { - assert_eq!(node.is_wikilink(), expected); - } - - #[cfg(feature = "wikilink")] - #[rstest] - #[case(Node::WikiLink(WikiLink{target: "target".to_string(), text: None, position: None}), true)] - #[case(Node::WikiLink(WikiLink{target: "Three laws of motion".to_string(), text: Some("Newton".to_string()), position: None}), true)] - #[case(Node::Link(Link{url: Url::new("https://example.com".to_string()), values: Vec::new(), title: None, position: None}), true)] - #[case(Node::Link(Link{url: Url::new("relative.md".to_string()), values: Vec::new(), title: None, position: None}), true)] - #[case(Node::Text(Text{value: "test".to_string(), position: None}), false)] - fn test_is_link_includes_wikilink(#[case] node: Node, #[case] expected: bool) { - assert_eq!(node.is_link(), expected); - } - - #[cfg(feature = "wikilink")] - #[rstest] - // no wikilinks — returns original text node unchanged - #[case("plain text", vec![Node::Text(Text{value: "plain text".to_string(), position: None})])] - // only a wikilink - #[case("[[target]]", vec![Node::WikiLink(WikiLink{target: "target".to_string(), text: None, position: None})])] - // wikilink with display text - #[case("[[target|display]]", vec![Node::WikiLink(WikiLink{target: "target".to_string(), text: Some("display".to_string()), position: None})])] - // wikilink with spaces in target - #[case("[[Three laws of motion]]", vec![Node::WikiLink(WikiLink{target: "Three laws of motion".to_string(), text: None, position: None})])] - // wikilink with .md extension - #[case("[[target.md]]", vec![Node::WikiLink(WikiLink{target: "target.md".to_string(), text: None, position: None})])] - // wikilink at start - #[case("[[target]] after", vec![ - Node::WikiLink(WikiLink{target: "target".to_string(), text: None, position: None}), - Node::Text(Text{value: " after".to_string(), position: None}), - ])] - // wikilink at end - #[case("before [[target]]", vec![ - Node::Text(Text{value: "before ".to_string(), position: None}), - Node::WikiLink(WikiLink{target: "target".to_string(), text: None, position: None}), - ])] - // wikilink in middle - #[case("before [[target]] after", vec![ - Node::Text(Text{value: "before ".to_string(), position: None}), - Node::WikiLink(WikiLink{target: "target".to_string(), text: None, position: None}), - Node::Text(Text{value: " after".to_string(), position: None}), - ])] - // multiple wikilinks - #[case("[[a]] and [[b]]", vec![ - Node::WikiLink(WikiLink{target: "a".to_string(), text: None, position: None}), - Node::Text(Text{value: " and ".to_string(), position: None}), - Node::WikiLink(WikiLink{target: "b".to_string(), text: None, position: None}), - ])] - // unclosed [[ treated as plain text - #[case("[[unclosed", vec![Node::Text(Text{value: "[[unclosed".to_string(), position: None})])] - // nested brackets invalid — treated as plain text - #[case("[[in[ner]]]", vec![Node::Text(Text{value: "[[in[ner]]]".to_string(), position: None})])] - // multibyte characters in surrounding text - #[case("日本語 [[ターゲット]] テキスト", vec![ - Node::Text(Text{value: "日本語 ".to_string(), position: None}), - Node::WikiLink(WikiLink{target: "ターゲット".to_string(), text: None, position: None}), - Node::Text(Text{value: " テキスト".to_string(), position: None}), - ])] - // multibyte characters inside wikilink target and display text - #[case("[[ページ|表示名]]", vec![ - Node::WikiLink(WikiLink{target: "ページ".to_string(), text: Some("表示名".to_string()), position: None}), - ])] - fn test_parse_wikilinks_in_text(#[case] input: &str, #[case] expected: Vec<Node>) { - let result = Node::parse_wikilinks_in_text(input, None); - assert_eq!(result, expected); - } - - #[cfg(feature = "wikilink")] - #[rstest] - #[case(Node::WikiLink(WikiLink{target: "target".to_string(), text: None, position: None}), "url", Some(AttrValue::String("target".to_string())))] - #[case(Node::WikiLink(WikiLink{target: "target".to_string(), text: None, position: None}), "value", Some(AttrValue::String("target".to_string())))] - #[case(Node::WikiLink(WikiLink{target: "target".to_string(), text: Some("display".to_string()), position: None}), "url", Some(AttrValue::String("target".to_string())))] - #[case(Node::WikiLink(WikiLink{target: "target".to_string(), text: Some("display".to_string()), position: None}), "value", Some(AttrValue::String("display".to_string())))] - #[case(Node::WikiLink(WikiLink{target: "target".to_string(), text: None, position: None}), "title", None)] - fn test_wikilink_attr(#[case] node: Node, #[case] attr: &str, #[case] expected: Option<AttrValue>) { - assert_eq!(node.attr(attr), expected); - } - - #[cfg(feature = "wikilink")] - #[rstest] - // no display text: with_value sets target - #[case( - Node::WikiLink(WikiLink{target: "target".to_string(), text: None, position: None}), - "new-target", - Node::WikiLink(WikiLink{target: "new-target".to_string(), text: None, position: None}) - )] - // with display text: with_value sets text, not target - #[case( - Node::WikiLink(WikiLink{target: "page".to_string(), text: Some("Display Text".to_string()), position: None}), - "DISPLAY TEXT", - Node::WikiLink(WikiLink{target: "page".to_string(), text: Some("DISPLAY TEXT".to_string()), position: None}) - )] - fn test_wikilink_with_value(#[case] node: Node, #[case] value: &str, #[case] expected: Node) { - assert_eq!(node.clone().with_value(value), expected); - assert_eq!(node.into_with_value(value), expected); - } - - #[cfg(feature = "wikilink")] - #[rstest] - // footnote with no wikilinks — values unchanged - #[case( - Node::Footnote(Footnote{ident: "1".to_string(), values: vec![Node::Text(Text{value: "plain text".to_string(), position: None})], position: None}), - vec![Node::Footnote(Footnote{ident: "1".to_string(), values: vec![Node::Text(Text{value: "plain text".to_string(), position: None})], position: None})] - )] - // footnote with a wikilink in text — expanded to WikiLink node - #[case( - Node::Footnote(Footnote{ident: "1".to_string(), values: vec![Node::Text(Text{value: "[[target]]".to_string(), position: None})], position: None}), - vec![Node::Footnote(Footnote{ident: "1".to_string(), values: vec![Node::WikiLink(WikiLink{target: "target".to_string(), text: None, position: None})], position: None})] - )] - // footnote with wikilink and display text - #[case( - Node::Footnote(Footnote{ident: "2".to_string(), values: vec![Node::Text(Text{value: "[[target|display]]".to_string(), position: None})], position: None}), - vec![Node::Footnote(Footnote{ident: "2".to_string(), values: vec![Node::WikiLink(WikiLink{target: "target".to_string(), text: Some("display".to_string()), position: None})], position: None})] - )] - // footnote with mixed text and wikilink - #[case( - Node::Footnote(Footnote{ident: "3".to_string(), values: vec![Node::Text(Text{value: "see [[note]]".to_string(), position: None})], position: None}), - vec![Node::Footnote(Footnote{ident: "3".to_string(), values: vec![ - Node::Text(Text{value: "see ".to_string(), position: None}), - Node::WikiLink(WikiLink{target: "note".to_string(), text: None, position: None}), - ], position: None})] - )] - fn test_expand_wikilinks_footnote(#[case] input: Node, #[case] expected: Vec<Node>) { - let result = Node::expand_wikilinks(vec![input]); - assert_eq!(result, expected); - } - - #[cfg(all(feature = "embed", feature = "wikilink"))] - #[rstest] - // embed only - #[case("![[note.md]]", vec![ - Node::Embed(Embed{target: "note.md".to_string(), display: None, position: None}), - ])] - // wikilink only - #[case("[[target]]", vec![ - Node::WikiLink(WikiLink{target: "target".to_string(), text: None, position: None}), - ])] - // embed and wikilink in same text - #[case("![[embed]] and [[link]]", vec![ - Node::Embed(Embed{target: "embed".to_string(), display: None, position: None}), - Node::Text(Text{value: " and ".to_string(), position: None}), - Node::WikiLink(WikiLink{target: "link".to_string(), text: None, position: None}), - ])] - // wikilink before embed - #[case("[[link]] and ![[embed]]", vec![ - Node::WikiLink(WikiLink{target: "link".to_string(), text: None, position: None}), - Node::Text(Text{value: " and ".to_string(), position: None}), - Node::Embed(Embed{target: "embed".to_string(), display: None, position: None}), - ])] - // embed with display hint - #[case("![[image.png|400]]", vec![ - Node::Embed(Embed{target: "image.png".to_string(), display: Some("400".to_string()), position: None}), - ])] - // wikilink with display text - #[case("[[page|display]]", vec![ - Node::WikiLink(WikiLink{target: "page".to_string(), text: Some("display".to_string()), position: None}), - ])] - // surrounding text - #[case("before ![[note]] after [[link]] end", vec![ - Node::Text(Text{value: "before ".to_string(), position: None}), - Node::Embed(Embed{target: "note".to_string(), display: None, position: None}), - Node::Text(Text{value: " after ".to_string(), position: None}), - Node::WikiLink(WikiLink{target: "link".to_string(), text: None, position: None}), - Node::Text(Text{value: " end".to_string(), position: None}), - ])] - // plain text with no links - #[case("just plain text", vec![ - Node::Text(Text{value: "just plain text".to_string(), position: None}), - ])] - // multibyte characters - #[case("日本語 ![[ファイル]] と [[ページ]]", vec![ - Node::Text(Text{value: "日本語 ".to_string(), position: None}), - Node::Embed(Embed{target: "ファイル".to_string(), display: None, position: None}), - Node::Text(Text{value: " と ".to_string(), position: None}), - Node::WikiLink(WikiLink{target: "ページ".to_string(), text: None, position: None}), - ])] - fn test_parse_inline_links_into(#[case] input: &str, #[case] expected: Vec<Node>) { - let mut result = Vec::new(); - Node::parse_inline_links_into(input, None, &mut result); - assert_eq!(result, expected); - } - - #[rstest] - #[case(Node::Html(Html{value: "<div>test</div>".to_string(), position: None}), true)] - #[case(Node::Text(Text{value: "test".to_string(), position: None}), false)] - fn test_is_html(#[case] node: Node, #[case] expected: bool) { - assert_eq!(node.is_html(), expected); - } - - #[rstest] - #[case(Node::node_values( - &Node::Strong(Strong{values: vec!["test".to_string().into()], position: None})), - vec!["test".to_string().into()])] - #[case(Node::node_values( - &Node::Text(Text{value: "test".to_string(), position: None})), - vec!["test".to_string().into()])] - #[case(Node::node_values( - &Node::Blockquote(Blockquote{values: vec!["test".to_string().into()], position: None})), - vec!["test".to_string().into()])] - #[case(Node::node_values( - &Node::Delete(Delete{values: vec!["test".to_string().into()], position: None})), - vec!["test".to_string().into()])] - #[case(Node::node_values( - &Node::Emphasis(Emphasis{values: vec!["test".to_string().into()], position: None})), - vec!["test".to_string().into()])] - #[case(Node::node_values( - &Node::Heading(Heading{depth: 1, values: vec!["test".to_string().into()], position: None})), - vec!["test".to_string().into()])] - #[case(Node::node_values( - &Node::List(List{ marker: None,values: vec!["test".to_string().into()], ordered: false, level: 1, checked: Some(false), index: 0, start: None, spread: false, position: None})), - vec!["test".to_string().into()])] - fn test_node_value(#[case] actual: Vec<Node>, #[case] expected: Vec<Node>) { - assert_eq!(actual, expected); - } - - #[rstest] - #[case(Node::Footnote(Footnote{ident: "test".to_string(), values: Vec::new(), position: None}), true)] - #[case(Node::Text(Text{value: "test".to_string(), position: None}), false)] - fn test_is_footnote(#[case] node: Node, #[case] expected: bool) { - assert_eq!(node.is_footnote(), expected); - } - - #[rstest] - #[case(Node::FootnoteRef(FootnoteRef{ident: "test".to_string(), label: None, position: None}), true)] - #[case(Node::Text(Text{value: "test".to_string(), position: None}), false)] - fn test_is_footnote_ref(#[case] node: Node, #[case] expected: bool) { - assert_eq!(node.is_footnote_ref(), expected); - } - - #[rstest] - #[case(Node::Math(Math{value: "x^2".to_string(), position: None}), true)] - #[case(Node::Text(Text{value: "test".to_string(), position: None}), false)] - fn test_is_math(#[case] node: Node, #[case] expected: bool) { - assert_eq!(node.is_math(), expected); - } - - #[rstest] - #[case(Node::Break(Break{position: None}), true)] - #[case(Node::Text(Text{value: "test".to_string(), position: None}), false)] - fn test_is_break(#[case] node: Node, #[case] expected: bool) { - assert_eq!(node.is_break(), expected); - } - - #[rstest] - #[case(Node::Yaml(Yaml{value: "key: value".to_string(), position: None}), true)] - #[case(Node::Text(Text{value: "test".to_string(), position: None}), false)] - fn test_is_yaml(#[case] node: Node, #[case] expected: bool) { - assert_eq!(node.is_yaml(), expected); - } - - #[rstest] - #[case(Node::Toml(Toml{value: "key = \"value\"".to_string(), position: None}), true)] - #[case(Node::Text(Text{value: "test".to_string(), position: None}), false)] - fn test_is_toml(#[case] node: Node, #[case] expected: bool) { - assert_eq!(node.is_toml(), expected); - } - - #[rstest] - #[case(Node::Definition(Definition{ident: attr_keys::IDENT.to_string(), url: Url::new(attr_keys::URL.to_string()), title: None, label: None, position: None}), true)] - #[case(Node::Text(Text{value: "test".to_string(), position: None}), false)] - fn test_is_definition(#[case] node: Node, #[case] expected: bool) { - assert_eq!(node.is_definition(), expected); - } - - #[rstest] - #[case(Node::Emphasis(Emphasis{values: vec!["test".to_string().into()], position: None}), true)] - #[case(Node::Text(Text{value: "test".to_string(), position: None}), false)] - fn test_is_emphasis(#[case] node: Node, #[case] expected: bool) { - assert_eq!(node.is_emphasis(), expected); - } - - #[rstest] - #[case(Node::MdxFlowExpression(MdxFlowExpression{value: "test".into(), position: None}), true)] - #[case(Node::Text(Text{value: "test".to_string(), position: None}), false)] - fn test_is_mdx_flow_expression(#[case] node: Node, #[case] expected: bool) { - assert_eq!(node.is_mdx_flow_expression(), expected); - } - - #[rstest] - #[case(Node::MdxTextExpression(MdxTextExpression{value: "test".into(), position: None}), true)] - #[case(Node::Text(Text{value: "test".to_string(), position: None}), false)] - fn test_is_mdx_text_expression(#[case] node: Node, #[case] expected: bool) { - assert_eq!(node.is_mdx_text_expression(), expected); - } - - #[rstest] - #[case(Node::MdxJsxFlowElement(MdxJsxFlowElement{name: None, attributes: Vec::new(), children: Vec::new(), position: None}), true)] - #[case(Node::Text(Text{value: "test".to_string(), position: None}), false)] - fn test_is_mdx_jsx_flow_element(#[case] node: Node, #[case] expected: bool) { - assert_eq!(node.is_mdx_jsx_flow_element(), expected); - } - - #[rstest] - #[case(Node::MdxJsxTextElement(MdxJsxTextElement{name: None, attributes: Vec::new(), children: Vec::new(), position: None}), true)] - #[case(Node::Text(Text{value: "test".to_string(), position: None}), false)] - fn test_is_mdx_jsx_text_element(#[case] node: Node, #[case] expected: bool) { - assert_eq!(node.is_mdx_jsx_text_element(), expected); - } - - #[rstest] - #[case(Node::MdxJsEsm(MdxJsEsm{value: "test".into(), position: None}), true)] - #[case(Node::Text(Text{value: "test".to_string(), position: None}), false)] - fn test_is_msx_js_esm(#[case] node: Node, #[case] expected: bool) { - assert_eq!(node.is_mdx_js_esm(), expected); - } - - #[rstest] - #[case::text(Node::Text(Text{value: "test".to_string(), position: None }), RenderOptions::default(), "test")] - #[case::list(Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 2, checked: None, ordered: false, values: vec!["test".to_string().into()], position: None}), RenderOptions::default(), " - test")] - #[case::list(Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 1, checked: None, ordered: false, values: vec!["test".to_string().into()], position: None}), RenderOptions { list_style: Some(ListStyle::Plus), ..Default::default() }, " + test")] - #[case::list(Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 1, checked: Some(true), ordered: false, values: vec!["test".to_string().into()], position: None}), RenderOptions { list_style: Some(ListStyle::Star), ..Default::default() }, " * [x] test")] - #[case::list(Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 1, checked: Some(false), ordered: false, values: vec!["test".to_string().into()], position: None}), RenderOptions::default(), " - [ ] test")] - #[case::list(Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 1, checked: None, ordered: true, values: vec!["test".to_string().into()], position: None}), RenderOptions::default(), " 1. test")] - #[case::list(Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 1, checked: Some(false), ordered: true, values: vec!["test".to_string().into()], position: None}), RenderOptions::default(), " 1. [ ] test")] - #[case::table_row(Node::TableRow(TableRow{values: vec![Node::TableCell(TableCell{column: 0, row: 0, values: vec!["test".to_string().into()], position: None})], position: None}), RenderOptions::default(), "|test|")] - #[case::table_row(Node::TableRow(TableRow{values: vec![Node::TableCell(TableCell{column: 0, row: 0, values: vec!["test".to_string().into()], position: None})], position: None}), RenderOptions::default(), "|test|")] - #[case::table_cell(Node::TableCell(TableCell{column: 0, row: 0, values: vec!["test".to_string().into()], position: None}), RenderOptions::default(), "test")] - #[case::table_cell(Node::TableCell(TableCell{column: 0, row: 0, values: vec!["test".to_string().into()], position: None}), RenderOptions::default(), "test")] - #[case::table_align(Node::TableAlign(TableAlign{align: vec![TableAlignKind::Left, TableAlignKind::Right, TableAlignKind::Center, TableAlignKind::None], position: None}), RenderOptions::default(), "|:---|---:|:---:|---|")] - #[case::block_quote(Node::Blockquote(Blockquote{values: vec!["test".to_string().into()], position: None}), RenderOptions::default(), "> test")] - #[case::block_quote(Node::Blockquote(Blockquote{values: vec!["test\ntest2".to_string().into()], position: None}), RenderOptions::default(), "> test\n> test2")] - #[case::code(Node::Code(Code{value: "code".to_string(), lang: Some("rust".to_string()), fence: true, meta: None, position: None}), RenderOptions::default(), "```rust\ncode\n```")] - #[case::code(Node::Code(Code{value: "code".to_string(), lang: None, fence: true, meta: None, position: None}), RenderOptions::default(), "```\ncode\n```")] - #[case::code(Node::Code(Code{value: "code".to_string(), lang: None, fence: false, meta: None, position: None}), RenderOptions::default(), " code")] - #[case::code(Node::Code(Code{value: "code".to_string(), lang: Some("rust".to_string()), fence: true, meta: Some("meta".to_string()), position: None}), RenderOptions::default(), "```rust meta\ncode\n```")] - #[case::code_empty_body_no_blank_line(Node::Code(Code{value: "".to_string(), lang: None, fence: true, meta: None, position: None}), RenderOptions::default(), "```\n```")] - #[case::code_fence_escalates_past_backtick_run_in_body(Node::Code(Code{value: "aaa\n```".to_string(), lang: None, fence: true, meta: None, position: None}), RenderOptions::default(), "````\naaa\n```\n````")] - #[case::code_fence_escalates_past_longer_run(Node::Code(Code{value: "`````".to_string(), lang: None, fence: true, meta: None, position: None}), RenderOptions::default(), "``````\n`````\n``````")] - #[case::definition(Node::Definition(Definition{ident: "id".to_string(), url: Url::new(attr_keys::URL.to_string()), title: None, label: Some(attr_keys::LABEL.to_string()), position: None}), RenderOptions::default(), "[label]: url")] - #[case::definition(Node::Definition(Definition{ident: "id".to_string(), url: Url::new(attr_keys::URL.to_string()), title: Some(Title::new(attr_keys::TITLE.to_string())), label: Some(attr_keys::LABEL.to_string()), position: None}), RenderOptions::default(), "[label]: url \"title\"")] - #[case::definition(Node::Definition(Definition{ident: "id".to_string(), url: Url::new("".to_string()), title: None, label: Some(attr_keys::LABEL.to_string()), position: None}), RenderOptions::default(), "[label]: <>")] - #[case::delete(Node::Delete(Delete{values: vec!["test".to_string().into()], position: None}), RenderOptions::default(), "~~test~~")] - #[case::emphasis(Node::Emphasis(Emphasis{values: vec!["test".to_string().into()], position: None}), RenderOptions::default(), "*test*")] - #[case::emphasis_nested_in_emphasis_alternates_delimiter(Node::Emphasis(Emphasis{values: vec![Node::Emphasis(Emphasis{values: vec!["foo".to_string().into()], position: None})], position: None}), RenderOptions::default(), "_*foo*_")] - #[case::footnote(Node::Footnote(Footnote{ident: "id".to_string(), values: vec![attr_keys::LABEL.to_string().into()], position: None}), RenderOptions::default(), "[^id]: label")] - #[case::footnote_ref(Node::FootnoteRef(FootnoteRef{ident: attr_keys::LABEL.to_string(), label: Some(attr_keys::LABEL.to_string()), position: None}), RenderOptions::default(), "[^label]")] - #[case::heading(Node::Heading(Heading{depth: 1, values: vec!["test".to_string().into()], position: None}), RenderOptions::default(), "# test")] - #[case::heading(Node::Heading(Heading{depth: 3, values: vec!["test".to_string().into()], position: None}), RenderOptions::default(), "### test")] - #[case::heading_multiline_h1_stays_setext(Node::Heading(Heading{depth: 1, values: vec!["Foo\nBar".to_string().into()], position: None}), RenderOptions::default(), "Foo\nBar\n===")] - #[case::heading_multiline_h2_stays_setext(Node::Heading(Heading{depth: 2, values: vec!["Foo\nBar".to_string().into()], position: None}), RenderOptions::default(), "Foo\nBar\n---")] - #[case::heading_multiline_h3_joins_with_space(Node::Heading(Heading{depth: 3, values: vec!["Foo\nBar".to_string().into()], position: None}), RenderOptions::default(), "### Foo Bar")] - #[case::heading_trailing_hash_escaped(Node::Heading(Heading{depth: 1, values: vec!["foo #".to_string().into()], position: None}), RenderOptions::default(), "# foo \\#")] - #[case::heading_trailing_hash_run_escaped(Node::Heading(Heading{depth: 3, values: vec!["foo ###".to_string().into()], position: None}), RenderOptions::default(), "### foo \\###")] - #[case::heading_trailing_hash_after_multibyte_char(Node::Heading(Heading{depth: 1, values: vec!["foo あ#".to_string().into()], position: None}), RenderOptions::default(), "# foo あ\\#")] - #[case::heading_no_trailing_hash_unaffected(Node::Heading(Heading{depth: 1, values: vec!["foo bar".to_string().into()], position: None}), RenderOptions::default(), "# foo bar")] - #[case::html(Node::Html(Html{value: "<div>test</div>".to_string(), position: None}), RenderOptions::default(), "<div>test</div>")] - #[case::image(Node::Image(Image{alt: attr_keys::ALT.to_string(), url: attr_keys::URL.to_string(), title: None, position: None}), RenderOptions::default(), "![alt](url)")] - #[case::image(Node::Image(Image{alt: attr_keys::ALT.to_string(), url: "url with space".to_string(), title: Some(attr_keys::TITLE.to_string()), position: None}), RenderOptions::default(), "![alt](<url with space> \"title\")")] - #[case::image_ref(Node::ImageRef(ImageRef{alt: attr_keys::ALT.to_string(), ident: "id".to_string(), label: Some("id".to_string()), position: None}), RenderOptions::default(), "![alt][id]")] - #[case::image_ref(Node::ImageRef(ImageRef{alt: "id".to_string(), ident: "id".to_string(), label: Some("id".to_string()), position: None}), RenderOptions::default(), "![id]")] - #[case::code_inline(Node::CodeInline(CodeInline{value: "code".into(), position: None}), RenderOptions::default(), "`code`")] - #[case::code_inline_trailing_backtick_needs_padding(Node::CodeInline(CodeInline{value: "\\[\\`".into(), position: None}), RenderOptions::default(), "`` \\[\\` ``")] - #[case::code_inline_internal_backtick_escalates_fence(Node::CodeInline(CodeInline{value: "foo ` bar".into(), position: None}), RenderOptions::default(), "``foo ` bar``")] - #[case::code_inline_both_ends_backtick_needs_longer_padded_fence(Node::CodeInline(CodeInline{value: "``".into(), position: None}), RenderOptions::default(), "``` `` ```")] - #[case::code_inline_leading_trailing_space_needs_extra_padding(Node::CodeInline(CodeInline{value: " `` ".into(), position: None}), RenderOptions::default(), "``` `` ```")] - #[case::code_inline_all_spaces_untouched(Node::CodeInline(CodeInline{value: " ".into(), position: None}), RenderOptions::default(), "` `")] - #[case::math_inline(Node::MathInline(MathInline{value: "x^2".into(), position: None}), RenderOptions::default(), "$x^2$")] - #[case::link(Node::Link(Link{url: Url::new(attr_keys::URL.to_string()), title: Some(Title::new(attr_keys::TITLE.to_string())), values: vec![attr_keys::VALUE.to_string().into()], position: None}), RenderOptions::default(), "[value](url \"title\")")] - #[case::link(Node::Link(Link{url: Url::new("".to_string()), title: None, values: vec![attr_keys::VALUE.to_string().into()], position: None}), RenderOptions::default(), "[value](<>)")] - #[case::link(Node::Link(Link{url: Url::new(attr_keys::URL.to_string()), title: None, values: vec![attr_keys::VALUE.to_string().into()], position: None}), RenderOptions::default(), "[value](url)")] - #[case::link_ref(Node::LinkRef(LinkRef{ident: "id".to_string(), values: vec!["id".to_string().into()], label: Some("id".to_string()), position: None}), RenderOptions::default(), "[id]")] - #[case::link_ref(Node::LinkRef(LinkRef{ident: "id".to_string(), values: vec!["open".to_string().into()], label: Some("id".to_string()), position: None}), RenderOptions::default(), "[open][id]")] - #[case::math(Node::Math(Math{value: "x^2".to_string(), position: None}), RenderOptions::default(), "$$\nx^2\n$$")] - #[case::strong(Node::Strong(Strong{values: vec!["test".to_string().into()], position: None}), RenderOptions::default(), "**test**")] - #[case::yaml(Node::Yaml(Yaml{value: "key: value".to_string(), position: None}), RenderOptions::default(), "---\nkey: value\n---")] - #[case::toml(Node::Toml(Toml{value: "key = \"value\"".to_string(), position: None}), RenderOptions::default(), "+++\nkey = \"value\"\n+++")] - #[case::break_(Node::Break(Break{position: None}), RenderOptions::default(), "\\\n")] - #[case::horizontal_rule(Node::HorizontalRule(HorizontalRule{ marker: None,position: None}), RenderOptions::default(), "***")] - #[case::mdx_jsx_flow_element(Node::MdxJsxFlowElement(MdxJsxFlowElement{ - name: Some("div".to_string()), - attributes: vec![ - MdxAttributeContent::Property(MdxJsxAttribute { - name: "className".into(), - value: Some(MdxAttributeValue::Literal("container".into())) - }) - ], - children: vec![ - "content".to_string().into() - ], - position: None - }), RenderOptions::default(), "<div className=\"container\">\n content\n</div>")] - #[case::mdx_jsx_flow_element(Node::MdxJsxFlowElement(MdxJsxFlowElement{ - name: Some("div".to_string()), - attributes: vec![ - MdxAttributeContent::Property(MdxJsxAttribute { - name: "className".into(), - value: Some(MdxAttributeValue::Literal("container".into())) - }) - ], - children: Vec::new(), - position: None - }), RenderOptions::default(), "<div className=\"container\" />")] - #[case::mdx_jsx_flow_element(Node::MdxJsxFlowElement(MdxJsxFlowElement{ - name: Some("div".to_string()), - attributes: Vec::new(), - children: Vec::new(), - position: None - }), RenderOptions::default(), "<div />")] - #[case::mdx_jsx_text_element(Node::MdxJsxTextElement(MdxJsxTextElement{ - name: Some("span".into()), - attributes: vec![ - MdxAttributeContent::Expression("...props".into()) - ], - children: vec![ - "inline".to_string().into() - ], - position: None - }), RenderOptions::default(), "<span {...props}>inline</span>")] - #[case::mdx_jsx_text_element(Node::MdxJsxTextElement(MdxJsxTextElement{ - name: Some("span".into()), - attributes: vec![ - MdxAttributeContent::Expression("...props".into()) - ], - children: vec![ - ], - position: None - }), RenderOptions::default(), "<span {...props} />")] - #[case::mdx_jsx_text_element(Node::MdxJsxTextElement(MdxJsxTextElement{ - name: Some("span".into()), - attributes: vec![ - ], - children: vec![ - ], - position: None - }), RenderOptions::default(), "<span />")] - #[case(Node::MdxTextExpression(MdxTextExpression{ - value: "count + 1".into(), - position: None, - }), RenderOptions::default(), "{count + 1}")] - #[case(Node::MdxJsEsm(MdxJsEsm{ - value: "import React from 'react'".into(), - position: None, - }), RenderOptions::default(), "import React from 'react'")] - #[case::fragment_empty(Node::Fragment(Fragment{values: vec![]}), RenderOptions::default(), "")] - #[case::fragment_single(Node::Fragment(Fragment{values: vec![ - Node::Text(Text{value: "hello".to_string(), position: None}) - ]}), RenderOptions::default(), "hello")] - #[case::fragment_multiple(Node::Fragment(Fragment{values: vec![ - Node::Text(Text{value: "hello".to_string(), position: None}), - Node::Text(Text{value: "world".to_string(), position: None}) - ]}), RenderOptions::default(), "hello\nworld")] - #[case::fragment_filters_empty(Node::Fragment(Fragment{values: vec![ - Node::Text(Text{value: "hello".to_string(), position: None}), - Node::Empty, - Node::Text(Text{value: "world".to_string(), position: None}) - ]}), RenderOptions::default(), "hello\nworld")] - #[case::fragment_all_empty(Node::Fragment(Fragment{values: vec![ - Node::Empty, - Node::Empty, - ]}), RenderOptions::default(), "")] - #[case::fragment_empty_text_as_blank_line(Node::Fragment(Fragment{values: vec![ - Node::Text(Text{value: "hello".to_string(), position: None}), - Node::Text(Text{value: "".to_string(), position: None}), - Node::Text(Text{value: "world".to_string(), position: None}) - ]}), RenderOptions::default(), "hello\n\nworld")] - #[case::fragment_filters_nested_empty_fragment(Node::Fragment(Fragment{values: vec![ - Node::Fragment(Fragment{values: vec![Node::Empty, Node::Empty]}), - Node::Text(Text{value: "hello".to_string(), position: None}), - Node::Fragment(Fragment{values: vec![Node::Empty]}), - ]}), RenderOptions::default(), "hello")] - #[case::fragment_filters_deeply_nested_empty_fragment(Node::Fragment(Fragment{values: vec![ - Node::Fragment(Fragment{values: vec![ - Node::Fragment(Fragment{values: vec![Node::Empty]}), - Node::Empty, - ]}), - Node::Text(Text{value: "hello".to_string(), position: None}), - ]}), RenderOptions::default(), "hello")] - #[case::fragment_keeps_nested_fragment_with_blank_line_text(Node::Fragment(Fragment{values: vec![ - Node::Fragment(Fragment{values: vec![Node::Empty]}), - Node::Text(Text{value: "hello".to_string(), position: None}), - Node::Fragment(Fragment{values: vec![Node::Text(Text{value: "".to_string(), position: None})]}), - Node::Text(Text{value: "world".to_string(), position: None}), - ]}), RenderOptions::default(), "hello\n\nworld")] - #[cfg_attr(feature = "wikilink", case::wikilink(Node::WikiLink(WikiLink{target: "target".to_string(), text: None, position: None}), RenderOptions::default(), "[[target]]"))] - #[cfg_attr(feature = "wikilink", case::wikilink_with_text(Node::WikiLink(WikiLink{target: "target".to_string(), text: Some("display text".to_string()), position: None}), RenderOptions::default(), "[[target|display text]]"))] - #[cfg_attr(feature = "wikilink", case::wikilink_same_text(Node::WikiLink(WikiLink{target: "target".to_string(), text: Some("target".to_string()), position: None}), RenderOptions::default(), "[[target]]"))] - fn test_to_string_with(#[case] node: Node, #[case] options: RenderOptions, #[case] expected: &str) { - assert_eq!(node.to_string_with(&options), expected); - } - - #[test] - fn test_node_partial_ord() { - let node1 = Node::Text(Text { - value: "test1".to_string(), - position: Some(Position { - start: Point { line: 1, column: 1 }, - end: Point { line: 1, column: 5 }, - }), - }); - - let node2 = Node::Text(Text { - value: "test2".to_string(), - position: Some(Position { - start: Point { line: 1, column: 6 }, - end: Point { line: 1, column: 10 }, - }), - }); - - let node3 = Node::Text(Text { - value: "test3".to_string(), - position: Some(Position { - start: Point { line: 2, column: 1 }, - end: Point { line: 2, column: 5 }, - }), - }); - - assert_eq!(node1.partial_cmp(&node2), Some(std::cmp::Ordering::Less)); - assert_eq!(node2.partial_cmp(&node1), Some(std::cmp::Ordering::Greater)); - - assert_eq!(node1.partial_cmp(&node3), Some(std::cmp::Ordering::Less)); - assert_eq!(node3.partial_cmp(&node1), Some(std::cmp::Ordering::Greater)); - - let node4 = Node::Text(Text { - value: "test4".to_string(), - position: None, - }); - - assert_eq!(node1.partial_cmp(&node4), Some(std::cmp::Ordering::Less)); - assert_eq!(node4.partial_cmp(&node1), Some(std::cmp::Ordering::Greater)); - - let node5 = Node::Text(Text { - value: "test5".to_string(), - position: None, - }); - - assert_eq!(node4.partial_cmp(&node5), Some(std::cmp::Ordering::Equal)); - - let node6 = Node::Code(Code { - value: "code".to_string(), - lang: None, - fence: true, - meta: None, - position: None, - }); - - assert_eq!(node6.partial_cmp(&node4), Some(std::cmp::Ordering::Less)); - assert_eq!(node4.partial_cmp(&node6), Some(std::cmp::Ordering::Greater)); - } - - #[rstest] - #[case(Node::Blockquote(Blockquote{values: Vec::new(), position: None}), "blockquote")] - #[case(Node::Break(Break{position: None}), "break")] - #[case(Node::Definition(Definition{ident: "".to_string(), url: Url::new("".to_string()), title: None, label: None, position: None}), "definition")] - #[case(Node::Delete(Delete{values: Vec::new(), position: None}), "delete")] - #[case(Node::Heading(Heading{depth: 1, values: Vec::new(), position: None}), "h1")] - #[case(Node::Heading(Heading{depth: 2, values: Vec::new(), position: None}), "h2")] - #[case(Node::Heading(Heading{depth: 3, values: Vec::new(), position: None}), "h3")] - #[case(Node::Heading(Heading{depth: 4, values: Vec::new(), position: None}), "h4")] - #[case(Node::Heading(Heading{depth: 5, values: Vec::new(), position: None}), "h5")] - #[case(Node::Heading(Heading{depth: 6, values: Vec::new(), position: None}), "h6")] - #[case(Node::Heading(Heading{depth: 7, values: Vec::new(), position: None}), "h")] - #[case(Node::Emphasis(Emphasis{values: Vec::new(), position: None}), "emphasis")] - #[case(Node::Footnote(Footnote{ident: "".to_string(), values: Vec::new(), position: None}), "footnote")] - #[case(Node::FootnoteRef(FootnoteRef{ident: "".to_string(), label: None, position: None}), "footnoteref")] - #[case(Node::Html(Html{value: "".to_string(), position: None}), "html")] - #[case(Node::Yaml(Yaml{value: "".to_string(), position: None}), "yaml")] - #[case(Node::Toml(Toml{value: "".to_string(), position: None}), "toml")] - #[case(Node::Image(Image{alt: "".to_string(), url: "".to_string(), title: None, position: None}), "image")] - #[case(Node::ImageRef(ImageRef{alt: "".to_string(), ident: "".to_string(), label: None, position: None}), "image_ref")] - #[case(Node::CodeInline(CodeInline{value: "".into(), position: None}), "code_inline")] - #[case(Node::MathInline(MathInline{value: "".into(), position: None}), "math_inline")] - #[case(Node::Link(Link{url: Url::new("".to_string()), title: None, values: Vec::new(), position: None}), "link")] - #[case(Node::LinkRef(LinkRef{ident: "".to_string(), values: Vec::new(), label: None, position: None}), "link_ref")] - #[case(Node::Math(Math{value: "".to_string(), position: None}), "math")] - #[case(Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: Vec::new(), position: None}), "list")] - #[case(Node::TableAlign(TableAlign{align: Vec::new(), position: None}), "table_align")] - #[case(Node::TableRow(TableRow{values: Vec::new(), position: None}), "table_row")] - #[case(Node::TableCell(TableCell{column: 0, row: 0, values: Vec::new(), position: None}), "table_cell")] - #[case(Node::Code(Code{value: "".to_string(), lang: None, fence: true, meta: None, position: None}), "code")] - #[case(Node::Strong(Strong{values: Vec::new(), position: None}), "strong")] - #[case(Node::HorizontalRule(HorizontalRule{ marker: None,position: None}), "Horizontal_rule")] - #[case(Node::MdxFlowExpression(MdxFlowExpression{value: "".into(), position: None}), "mdx_flow_expression")] - #[case(Node::MdxJsxFlowElement(MdxJsxFlowElement{name: None, attributes: Vec::new(), children: Vec::new(), position: None}), "mdx_jsx_flow_element")] - #[case(Node::MdxJsxTextElement(MdxJsxTextElement{name: None, attributes: Vec::new(), children: Vec::new(), position: None}), "mdx_jsx_text_element")] - #[case(Node::MdxTextExpression(MdxTextExpression{value: "".into(), position: None}), "mdx_text_expression")] - #[case(Node::MdxJsEsm(MdxJsEsm{value: "".into(), position: None}), "mdx_js_esm")] - #[case(Node::Text(Text{value: "".to_string(), position: None}), "text")] - fn test_name(#[case] node: Node, #[case] expected: &str) { - assert_eq!(node.name(), expected); - } - - #[rstest] - #[case(Node::Text(Text{value: "test".to_string(), position: None}), "test")] - #[case(Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: vec![Node::Text(Text{value: "test".to_string(), position: None})], position: None}), "test")] - #[case(Node::Blockquote(Blockquote{values: vec![Node::Text(Text{value: "test".to_string(), position: None})], position: None}), "test")] - #[case(Node::Delete(Delete{values: vec![Node::Text(Text{value: "test".to_string(), position: None})], position: None}), "test")] - #[case(Node::Heading(Heading{depth: 1, values: vec![Node::Text(Text{value: "test".to_string(), position: None})], position: None}), "test")] - #[case(Node::Emphasis(Emphasis{values: vec![Node::Text(Text{value: "test".to_string(), position: None})], position: None}), "test")] - #[case(Node::Footnote(Footnote{ident: "test".to_string(), values: vec![Node::Text(Text{value: "test".to_string(), position: None})], position: None}), "test")] - #[case(Node::FootnoteRef(FootnoteRef{ident: "test".to_string(), label: None, position: None}), "test")] - #[case(Node::Html(Html{value: "test".to_string(), position: None}), "test")] - #[case(Node::Yaml(Yaml{value: "test".to_string(), position: None}), "test")] - #[case(Node::Toml(Toml{value: "test".to_string(), position: None}), "test")] - #[case(Node::Image(Image{alt: attr_keys::ALT.to_string(), url: "test".to_string(), title: None, position: None}), "test")] - #[case(Node::ImageRef(ImageRef{alt: attr_keys::ALT.to_string(), ident: "test".to_string(), label: None, position: None}), "test")] - #[case(Node::CodeInline(CodeInline{value: "test".into(), position: None}), "test")] - #[case(Node::MathInline(MathInline{value: "test".into(), position: None}), "test")] - #[case(Node::Link(Link{url: Url::new("test".to_string()), title: None, values: Vec::new(), position: None}), "test")] - #[case(Node::LinkRef(LinkRef{ident: "test".to_string(), values: Vec::new(), label: None, position: None}), "test")] - #[case(Node::Math(Math{value: "test".to_string(), position: None}), "test")] - #[case(Node::Code(Code{value: "test".to_string(), lang: None, fence: true, meta: None, position: None}), "test")] - #[case(Node::Strong(Strong{values: vec![Node::Text(Text{value: "test".to_string(), position: None})], position: None}), "test")] - #[case(Node::TableCell(TableCell{column: 0, row: 0, values: vec![Node::Text(Text{value: "test".to_string(), position: None})], position: None}), "test")] - #[case(Node::TableRow(TableRow{values: vec![Node::TableCell(TableCell{column: 0, row: 0, values: vec![Node::Text(Text{value: "test".to_string(), position: None})], position: None})], position: None}), "test")] - #[case(Node::Break(Break{position: None}), "")] - #[case(Node::HorizontalRule(HorizontalRule{ marker: None,position: None}), "")] - #[case(Node::TableAlign(TableAlign{align: Vec::new(), position: None}), "")] - #[case(Node::MdxFlowExpression(MdxFlowExpression{value: "test".into(), position: None}), "test")] - #[case(Node::MdxTextExpression(MdxTextExpression{value: "test".into(), position: None}), "test")] - #[case(Node::MdxJsEsm(MdxJsEsm{value: "test".into(), position: None}), "test")] - #[case(Node::MdxJsxFlowElement(MdxJsxFlowElement{name: Some(attr_keys::NAME.to_string()), attributes: Vec::new(), children: vec![Node::Text(Text{value: "test".to_string(), position: None})], position: None}), "test")] - #[case(Node::Definition(Definition{ident: "test".to_string(), url: Url::new(attr_keys::URL.to_string()), title: None, label: None, position: None}), attr_keys::URL)] - #[case(Node::Fragment(Fragment {values: vec![Node::Text(Text{value: "test".to_string(), position: None})]}), "test")] - fn test_value(#[case] node: Node, #[case] expected: &str) { - assert_eq!(node.value(), expected); - } - - #[rstest] - #[case(Node::Text(Text{value: "test".to_string(), position: None}), None)] - #[case(Node::Text(Text{value: "test".to_string(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] - #[case(Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: Vec::new(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] - #[case(Node::Blockquote(Blockquote{values: Vec::new(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] - #[case(Node::Delete(Delete{values: Vec::new(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] - #[case(Node::Heading(Heading{depth: 1, values: Vec::new(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] - #[case(Node::Emphasis(Emphasis{values: Vec::new(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] - #[case(Node::Footnote(Footnote{ident: "".to_string(), values: Vec::new(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] - #[case(Node::FootnoteRef(FootnoteRef{ident: "".to_string(), label: None, position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] - #[case(Node::Html(Html{value: "".to_string(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] - #[case(Node::Yaml(Yaml{value: "".to_string(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] - #[case(Node::Toml(Toml{value: "".to_string(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] - #[case(Node::Image(Image{alt: "".to_string(), url: "".to_string(), title: None, position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] - #[case(Node::ImageRef(ImageRef{alt: "".to_string(), ident: "".to_string(), label: None, position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] - #[case(Node::CodeInline(CodeInline{value: "".into(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] - #[case(Node::MathInline(MathInline{value: "".into(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] - #[case(Node::Link(Link{url: Url("".to_string()), title: None, values: Vec::new(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] - #[case(Node::LinkRef(LinkRef{ident: "".to_string(), values: Vec::new(), label: None, position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] - #[case(Node::Math(Math{value: "".to_string(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] - #[case(Node::Code(Code{value: "".to_string(), lang: None, fence: true, meta: None, position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] - #[case(Node::Strong(Strong{values: Vec::new(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] - #[case(Node::TableCell(TableCell{column: 0, row: 0, values: Vec::new(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] - #[case(Node::TableRow(TableRow{values: Vec::new(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] - #[case(Node::TableAlign(TableAlign{align: Vec::new(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] - #[case(Node::Break(Break{position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] - #[case(Node::HorizontalRule(HorizontalRule{ marker: None,position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] - #[case(Node::MdxFlowExpression(MdxFlowExpression{value: "test".into(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] - #[case(Node::MdxTextExpression(MdxTextExpression{value: "test".into(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] - #[case(Node::MdxJsEsm(MdxJsEsm{value: "test".into(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] - #[case(Node::MdxJsxFlowElement(MdxJsxFlowElement{name: Some("div".to_string()), attributes: Vec::new(), children: Vec::new(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] - #[case(Node::MdxJsxTextElement(MdxJsxTextElement{name: Some("span".into()), attributes: Vec::new(), children: Vec::new(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] - #[case(Node::Definition(Definition{ident: "".to_string(), url: Url("".to_string()), title: None, label: None, position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] - #[case(Node::Fragment(Fragment{values: Vec::new()}), None)] - #[case(Node::Fragment(Fragment{values: vec![ - Node::Text(Text{value: "test1".to_string(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), - Node::Text(Text{value: "test2".to_string(), position: Some(Position{start: Point{line: 1, column: 6}, end: Point{line: 1, column: 10}})}) - ]}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 10}}))] - #[case(Node::Fragment(Fragment{values: vec![ - Node::Text(Text{value: "test".to_string(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), - Node::Text(Text{value: "test2".to_string(), position: None}) - ]}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] - #[case(Node::Fragment(Fragment{values: vec![ - Node::Text(Text{value: "test".to_string(), position: None}), - Node::Text(Text{value: "test2".to_string(), position: Some(Position{start: Point{line: 1, column: 6}, end: Point{line: 1, column: 10}})}) - ]}), Some(Position{start: Point{line: 1, column: 6}, end: Point{line: 1, column: 10}}))] - #[case(Node::Fragment(Fragment{values: vec![ - Node::Text(Text{value: "test2".to_string(), position: Some(Position{start: Point{line: 1, column: 6}, end: Point{line: 1, column: 10}})}), - Node::Text(Text{value: "test".to_string(), position: None}) - ]}), Some(Position{start: Point{line: 1, column: 6}, end: Point{line: 1, column: 10}}))] - #[case(Node::Fragment(Fragment{values: vec![ - Node::Text(Text{value: "test".to_string(), position: None}), - Node::Text(Text{value: "test2".to_string(), position: None}) - ]}), None)] - #[case(Node::Empty, None)] - fn test_position(#[case] node: Node, #[case] expected: Option<Position>) { - assert_eq!(node.position(), expected); - } - - #[rstest] - #[case(Node::Blockquote(Blockquote{values: vec![ - Node::Text(Text{value: "first".to_string(), position: None}), - Node::Text(Text{value: "second".to_string(), position: None}) - ], position: None}), 0, Some(Node::Text(Text{value: "first".to_string(), position: None})))] - #[case(Node::Blockquote(Blockquote{values: vec![ - Node::Text(Text{value: "first".to_string(), position: None}), - Node::Text(Text{value: "second".to_string(), position: None}) - ], position: None}), 1, Some(Node::Text(Text{value: "second".to_string(), position: None})))] - #[case(Node::Blockquote(Blockquote{values: vec![ - Node::Text(Text{value: "first".to_string(), position: None}) - ], position: None}), 1, None)] - #[case(Node::Delete(Delete{values: vec![ - Node::Text(Text{value: "first".to_string(), position: None}), - Node::Text(Text{value: "second".to_string(), position: None}) - ], position: None}), 0, Some(Node::Text(Text{value: "first".to_string(), position: None})))] - #[case(Node::Emphasis(Emphasis{values: vec![ - Node::Text(Text{value: "first".to_string(), position: None}), - Node::Text(Text{value: "second".to_string(), position: None}) - ], position: None}), 1, Some(Node::Text(Text{value: "second".to_string(), position: None})))] - #[case(Node::Strong(Strong{values: vec![ - Node::Text(Text{value: "first".to_string(), position: None}) - ], position: None}), 0, Some(Node::Text(Text{value: "first".to_string(), position: None})))] - #[case(Node::Heading(Heading{depth: 1, values: vec![ - Node::Text(Text{value: "first".to_string(), position: None}), - Node::Text(Text{value: "second".to_string(), position: None}) - ], position: None}), 0, Some(Node::Text(Text{value: "first".to_string(), position: None})))] - #[case(Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: vec![ - Node::Text(Text{value: "first".to_string(), position: None}), - Node::Text(Text{value: "second".to_string(), position: None}) - ], position: None}), 1, Some(Node::Text(Text{value: "second".to_string(), position: None})))] - #[case(Node::TableCell(TableCell{column: 0, row: 0, values: vec![ - Node::Text(Text{value: "cell content".to_string(), position: None}) - ], position: None}), 0, Some(Node::Text(Text{value: "cell content".to_string(), position: None})))] - #[case(Node::TableRow(TableRow{values: vec![ - Node::TableCell(TableCell{column: 0, row: 0, values: Vec::new(), position: None}), - Node::TableCell(TableCell{column: 1, row: 0, values: Vec::new(), position: None}) - ], position: None}), 1, Some(Node::TableCell(TableCell{column: 1, row: 0, values: Vec::new(), position: None})))] - #[case(Node::Text(Text{value: "plain text".to_string(), position: None}), 0, None)] - #[case(Node::Code(Code{value: "code".to_string(), lang: None, fence: true, meta: None, position: None}), 0, None)] - #[case(Node::Html(Html{value: "<div>".to_string(), position: None}), 0, None)] - fn test_find_at_index(#[case] node: Node, #[case] index: usize, #[case] expected: Option<Node>) { - assert_eq!(node.find_at_index(index), expected); - } - - #[rstest] - #[case(Node::Blockquote(Blockquote{values: vec!["test".to_string().into()], position: None}), - Node::Fragment(Fragment{values: vec!["test".to_string().into()]}))] - #[case(Node::Delete(Delete{values: vec!["test".to_string().into()], position: None}), - Node::Fragment(Fragment{values: vec!["test".to_string().into()]}))] - #[case(Node::Heading(Heading{depth: 1, values: vec!["test".to_string().into()], position: None}), - Node::Fragment(Fragment{values: vec!["test".to_string().into()]}))] - #[case(Node::Emphasis(Emphasis{values: vec!["test".to_string().into()], position: None}), - Node::Fragment(Fragment{values: vec!["test".to_string().into()]}))] - #[case(Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: vec!["test".to_string().into()], position: None}), - Node::Fragment(Fragment{values: vec!["test".to_string().into()]}))] - #[case(Node::Strong(Strong{values: vec!["test".to_string().into()], position: None}), - Node::Fragment(Fragment{values: vec!["test".to_string().into()]}))] - #[case(Node::Link(Link{url: Url(attr_keys::URL.to_string()), title: None, values: vec!["test".to_string().into()], position: None}), - Node::Fragment(Fragment{values: vec!["test".to_string().into()]}))] - #[case(Node::LinkRef(LinkRef{ident: "id".to_string(), values: vec!["test".to_string().into()], label: None, position: None}), - Node::Fragment(Fragment{values: vec!["test".to_string().into()]}))] - #[case(Node::Footnote(Footnote{ident: "id".to_string(), values: vec!["test".to_string().into()], position: None}), - Node::Fragment(Fragment{values: vec!["test".to_string().into()]}))] - #[case(Node::TableCell(TableCell{column: 0, row: 0, values: vec!["test".to_string().into()], position: None}), - Node::Fragment(Fragment{values: vec!["test".to_string().into()]}))] - #[case(Node::TableRow(TableRow{values: vec!["test".to_string().into()], position: None}), - Node::Fragment(Fragment{values: vec!["test".to_string().into()]}))] - #[case(Node::Fragment(Fragment{values: vec!["test".to_string().into()]}), - Node::Fragment(Fragment{values: vec!["test".to_string().into()]}))] - #[case(Node::Text(Text{value: "test".to_string(), position: None}), - Node::Empty)] - #[case(Node::Code(Code{value: "test".to_string(), lang: None, fence: true, meta: None, position: None}), - Node::Empty)] - #[case(Node::Image(Image{alt: attr_keys::ALT.to_string(), url: attr_keys::URL.to_string(), title: None, position: None}), - Node::Empty)] - #[case(Node::Empty, Node::Empty)] - fn test_to_fragment(#[case] node: Node, #[case] expected: Node) { - assert_eq!(node.clone().to_fragment(), expected); - assert_eq!(node.into_fragment(), expected); - } - - // Regression coverage for the 0.6.2 blank-line bug: `eval_markdown_node` - // replaces a non-matching `select()` result with `child_node.to_fragment()`, - // not `Node::Empty` directly. For container nodes this yields a `Fragment` - // wrapping the (now `Empty`) children, which must still render to "" with - // no stray blank lines, while a partially-matching container must render - // only the surviving content. - #[rstest] - #[case::blockquote_all_empty(Node::Blockquote(Blockquote{values: vec![Node::Empty, Node::Empty], position: None}), "")] - #[case::blockquote_mixed(Node::Blockquote(Blockquote{values: vec![Node::Empty, Node::Text(Text{value: "kept".to_string(), position: None})], position: None}), "kept")] - #[case::delete_all_empty(Node::Delete(Delete{values: vec![Node::Empty, Node::Empty], position: None}), "")] - #[case::delete_mixed(Node::Delete(Delete{values: vec![Node::Text(Text{value: "kept".to_string(), position: None}), Node::Empty], position: None}), "kept")] - #[case::heading_all_empty(Node::Heading(Heading{depth: 1, values: vec![Node::Empty, Node::Empty], position: None}), "")] - #[case::heading_mixed(Node::Heading(Heading{depth: 1, values: vec![Node::Empty, Node::Text(Text{value: "kept".to_string(), position: None})], position: None}), "kept")] - #[case::emphasis_all_empty(Node::Emphasis(Emphasis{values: vec![Node::Empty, Node::Empty], position: None}), "")] - #[case::emphasis_mixed(Node::Emphasis(Emphasis{values: vec![Node::Empty, Node::Text(Text{value: "kept".to_string(), position: None})], position: None}), "kept")] - #[case::list_all_empty(Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: vec![Node::Empty, Node::Empty], position: None}), "")] - #[case::list_mixed(Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: vec![Node::Empty, Node::Text(Text{value: "kept".to_string(), position: None})], position: None}), "kept")] - #[case::strong_all_empty(Node::Strong(Strong{values: vec![Node::Empty, Node::Empty], position: None}), "")] - #[case::strong_mixed(Node::Strong(Strong{values: vec![Node::Text(Text{value: "kept".to_string(), position: None}), Node::Empty], position: None}), "kept")] - #[case::link_all_empty(Node::Link(Link{url: Url(attr_keys::URL.to_string()), title: None, values: vec![Node::Empty, Node::Empty], position: None}), "")] - #[case::link_mixed(Node::Link(Link{url: Url(attr_keys::URL.to_string()), title: None, values: vec![Node::Empty, Node::Text(Text{value: "kept".to_string(), position: None})], position: None}), "kept")] - #[case::link_ref_all_empty(Node::LinkRef(LinkRef{ident: "id".to_string(), values: vec![Node::Empty, Node::Empty], label: None, position: None}), "")] - #[case::link_ref_mixed(Node::LinkRef(LinkRef{ident: "id".to_string(), values: vec![Node::Empty, Node::Text(Text{value: "kept".to_string(), position: None})], label: None, position: None}), "kept")] - #[case::footnote_all_empty(Node::Footnote(Footnote{ident: "id".to_string(), values: vec![Node::Empty, Node::Empty], position: None}), "")] - #[case::footnote_mixed(Node::Footnote(Footnote{ident: "id".to_string(), values: vec![Node::Empty, Node::Text(Text{value: "kept".to_string(), position: None})], position: None}), "kept")] - #[case::table_cell_all_empty(Node::TableCell(TableCell{column: 0, row: 0, values: vec![Node::Empty, Node::Empty], position: None}), "")] - #[case::table_cell_mixed(Node::TableCell(TableCell{column: 0, row: 0, values: vec![Node::Empty, Node::Text(Text{value: "kept".to_string(), position: None})], position: None}), "kept")] - fn test_to_fragment_then_render_skips_non_matching(#[case] node: Node, #[case] expected: &str) { - assert_eq!(node.to_fragment().to_string_with(&RenderOptions::default()), expected); - } - - #[rstest] - #[case( - &mut Node::Blockquote(Blockquote{values: vec![ - Node::Text(Text{value: "old".to_string(), position: None}) - ], position: None}), - Node::Fragment(Fragment{values: vec![ - Node::Text(Text{value: "new".to_string(), position: None}) - ]}), - Node::Blockquote(Blockquote{values: vec![ - Node::Text(Text{value: "new".to_string(), position: None}) - ], position: None}) - )] - #[case( - &mut Node::Delete(Delete{values: vec![ - Node::Text(Text{value: "old".to_string(), position: None}) - ], position: None}), - Node::Fragment(Fragment{values: vec![ - Node::Text(Text{value: "new".to_string(), position: None}) - ]}), - Node::Delete(Delete{values: vec![ - Node::Text(Text{value: "new".to_string(), position: None}) - ], position: None}) - )] - #[case( - &mut Node::Emphasis(Emphasis{values: vec![ - Node::Text(Text{value: "old".to_string(), position: None}) - ], position: None}), - Node::Fragment(Fragment{values: vec![ - Node::Text(Text{value: "new".to_string(), position: None}) - ]}), - Node::Emphasis(Emphasis{values: vec![ - Node::Text(Text{value: "new".to_string(), position: None}) - ], position: None}) - )] - #[case( - &mut Node::Strong(Strong{values: vec![ - Node::Text(Text{value: "old".to_string(), position: None}) - ], position: None}), - Node::Fragment(Fragment{values: vec![ - Node::Text(Text{value: "new".to_string(), position: None}) - ]}), - Node::Strong(Strong{values: vec![ - Node::Text(Text{value: "new".to_string(), position: None}) - ], position: None}) - )] - #[case( - &mut Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: vec![ - Node::Text(Text{value: "old".to_string(), position: None}) - ], position: None}), - Node::Fragment(Fragment{values: vec![ - Node::Text(Text{value: "new".to_string(), position: None}) - ]}), - Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: vec![ - Node::Text(Text{value: "new".to_string(), position: None}) - ], position: None}) - )] - #[case( - &mut Node::Heading(Heading{depth: 1, values: vec![ - Node::Text(Text{value: "old".to_string(), position: None}) - ], position: None}), - Node::Fragment(Fragment{values: vec![ - Node::Text(Text{value: "new".to_string(), position: None}) - ]}), - Node::Heading(Heading{depth: 1, values: vec![ - Node::Text(Text{value: "new".to_string(), position: None}) - ], position: None}) - )] - #[case( - &mut Node::Link(Link{url: Url(attr_keys::URL.to_string()), title: None, values: vec![ - Node::Text(Text{value: "old".to_string(), position: None}) - ], position: None}), - Node::Fragment(Fragment{values: vec![ - Node::Text(Text{value: "new".to_string(), position: None}) - ]}), - Node::Link(Link{url: Url(attr_keys::URL.to_string()), title: None, values: vec![ - Node::Text(Text{value: "new".to_string(), position: None}) - ], position: None}) - )] - #[case( - &mut Node::LinkRef(LinkRef{ident: "id".to_string(), values: vec![ - Node::Text(Text{value: "old".to_string(), position: None}) - ], label: None, position: None}), - Node::Fragment(Fragment{values: vec![ - Node::Text(Text{value: "new".to_string(), position: None}) - ]}), - Node::LinkRef(LinkRef{ident: "id".to_string(), values: vec![ - Node::Text(Text{value: "new".to_string(), position: None}) - ], label: None, position: None}) - )] - #[case( - &mut Node::Footnote(Footnote{ident: "id".to_string(), values: vec![ - Node::Text(Text{value: "old".to_string(), position: None}) - ], position: None}), - Node::Fragment(Fragment{values: vec![ - Node::Text(Text{value: "new".to_string(), position: None}) - ]}), - Node::Footnote(Footnote{ident: "id".to_string(), values: vec![ - Node::Text(Text{value: "new".to_string(), position: None}) - ], position: None}) - )] - #[case( - &mut Node::TableCell(TableCell{column: 0, row: 0, values: vec![ - Node::Text(Text{value: "old".to_string(), position: None}) - ], position: None}), - Node::Fragment(Fragment{values: vec![ - Node::Text(Text{value: "new".to_string(), position: None}) - ]}), - Node::TableCell(TableCell{column: 0, row: 0, values: vec![ - Node::Text(Text{value: "new".to_string(), position: None}) - ], position: None}) - )] - #[case( - &mut Node::TableRow(TableRow{values: vec![ - Node::TableCell(TableCell{column: 0, row: 0, values: vec![ - Node::Text(Text{value: "old".to_string(), position: None}) - ], position: None}) - ], position: None}), - Node::Fragment(Fragment{values: vec![ - Node::TableCell(TableCell{column: 0, row: 0, values: vec![ - Node::Text(Text{value: "new".to_string(), position: None}) - ], position: None}) - ]}), - Node::TableRow(TableRow{values: vec![ - Node::TableCell(TableCell{column: 0, row: 0, values: vec![ - Node::Text(Text{value: "new".to_string(), position: None}) - ], position: None}) - ], position: None}) - )] - #[case( - &mut Node::Text(Text{value: "old".to_string(), position: None}), - Node::Fragment(Fragment{values: vec![ - Node::Text(Text{value: "new".to_string(), position: None}) - ]}), - Node::Text(Text{value: "old".to_string(), position: None}) - )] - #[case( - &mut Node::Blockquote(Blockquote{values: vec![ - Node::Text(Text{value: "text1".to_string(), position: None}), - Node::Text(Text{value: "text2".to_string(), position: None}) - ], position: None}), - Node::Fragment(Fragment{values: vec![ - Node::Text(Text{value: "new1".to_string(), position: None}), - Node::Text(Text{value: "new2".to_string(), position: None}) - ]}), - Node::Blockquote(Blockquote{values: vec![ - Node::Text(Text{value: "new1".to_string(), position: None}), - Node::Text(Text{value: "new2".to_string(), position: None}) - ], position: None}) - )] - #[case( - &mut Node::Strong(Strong{values: vec![ - Node::Text(Text{value: "text1".to_string(), position: None}), - Node::Text(Text{value: "text2".to_string(), position: None}) - ], position: None}), - Node::Fragment(Fragment{values: vec![ - Node::Empty, - Node::Text(Text{value: "new2".to_string(), position: None}) - ]}), - Node::Strong(Strong{values: vec![ - Node::Text(Text{value: "text1".to_string(), position: None}), - Node::Text(Text{value: "new2".to_string(), position: None}) - ], position: None}) - )] - #[case( - &mut Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: vec![ - Node::Text(Text{value: "text1".to_string(), position: None}), - Node::Text(Text{value: "text2".to_string(), position: None}) - ], position: None}), - Node::Fragment(Fragment{values: vec![ - Node::Text(Text{value: "new1".to_string(), position: None}), - Node::Fragment(Fragment{values: Vec::new()}) - ]}), - Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: vec![ - Node::Text(Text{value: "new1".to_string(), position: None}), - Node::Text(Text{value: "text2".to_string(), position: None}) - ], position: None}) - )] - fn test_apply_fragment(#[case] node: &mut Node, #[case] fragment: Node, #[case] expected: Node) { - node.apply_fragment(fragment); - assert_eq!(*node, expected); - } - - #[rstest] - #[case(Node::Text(Text{value: "test".to_string(), position: None}), - Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}, - Node::Text(Text{value: "test".to_string(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}))] - #[case(Node::Code(Code{value: "code".to_string(), lang: None, fence: true, meta: None, position: None}), - Position{start: Point{line: 1, column: 1}, end: Point{line: 3, column: 3}}, - Node::Code(Code{value: "code".to_string(), lang: None, fence: true, meta: None, position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 3, column: 3}})}))] - #[case(Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 1, checked: None, ordered: false, values: vec![], position: None}), - Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}, - Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 1, checked: None, ordered: false, values: vec![], position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}))] - #[case(Node::Definition(Definition{ident: "id".to_string(), url: Url::new(attr_keys::URL.to_string()), title: None, label: None, position: None}), - Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 10}}, - Node::Definition(Definition{ident: "id".to_string(), url: Url::new(attr_keys::URL.to_string()), title: None, label: None, position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 10}})}))] - #[case(Node::Delete(Delete{values: vec![Node::Text(Text{value: "test".to_string(), position: None})], position: None}), - Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}, - Node::Delete(Delete{values: vec![Node::Text(Text{value: "test".to_string(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})})], position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}))] - #[case(Node::Emphasis(Emphasis{values: vec![Node::Text(Text{value: "test".to_string(), position: None})], position: None}), - Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}, - Node::Emphasis(Emphasis{values: vec![Node::Text(Text{value: "test".to_string(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})})], position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}))] - #[case(Node::Footnote(Footnote{ident: "id".to_string(), values: vec![Node::Text(Text{value: "test".to_string(), position: None})], position: None}), - Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}, - Node::Footnote(Footnote{ident: "id".to_string(), values: vec![Node::Text(Text{value: "test".to_string(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})})], position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}))] - #[case(Node::FootnoteRef(FootnoteRef{ident: "id".to_string(), label: Some(attr_keys::LABEL.to_string()), position: None}), - Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}, - Node::FootnoteRef(FootnoteRef{ident: "id".to_string(), label: Some(attr_keys::LABEL.to_string()), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}))] - #[case(Node::Html(Html{value: "<div>test</div>".to_string(), position: None}), - Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 15}}, - Node::Html(Html{value: "<div>test</div>".to_string(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 15}})}))] - #[case(Node::Yaml(Yaml{value: "key: value".to_string(), position: None}), - Position{start: Point{line: 1, column: 1}, end: Point{line: 3, column: 4}}, - Node::Yaml(Yaml{value: "key: value".to_string(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 3, column: 4}})}))] - #[case(Node::Toml(Toml{value: "key = \"value\"".to_string(), position: None}), - Position{start: Point{line: 1, column: 1}, end: Point{line: 3, column: 4}}, - Node::Toml(Toml{value: "key = \"value\"".to_string(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 3, column: 4}})}))] - #[case(Node::Image(Image{alt: attr_keys::ALT.to_string(), url: attr_keys::URL.to_string(), title: None, position: None}), - Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 12}}, - Node::Image(Image{alt: attr_keys::ALT.to_string(), url: attr_keys::URL.to_string(), title: None, position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 12}})}))] - #[case(Node::ImageRef(ImageRef{alt: attr_keys::ALT.to_string(), ident: "id".to_string(), label: None, position: None}), - Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 10}}, - Node::ImageRef(ImageRef{alt: attr_keys::ALT.to_string(), ident: "id".to_string(), label: None, position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 10}})}))] - #[case(Node::CodeInline(CodeInline{value: "code".into(), position: None}), - Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 7}}, - Node::CodeInline(CodeInline{value: "code".into(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 7}})}))] - #[case(Node::MathInline(MathInline{value: "x^2".into(), position: None}), - Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}, - Node::MathInline(MathInline{value: "x^2".into(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}))] - #[case(Node::Link(Link{url: Url::new(attr_keys::URL.to_string()), title: None, values: vec![Node::Text(Text{value: "text".to_string(), position: None})], position: None}), - Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 10}}, - Node::Link(Link{url: Url::new(attr_keys::URL.to_string()), title: None, values: vec![Node::Text(Text{value: "text".to_string(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 10}})})], position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 10}})}))] - #[case(Node::LinkRef(LinkRef{ident: "id".to_string(), values: vec![Node::Text(Text{value: "text".to_string(), position: None})], label: None, position: None}), - Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 10}}, - Node::LinkRef(LinkRef{ident: "id".to_string(), values: vec![Node::Text(Text{value: "text".to_string(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 10}})})], label: None, position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 10}})}))] - #[case(Node::Math(Math{value: "x^2".to_string(), position: None}), - Position{start: Point{line: 1, column: 1}, end: Point{line: 3, column: 3}}, - Node::Math(Math{value: "x^2".to_string(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 3, column: 3}})}))] - #[case(Node::TableCell(TableCell{column: 0, row: 0, values: vec![Node::Text(Text{value: "cell".to_string(), position: None})], position: None}), - Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 6}}, - Node::TableCell(TableCell{column: 0, row: 0, values: vec![Node::Text(Text{value: "cell".to_string(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 6}})})], position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 6}})}))] - #[case(Node::TableAlign(TableAlign{align: vec![TableAlignKind::Left, TableAlignKind::Right], position: None}), - Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 15}}, - Node::TableAlign(TableAlign{align: vec![TableAlignKind::Left, TableAlignKind::Right], position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 15}})}))] - #[case(Node::MdxFlowExpression(MdxFlowExpression{value: "test".into(), position: None}), - Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 7}}, - Node::MdxFlowExpression(MdxFlowExpression{value: "test".into(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 7}})}))] - #[case(Node::MdxTextExpression(MdxTextExpression{value: "test".into(), position: None}), - Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 7}}, - Node::MdxTextExpression(MdxTextExpression{value: "test".into(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 7}})}))] - #[case(Node::MdxJsEsm(MdxJsEsm{value: "import React from 'react'".into(), position: None}), - Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 25}}, - Node::MdxJsEsm(MdxJsEsm{value: "import React from 'react'".into(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 25}})}))] - #[case(Node::MdxJsxTextElement(MdxJsxTextElement{name: Some("span".into()), attributes: Vec::new(), children: vec![Node::Text(Text{value: "text".to_string(), position: None})], position: None}), - Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 20}}, - Node::MdxJsxTextElement(MdxJsxTextElement{name: Some("span".into()), attributes: Vec::new(), children: vec![Node::Text(Text{value: "text".to_string(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 20}})})], position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 20}})}))] - #[case(Node::Break(Break{position: None}), - Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 2}}, - Node::Break(Break{position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 2}})}))] - #[case(Node::Empty, - Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}, - Node::Empty)] - #[case(Node::Fragment(Fragment{values: vec![ - Node::Text(Text{value: "test1".to_string(), position: None}), - Node::Text(Text{value: "test2".to_string(), position: None}) - ]}), - Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 10}}, - Node::Fragment(Fragment{values: vec![ - Node::Text(Text{value: "test1".to_string(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 10}})}), - Node::Text(Text{value: "test2".to_string(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 10}})}) - ]}))] - #[case(Node::Blockquote(Blockquote{values: vec![ - Node::Text(Text{value: "test".to_string(), position: None})], position: None}), - Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}, - Node::Blockquote(Blockquote{values: vec![ - Node::Text(Text{value: "test".to_string(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}) - ], position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}))] - #[case(Node::Heading(Heading{depth: 1, values: vec![ - Node::Text(Text{value: "test".to_string(), position: None})], position: None}), - Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}, - Node::Heading(Heading{depth: 1, values: vec![ - Node::Text(Text{value: "test".to_string(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}) - ], position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}))] - #[case(Node::Strong(Strong{values: vec![ - Node::Text(Text{value: "test".to_string(), position: None})], position: None}), - Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}, - Node::Strong(Strong{values: vec![ - Node::Text(Text{value: "test".to_string(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}) - ], position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}))] - #[case(Node::TableRow(TableRow{values: vec![ - Node::TableCell(TableCell{column: 0, row: 0, values: vec![ - Node::Text(Text{value: "cell".to_string(), position: None}) - ], position: None}) - ], position: None}), - Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 10}}, - Node::TableRow(TableRow{values: vec![ - Node::TableCell(TableCell{column: 0, row: 0, values: vec![ - Node::Text(Text{value: "cell".to_string(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 10}})}) - ], position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 10}})}) - ], position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 10}})}))] - #[case(Node::MdxJsxFlowElement(MdxJsxFlowElement{ - name: Some("div".to_string()), - attributes: Vec::new(), - children: vec![Node::Text(Text{value: "content".to_string(), position: None})], - position: None - }), - Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 20}}, - Node::MdxJsxFlowElement(MdxJsxFlowElement{ - name: Some("div".to_string()), - attributes: Vec::new(), - children: vec![Node::Text(Text{value: "content".to_string(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 20}})})], - position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 20}}) - }))] - fn test_set_position(#[case] mut node: Node, #[case] position: Position, #[case] expected: Node) { - node.set_position(Some(position)); - assert_eq!(node, expected); - } - - fn some_position() -> Option<Position> { - Some(Position { - start: Point { line: 1, column: 1 }, - end: Point { line: 1, column: 5 }, - }) - } - - #[test] - fn test_strip_positions_top_level() { - let mut node = Node::Text(Text { - value: "test".to_string(), - position: some_position(), - }); - node.strip_positions(); - assert_eq!(node.position(), None); - } - - #[test] - fn test_strip_positions_recurses_into_children() { - let mut node = Node::Blockquote(Blockquote { - values: vec![Node::Strong(Strong { - values: vec![Node::Text(Text { - value: "test".to_string(), - position: some_position(), - })], - position: some_position(), - })], - position: some_position(), - }); - - node.strip_positions(); - - assert_eq!(node.position(), None); - let strong = &node.children()[0]; - assert_eq!(strong.position(), None); - assert_eq!(strong.children()[0].position(), None); - } - - #[test] - fn test_clear_text_position_at_leaf_only() { - let mut node = Node::Heading(Heading { - depth: 1, - values: vec![Node::Text(Text { - value: "title".to_string(), - position: some_position(), - })], - position: some_position(), - }); - - node.clear_text_position_at(0); - - assert_eq!(node.position(), some_position(), "outer node keeps its position"); - assert_eq!(node.children()[0].position(), None, "escaped leaf loses its position"); - } - - #[test] - fn test_clear_text_position_at_selected_index() { - let mut node = Node::List(List { - marker: None, - start: None, - spread: false, - index: 0, - level: 0, - checked: None, - ordered: false, - values: vec![ - Node::Text(Text { - value: "a".to_string(), - position: some_position(), - }), - Node::Text(Text { - value: "b".to_string(), - position: some_position(), - }), - ], - position: some_position(), - }); - - node.clear_text_position_at(1); - - assert_eq!( - node.children()[0].position(), - some_position(), - "untouched sibling keeps its position" - ); - assert_eq!(node.children()[1].position(), None, "selected leaf loses its position"); - } - - #[rstest] - #[case(Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: vec!["test".to_string().into()], position: None}), true)] - #[case(Node::List(List{ marker: None,start: None, spread: false, index: 1, level: 2, checked: Some(true), ordered: false, values: vec!["test".to_string().into()], position: None}), true)] - #[case(Node::Text(Text{value: "test".to_string(), position: None}), false)] - fn test_is_list(#[case] node: Node, #[case] expected: bool) { - assert_eq!(node.is_list(), expected); - } - - #[rstest] - #[case(Url::new("https://example.com".to_string()), RenderOptions{link_url_style: UrlSurroundStyle::None, ..Default::default()}, "https://example.com")] - #[case(Url::new("https://example.com".to_string()), RenderOptions{link_url_style: UrlSurroundStyle::Angle, ..Default::default()}, "<https://example.com>")] - #[case(Url::new("".to_string()), RenderOptions::default(), "<>")] - fn test_url_to_string_with(#[case] url: Url, #[case] options: RenderOptions, #[case] expected: &str) { - assert_eq!(url.to_string_with(&options), expected); - } - - #[rstest] - #[case(Title::new(attr_keys::TITLE.to_string()), RenderOptions::default(), "\"title\"")] - #[case(Title::new(r#"title with "quotes""#.to_string()), RenderOptions::default(), r#""title with \"quotes\"""#)] - #[case(Title::new("title with spaces".to_string()), RenderOptions::default(), "\"title with spaces\"")] - #[case(Title::new("".to_string()), RenderOptions::default(), "\"\"")] - #[case(Title::new(attr_keys::TITLE.to_string()), RenderOptions{link_title_style: TitleSurroundStyle::Single, ..Default::default()}, "'title'")] - #[case(Title::new("title with 'quotes'".to_string()), RenderOptions{link_title_style: TitleSurroundStyle::Double, ..Default::default()}, "\"title with 'quotes'\"")] - #[case(Title::new(attr_keys::TITLE.to_string()), RenderOptions{link_title_style: TitleSurroundStyle::Paren, ..Default::default()}, "(title)")] - fn test_title_to_string_with(#[case] title: Title, #[case] options: RenderOptions, #[case] expected: &str) { - assert_eq!(title.to_string_with(&options), expected); - } - - #[rstest] - #[case(Node::Fragment(Fragment{values: vec![]}), true)] - #[case(Node::Fragment(Fragment{values: vec![ - Node::Text(Text{value: "not_empty".to_string(), position: None}) - ]}), false)] - #[case(Node::Fragment(Fragment{values: vec![ - Node::Fragment(Fragment{values: vec![]}), - Node::Fragment(Fragment{values: vec![]}) - ]}), true)] - #[case(Node::Fragment(Fragment{values: vec![ - Node::Fragment(Fragment{values: vec![]}), - Node::Text(Text{value: "not_empty".to_string(), position: None}) - ]}), false)] - #[case(Node::Text(Text{value: "not_fragment".to_string(), position: None}), false)] - fn test_is_empty_fragment(#[case] node: Node, #[case] expected: bool) { - assert_eq!(node.is_empty_fragment(), expected); - } - - #[rstest] - #[case::footnote(Node::Footnote(Footnote{ident: "id".to_string(), values: Vec::new(), position: None}), attr_keys::IDENT, Some(AttrValue::String("id".to_string())))] - #[case::footnote(Node::Footnote(Footnote{ident: "id".to_string(), values: Vec::new(), position: None}), "unknown", None)] - #[case::html(Node::Html(Html{value: "<div>test</div>".to_string(), position: None}), attr_keys::VALUE, Some(AttrValue::String("<div>test</div>".to_string())))] - #[case::html(Node::Html(Html{value: "<div>test</div>".to_string(), position: None}), "unknown", None)] - #[case::text(Node::Text(Text{value: "text".to_string(), position: None}), attr_keys::VALUE, Some(AttrValue::String("text".to_string())))] - #[case::text(Node::Text(Text{value: "text".to_string(), position: None}), "unknown", None)] - #[case::code(Node::Code(Code{value: "code".to_string(), lang: Some("rust".to_string()), meta: Some("meta".to_string()), fence: true, position: None}), attr_keys::VALUE, Some(AttrValue::String("code".to_string())))] - #[case::code(Node::Code(Code{value: "code".to_string(), lang: Some("rust".to_string()), meta: Some("meta".to_string()), fence: true, position: None}), attr_keys::LANG, Some(AttrValue::String("rust".to_string())))] - #[case::code(Node::Code(Code{value: "code".to_string(), lang: Some("rust".to_string()), meta: Some("meta".to_string()), fence: true, position: None}), "meta", Some(AttrValue::String("meta".to_string())))] - #[case::code(Node::Code(Code{value: "code".to_string(), lang: Some("rust".to_string()), meta: Some("meta".to_string()), fence: true, position: None}), attr_keys::FENCE, Some(AttrValue::Boolean(true)))] - #[case::code(Node::Code(Code{value: "code".to_string(), lang: None, meta: None, fence: false, position: None}), attr_keys::FENCE, Some(AttrValue::Boolean(false)))] - #[case::code_inline(Node::CodeInline(CodeInline{value: "inline".into(), position: None}), attr_keys::VALUE, Some(AttrValue::String("inline".to_string())))] - #[case::math_inline(Node::MathInline(MathInline{value: "math".into(), position: None}), attr_keys::VALUE, Some(AttrValue::String("math".to_string())))] - #[case::math(Node::Math(Math{value: "math".to_string(), position: None}), attr_keys::VALUE, Some(AttrValue::String("math".to_string())))] - #[case::yaml(Node::Yaml(Yaml{value: "yaml".to_string(), position: None}), attr_keys::VALUE, Some(AttrValue::String("yaml".to_string())))] - #[case::toml(Node::Toml(Toml{value: "toml".to_string(), position: None}), attr_keys::VALUE, Some(AttrValue::String("toml".to_string())))] - #[case::image(Node::Image(Image{alt: attr_keys::ALT.to_string(), url: attr_keys::URL.to_string(), title: Some(attr_keys::TITLE.to_string()), position: None}), attr_keys::ALT, Some(AttrValue::String(attr_keys::ALT.to_string())))] - #[case::image(Node::Image(Image{alt: attr_keys::ALT.to_string(), url: attr_keys::URL.to_string(), title: Some(attr_keys::TITLE.to_string()), position: None}), attr_keys::URL, Some(AttrValue::String(attr_keys::URL.to_string())))] - #[case::image(Node::Image(Image{alt: attr_keys::ALT.to_string(), url: attr_keys::URL.to_string(), title: Some(attr_keys::TITLE.to_string()), position: None}), attr_keys::TITLE, Some(AttrValue::String(attr_keys::TITLE.to_string())))] - #[case::image_ref(Node::ImageRef(ImageRef{alt: attr_keys::ALT.to_string(), ident: "id".to_string(), label: Some(attr_keys::LABEL.to_string()), position: None}), attr_keys::ALT, Some(AttrValue::String(attr_keys::ALT.to_string())))] - #[case::image_ref(Node::ImageRef(ImageRef{alt: attr_keys::ALT.to_string(), ident: "id".to_string(), label: Some(attr_keys::LABEL.to_string()), position: None}), attr_keys::IDENT, Some(AttrValue::String("id".to_string())))] - #[case::image_ref(Node::ImageRef(ImageRef{alt: attr_keys::ALT.to_string(), ident: "id".to_string(), label: Some(attr_keys::LABEL.to_string()), position: None}), attr_keys::LABEL, Some(AttrValue::String(attr_keys::LABEL.to_string())))] - #[case::link(Node::Link(Link{url: Url::new(attr_keys::URL.to_string()), title: Some(Title::new(attr_keys::TITLE.to_string())), values: Vec::new(), position: None}), attr_keys::URL, Some(AttrValue::String(attr_keys::URL.to_string())))] - #[case::link(Node::Link(Link{url: Url::new(attr_keys::URL.to_string()), title: Some(Title::new(attr_keys::TITLE.to_string())), values: Vec::new(), position: None}), attr_keys::TITLE, Some(AttrValue::String(attr_keys::TITLE.to_string())))] - #[case::link_ref(Node::LinkRef(LinkRef{ident: "id".to_string(), values: Vec::new(), label: Some(attr_keys::LABEL.to_string()), position: None}), attr_keys::IDENT, Some(AttrValue::String("id".to_string())))] - #[case::link_ref(Node::LinkRef(LinkRef{ident: "id".to_string(), values: Vec::new(), label: Some(attr_keys::LABEL.to_string()), position: None}), attr_keys::LABEL, Some(AttrValue::String(attr_keys::LABEL.to_string())))] - #[case::footnote_ref(Node::FootnoteRef(FootnoteRef{ident: "id".to_string(), label: Some(attr_keys::LABEL.to_string()), position: None}), attr_keys::IDENT, Some(AttrValue::String("id".to_string())))] - #[case::footnote_ref(Node::FootnoteRef(FootnoteRef{ident: "id".to_string(), label: Some(attr_keys::LABEL.to_string()), position: None}), attr_keys::LABEL, Some(AttrValue::String(attr_keys::LABEL.to_string())))] - #[case::definition(Node::Definition(Definition{ident: "id".to_string(), url: Url::new(attr_keys::URL.to_string()), title: Some(Title::new(attr_keys::TITLE.to_string())), label: Some(attr_keys::LABEL.to_string()), position: None}), attr_keys::IDENT, Some(AttrValue::String("id".to_string())))] - #[case::definition(Node::Definition(Definition{ident: "id".to_string(), url: Url::new(attr_keys::URL.to_string()), title: Some(Title::new(attr_keys::TITLE.to_string())), label: Some(attr_keys::LABEL.to_string()), position: None}), attr_keys::URL, Some(AttrValue::String(attr_keys::URL.to_string())))] - #[case::definition(Node::Definition(Definition{ident: "id".to_string(), url: Url::new(attr_keys::URL.to_string()), title: Some(Title::new(attr_keys::TITLE.to_string())), label: Some(attr_keys::LABEL.to_string()), position: None}), attr_keys::TITLE, Some(AttrValue::String(attr_keys::TITLE.to_string())))] - #[case::definition(Node::Definition(Definition{ident: "id".to_string(), url: Url::new(attr_keys::URL.to_string()), title: Some(Title::new(attr_keys::TITLE.to_string())), label: Some(attr_keys::LABEL.to_string()), position: None}), attr_keys::LABEL, Some(AttrValue::String(attr_keys::LABEL.to_string())))] - #[case::heading(Node::Heading(Heading{depth: 3, values: Vec::new(), position: None}), "depth", Some(AttrValue::Integer(3)))] - #[case::list(Node::List(List{ marker: None,start: None, spread: false, index: 2, level: 1, checked: Some(true), ordered: true, values: Vec::new(), position: None}), "index", Some(AttrValue::Integer(2)))] - #[case::list(Node::List(List{ marker: None,start: None, spread: false, index: 2, level: 1, checked: Some(true), ordered: true, values: Vec::new(), position: None}), "level", Some(AttrValue::Integer(1)))] - #[case::list(Node::List(List{ marker: None,start: None, spread: false, index: 2, level: 1, checked: Some(true), ordered: true, values: Vec::new(), position: None}), "ordered", Some(AttrValue::Boolean(true)))] - #[case::list(Node::List(List{ marker: None,start: None, spread: false, index: 2, level: 1, checked: Some(true), ordered: true, values: Vec::new(), position: None}), attr_keys::CHECKED, Some(AttrValue::Boolean(true)))] - #[case::table_cell(Node::TableCell(TableCell{column: 1, row: 2, values: Vec::new(), position: None}), "column", Some(AttrValue::Integer(1)))] - #[case::table_cell(Node::TableCell(TableCell{column: 1, row: 2, values: Vec::new(), position: None}), "row", Some(AttrValue::Integer(2)))] - #[case::table_align(Node::TableAlign(TableAlign{align: vec![TableAlignKind::Left, TableAlignKind::Right], position: None}), "align", Some(AttrValue::String(":---,---:".to_string())))] - #[case::mdx_flow_expression(Node::MdxFlowExpression(MdxFlowExpression{value: "expr".into(), position: None}), attr_keys::VALUE, Some(AttrValue::String("expr".to_string())))] - #[case::mdx_flow_expression(Node::MdxTextExpression(MdxTextExpression{value: "expr".into(), position: None}), attr_keys::VALUE, Some(AttrValue::String("expr".to_string())))] - #[case::mdx_js_esm(Node::MdxJsEsm(MdxJsEsm{value: "esm".into(), position: None}), attr_keys::VALUE, Some(AttrValue::String("esm".to_string())))] - #[case::mdx_jsx_flow_element(Node::MdxJsxFlowElement(MdxJsxFlowElement{name: Some("div".to_string()), attributes: Vec::new(), children: Vec::new(), position: None}), attr_keys::NAME, Some(AttrValue::String("div".to_string())))] - #[case::mdx_jsx_flow_element(Node::MdxJsxTextElement(MdxJsxTextElement{name: Some("span".into()), attributes: Vec::new(), children: Vec::new(), position: None}), attr_keys::NAME, Some(AttrValue::String("span".to_string())))] - #[case::break_(Node::Break(Break{position: None}), attr_keys::VALUE, None)] - #[case::horizontal_rule(Node::HorizontalRule(HorizontalRule{ marker: None,position: None}), attr_keys::VALUE, None)] - #[case::fragment(Node::Fragment(Fragment{values: Vec::new()}), attr_keys::VALUE, Some(AttrValue::String("".to_string())))] - #[case::heading(Node::Heading(Heading{depth: 1, values: vec![Node::Text(Text{value: "heading text".to_string(), position: None})], position: None}), attr_keys::VALUE, Some(AttrValue::String("heading text".to_string())))] - #[case::heading(Node::Heading(Heading{depth: 2, values: vec![], position: None}), attr_keys::VALUE, Some(AttrValue::String("".to_string())))] - #[case::heading(Node::Heading(Heading{depth: 3, values: vec![ - Node::Text(Text{value: "first".to_string(), position: None}), - Node::Text(Text{value: "second".to_string(), position: None}), - ], position: None}), attr_keys::VALUE, Some(AttrValue::String("firstsecond".to_string())))] - #[case( - Node::List(List { marker: None, - index: 0, - level: 1, - checked: None, - ordered: false, - start: None, spread: false, - values: vec![ - Node::Text(Text { value: "item1".to_string(), position: None }), - Node::Text(Text { value: "item2".to_string(), position: None }), - ], - position: None, - }), - attr_keys::VALUE, - Some(AttrValue::String("item1item2".to_string())) - )] - #[case( - Node::TableCell(TableCell { - column: 1, - row: 2, - values: vec![Node::Text(Text { - value: "cell_value".to_string(), - position: None, - })], - position: None, - }), - attr_keys::VALUE, - Some(AttrValue::String("cell_value".to_string())) - )] - #[case::footnote( - Node::Footnote(Footnote { - ident: "id".to_string(), - values: vec![Node::Text(Text { - value: "footnote value".to_string(), - position: None, - })], - position: None, - }), - attr_keys::VALUE, - Some(AttrValue::String("footnote value".to_string())) - )] - #[case::link( - Node::Link(Link { - url: Url::new("https://example.com".to_string()), - title: Some(Title::new("Example".to_string())), - values: vec![Node::Text(Text { - value: "link text".to_string(), - position: None, - })], - position: None, - }), - attr_keys::VALUE, - Some(AttrValue::String("link text".to_string())) - )] - #[case::empty(Node::Empty, attr_keys::VALUE, None)] - #[case::heading( - Node::Heading(Heading { - depth: 1, - values: vec![ - Node::Text(Text { - value: "child1".to_string(), - position: None, - }), - Node::Text(Text { - value: "child2".to_string(), - position: None, - }), - ], - position: None, - }), - attr_keys::CHILDREN, - Some(AttrValue::Array(vec![ - Node::Text(Text { - value: "child1".to_string(), - position: None, - }), - Node::Text(Text { - value: "child2".to_string(), - position: None, - }), - ])) - )] - #[case::list( - Node::List(List { marker: None, - index: 0, - level: 1, - checked: None, - ordered: false, - start: None, spread: false, - values: vec![ - Node::Text(Text { - value: "item1".to_string(), - position: None, - }), - ], - position: None, - }), - attr_keys::CHILDREN, - Some(AttrValue::Array(vec![ - Node::Text(Text { - value: "item1".to_string(), - position: None, - }), - ])) - )] - #[case::blockquote( - Node::Blockquote(Blockquote { - values: vec![ - Node::Text(Text { - value: "quote".to_string(), - position: None, - }), - ], - position: None, - }), - attr_keys::VALUES, - Some(AttrValue::Array(vec![ - Node::Text(Text { - value: "quote".to_string(), - position: None, - }), - ])) - )] - #[case::link( - Node::Link(Link { - url: Url::new(attr_keys::URL.to_string()), - title: None, - values: vec![ - Node::Text(Text { - value: "link".to_string(), - position: None, - }), - ], - position: None, - }), - attr_keys::VALUES, - Some(AttrValue::Array(vec![ - Node::Text(Text { - value: "link".to_string(), - position: None, - }), - ])) - )] - #[case::table_cell( - Node::TableCell(TableCell { - column: 0, - row: 0, - values: vec![ - Node::Text(Text { - value: "cell".to_string(), - position: None, - }), - ], - position: None, - }), - attr_keys::CHILDREN, - Some(AttrValue::Array(vec![ - Node::Text(Text { - value: "cell".to_string(), - position: None, - }), - ])) - )] - #[case::strong( - Node::Strong(Strong { - values: vec![ - Node::Text(Text { - value: "bold".to_string(), - position: None, - }), - ], - position: None, - }), - attr_keys::CHILDREN, - Some(AttrValue::Array(vec![ - Node::Text(Text { - value: "bold".to_string(), - position: None, - }), - ])) - )] - #[case::em( - Node::Emphasis(Emphasis { - values: vec![], - position: None, - }), - attr_keys::CHILDREN, - Some(AttrValue::Array(vec![])) - )] - fn test_attr(#[case] node: Node, #[case] attr: &str, #[case] expected: Option<AttrValue>) { - assert_eq!(node.attr(attr), expected); - } - - #[rstest] - #[case::heading_with_position( - Node::Heading(Heading { - depth: 1, - values: vec![], - position: Some(Position { - start: Point { line: 3, column: 1 }, - end: Point { line: 5, column: 4 }, - }), - }), - attr_keys::LINE, - Some(AttrValue::Integer(3)) - )] - #[case::code_end_line( - Node::Code(Code { - value: "x".to_string(), - lang: None, - meta: None, - fence: true, - position: Some(Position { - start: Point { line: 3, column: 1 }, - end: Point { line: 5, column: 4 }, - }), - }), - attr_keys::END_LINE, - Some(AttrValue::Integer(5)) - )] - #[case::synthetic_node_has_no_line( - Node::Heading(Heading { - depth: 1, - values: vec![], - position: None, - }), - attr_keys::LINE, - None - )] - #[case::synthetic_node_has_no_end_line( - Node::Heading(Heading { - depth: 1, - values: vec![], - position: None, - }), - attr_keys::END_LINE, - None - )] - fn test_line_attr(#[case] node: Node, #[case] attr: &str, #[case] expected: Option<AttrValue>) { - assert_eq!(node.attr(attr), expected); - } - - #[rstest] - #[case::heading( - Node::Heading(Heading{depth: 1, values: vec![], position: None}), - vec![Node::Text(Text{value: "child".to_string(), position: None})], - Node::Heading(Heading{depth: 1, values: vec![Node::Text(Text{value: "child".to_string(), position: None})], position: None}) - )] - #[case::list( - Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: vec![], position: None}), - vec![Node::Text(Text{value: "item".to_string(), position: None})], - Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: vec![Node::Text(Text{value: "item".to_string(), position: None})], position: None}) - )] - #[case::blockquote( - Node::Blockquote(Blockquote{values: vec![], position: None}), - vec![Node::Text(Text{value: "quote".to_string(), position: None})], - Node::Blockquote(Blockquote{values: vec![Node::Text(Text{value: "quote".to_string(), position: None})], position: None}) - )] - #[case::link( - Node::Link(Link{url: Url::new(attr_keys::URL.to_string()), title: None, values: vec![], position: None}), - vec![Node::Text(Text{value: "link".to_string(), position: None})], - Node::Link(Link{url: Url::new(attr_keys::URL.to_string()), title: None, values: vec![Node::Text(Text{value: "link".to_string(), position: None})], position: None}) - )] - #[case::footnote( - Node::Footnote(Footnote{ident: "1".to_string(), values: vec![], position: None}), - vec![Node::Text(Text{value: "note".to_string(), position: None})], - Node::Footnote(Footnote{ident: "1".to_string(), values: vec![Node::Text(Text{value: "note".to_string(), position: None})], position: None}) - )] - #[case::table_cell( - Node::TableCell(TableCell{column: 0, row: 0, values: vec![], position: None}), - vec![Node::Text(Text{value: "cell".to_string(), position: None})], - Node::TableCell(TableCell{column: 0, row: 0, values: vec![Node::Text(Text{value: "cell".to_string(), position: None})], position: None}) - )] - #[case::table_row( - Node::TableRow(TableRow{values: vec![], position: None}), - vec![Node::Text(Text{value: "row".to_string(), position: None})], - Node::TableRow(TableRow{values: vec![Node::Text(Text{value: "row".to_string(), position: None})], position: None}) - )] - #[case::strong( - Node::Strong(Strong{values: vec![], position: None}), - vec![Node::Text(Text{value: "bold".to_string(), position: None})], - Node::Strong(Strong{values: vec![Node::Text(Text{value: "bold".to_string(), position: None})], position: None}) - )] - #[case::delete( - Node::Delete(Delete{values: vec![], position: None}), - vec![Node::Text(Text{value: "del".to_string(), position: None})], - Node::Delete(Delete{values: vec![Node::Text(Text{value: "del".to_string(), position: None})], position: None}) - )] - #[case::emphasis( - Node::Emphasis(Emphasis{values: vec![], position: None}), - vec![Node::Text(Text{value: "em".to_string(), position: None})], - Node::Emphasis(Emphasis{values: vec![Node::Text(Text{value: "em".to_string(), position: None})], position: None}) - )] - #[case::fragment( - Node::Fragment(Fragment{values: vec![]}), - vec![Node::Text(Text{value: "frag".to_string(), position: None})], - Node::Fragment(Fragment{values: vec![Node::Text(Text{value: "frag".to_string(), position: None})]}) - )] - #[case::mdx_jsx_flow_element( - Node::MdxJsxFlowElement(MdxJsxFlowElement{name: Some("div".to_string()), attributes: Vec::new(), children: vec![], position: None}), - vec![Node::Text(Text{value: "mdx".to_string(), position: None})], - Node::MdxJsxFlowElement(MdxJsxFlowElement{name: Some("div".to_string()), attributes: Vec::new(), children: vec![Node::Text(Text{value: "mdx".to_string(), position: None})], position: None}) - )] - #[case::mdx_jsx_text_element( - Node::MdxJsxTextElement(MdxJsxTextElement{name: Some("span".into()), attributes: Vec::new(), children: vec![], position: None}), - vec![Node::Text(Text{value: "mdx".to_string(), position: None})], - Node::MdxJsxTextElement(MdxJsxTextElement{name: Some("span".into()), attributes: Vec::new(), children: vec![Node::Text(Text{value: "mdx".to_string(), position: None})], position: None}) - )] - #[case::leaf_node_is_noop( - Node::Text(Text{value: "leaf".to_string(), position: None}), - vec![Node::Text(Text{value: "ignored".to_string(), position: None})], - Node::Text(Text{value: "leaf".to_string(), position: None}) - )] - fn test_set_children(#[case] mut node: Node, #[case] children: Vec<Node>, #[case] expected: Node) { - node.set_children(children.clone()); - assert_eq!(node, expected); - assert_eq!( - node.children(), - if matches!(expected, Node::Text(_)) { - Vec::new() - } else { - children - } - ); - } - - #[rstest] - #[case( - Node::Text(Text{value: "old".to_string(), position: None}), - attr_keys::VALUE, - "new", - Node::Text(Text{value: "new".to_string(), position: None}) - )] - #[case( - Node::Code(Code{value: "old".to_string(), lang: Some("rust".to_string()), fence: true, meta: None, position: None}), - attr_keys::VALUE, - "new_code", - Node::Code(Code{value: "new_code".to_string(), lang: Some("rust".to_string()), fence: true, meta: None, position: None}) - )] - #[case( - Node::Code(Code{value: "code".to_string(), lang: Some("rust".to_string()), fence: true, meta: None, position: None}), - attr_keys::LANG, - "python", - Node::Code(Code{value: "code".to_string(), lang: Some("python".to_string()), fence: true, meta: None, position: None}) - )] - #[case( - Node::Code(Code{value: "code".to_string(), lang: None, fence: false, meta: None, position: None}), - attr_keys::FENCE, - "true", - Node::Code(Code{value: "code".to_string(), lang: None, fence: true, meta: None, position: None}) - )] - #[case( - Node::Image(Image{alt: attr_keys::ALT.to_string(), url: attr_keys::URL.to_string(), title: None, position: None}), - attr_keys::ALT, - "new_alt", - Node::Image(Image{alt: "new_alt".to_string(), url: attr_keys::URL.to_string(), title: None, position: None}) - )] - #[case( - Node::Image(Image{alt: attr_keys::ALT.to_string(), url: attr_keys::URL.to_string(), title: None, position: None}), - attr_keys::URL, - "new_url", - Node::Image(Image{alt: attr_keys::ALT.to_string(), url: "new_url".to_string(), title: None, position: None}) - )] - #[case( - Node::Image(Image{alt: attr_keys::ALT.to_string(), url: attr_keys::URL.to_string(), title: Some(attr_keys::TITLE.to_string()), position: None}), - attr_keys::TITLE, - "new_title", - Node::Image(Image{alt: attr_keys::ALT.to_string(), url: attr_keys::URL.to_string(), title: Some("new_title".to_string()), position: None}) - )] - #[case( - Node::Heading(Heading{depth: 2, values: vec![], position: None}), - "depth", - "3", - Node::Heading(Heading{depth: 3, values: vec![], position: None}) - )] - #[case( - Node::List(List{ marker: None,start: None, spread: false, index: 1, level: 2, checked: Some(true), ordered: false, values: vec![], position: None}), - attr_keys::CHECKED, - "false", - Node::List(List{ marker: None,start: None, spread: false, index: 1, level: 2, checked: Some(false), ordered: false, values: vec![], position: None}) - )] - #[case( - Node::List(List{ marker: None,start: None, spread: false, index: 1, level: 2, checked: Some(true), ordered: false, values: vec![], position: None}), - "ordered", - "true", - Node::List(List{ marker: None,start: None, spread: false, index: 1, level: 2, checked: Some(true), ordered: true, values: vec![], position: None}) - )] - #[case( - Node::TableCell(TableCell{column: 1, row: 2, values: vec![], position: None}), - "column", - "3", - Node::TableCell(TableCell{column: 3, row: 2, values: vec![], position: None}) - )] - #[case( - Node::TableCell(TableCell{column: 1, row: 2, values: vec![], position: None}), - "row", - "5", - Node::TableCell(TableCell{column: 1, row: 5, values: vec![], position: None}) - )] - #[case( - Node::Definition(Definition{ident: "id".to_string(), url: Url::new(attr_keys::URL.to_string()), title: None, label: None, position: None}), - attr_keys::IDENT, - "new_id", - Node::Definition(Definition{ident: "new_id".to_string(), url: Url::new(attr_keys::URL.to_string()), title: None, label: None, position: None}) - )] - #[case( - Node::Definition(Definition{ident: "id".to_string(), url: Url::new(attr_keys::URL.to_string()), title: None, label: None, position: None}), - attr_keys::URL, - "new_url", - Node::Definition(Definition{ident: "id".to_string(), url: Url::new("new_url".to_string()), title: None, label: None, position: None}) - )] - #[case( - Node::Definition(Definition{ident: "id".to_string(), url: Url::new(attr_keys::URL.to_string()), title: None, label: None, position: None}), - attr_keys::LABEL, - "new_label", - Node::Definition(Definition{ident: "id".to_string(), url: Url::new(attr_keys::URL.to_string()), title: None, label: Some("new_label".to_string()), position: None}) - )] - #[case( - Node::Definition(Definition{ident: "id".to_string(), url: Url::new(attr_keys::URL.to_string()), title: None, label: None, position: None}), - attr_keys::TITLE, - "new_title", - Node::Definition(Definition{ident: "id".to_string(), url: Url::new(attr_keys::URL.to_string()), title: Some(Title::new("new_title".to_string())), label: None, position: None}) - )] - #[case( - Node::ImageRef(ImageRef{alt: attr_keys::ALT.to_string(), ident: "id".to_string(), label: Some(attr_keys::LABEL.to_string()), position: None}), - attr_keys::ALT, - "new_alt", - Node::ImageRef(ImageRef{alt: "new_alt".to_string(), ident: "id".to_string(), label: Some(attr_keys::LABEL.to_string()), position: None}) - )] - #[case( - Node::ImageRef(ImageRef{alt: attr_keys::ALT.to_string(), ident: "id".to_string(), label: Some(attr_keys::LABEL.to_string()), position: None}), - attr_keys::IDENT, - "new_id", - Node::ImageRef(ImageRef{alt: attr_keys::ALT.to_string(), ident: "new_id".to_string(), label: Some(attr_keys::LABEL.to_string()), position: None}) - )] - #[case( - Node::ImageRef(ImageRef{alt: attr_keys::ALT.to_string(), ident: "id".to_string(), label: Some(attr_keys::LABEL.to_string()), position: None}), - attr_keys::LABEL, - "new_label", - Node::ImageRef(ImageRef{alt: attr_keys::ALT.to_string(), ident: "id".to_string(), label: Some("new_label".to_string()), position: None}) - )] - #[case( - Node::ImageRef(ImageRef{alt: attr_keys::ALT.to_string(), ident: "id".to_string(), label: None, position: None}), - attr_keys::LABEL, - "new_label", - Node::ImageRef(ImageRef{alt: attr_keys::ALT.to_string(), ident: "id".to_string(), label: Some("new_label".to_string()), position: None}) - )] - #[case( - Node::LinkRef(LinkRef{ident: "id".to_string(), values: vec![], label: Some(attr_keys::LABEL.to_string()), position: None}), - attr_keys::IDENT, - "new_id", - Node::LinkRef(LinkRef{ident: "new_id".to_string(), values: vec![], label: Some(attr_keys::LABEL.to_string()), position: None}) - )] - #[case( - Node::LinkRef(LinkRef{ident: "id".to_string(), values: vec![], label: Some(attr_keys::LABEL.to_string()), position: None}), - attr_keys::LABEL, - "new_label", - Node::LinkRef(LinkRef{ident: "id".to_string(), values: vec![], label: Some("new_label".to_string()), position: None}) - )] - #[case( - Node::LinkRef(LinkRef{ident: "id".to_string(), values: vec![], label: None, position: None}), - attr_keys::LABEL, - "new_label", - Node::LinkRef(LinkRef{ident: "id".to_string(), values: vec![], label: Some("new_label".to_string()), position: None}) - )] - #[case( - Node::LinkRef(LinkRef{ident: "id".to_string(), values: vec![], label: Some(attr_keys::LABEL.to_string()), position: None}), - "unknown", - "ignored", - Node::LinkRef(LinkRef{ident: "id".to_string(), values: vec![], label: Some(attr_keys::LABEL.to_string()), position: None}) - )] - #[case( - Node::FootnoteRef(FootnoteRef{ident: "id".to_string(), label: Some(attr_keys::LABEL.to_string()), position: None}), - attr_keys::IDENT, - "new_id", - Node::FootnoteRef(FootnoteRef{ident: "new_id".to_string(), label: Some(attr_keys::LABEL.to_string()), position: None}) - )] - #[case( - Node::FootnoteRef(FootnoteRef{ident: "id".to_string(), label: Some(attr_keys::LABEL.to_string()), position: None}), - attr_keys::LABEL, - "new_label", - Node::FootnoteRef(FootnoteRef{ident: "id".to_string(), label: Some("new_label".to_string()), position: None}) - )] - #[case( - Node::FootnoteRef(FootnoteRef{ident: "id".to_string(), label: None, position: None}), - attr_keys::LABEL, - "new_label", - Node::FootnoteRef(FootnoteRef{ident: "id".to_string(), label: Some("new_label".to_string()), position: None}) - )] - #[case( - Node::FootnoteRef(FootnoteRef{ident: "id".to_string(), label: Some(attr_keys::LABEL.to_string()), position: None}), - "unknown", - "ignored", - Node::FootnoteRef(FootnoteRef{ident: "id".to_string(), label: Some(attr_keys::LABEL.to_string()), position: None}) - )] - #[case(Node::Empty, attr_keys::VALUE, "ignored", Node::Empty)] - #[case( - Node::TableAlign(TableAlign{align: vec![TableAlignKind::Left, TableAlignKind::Right], position: None}), - "align", - "---,:---:", - Node::TableAlign(TableAlign{align: vec![TableAlignKind::None, TableAlignKind::Center], position: None}) - )] - #[case( - Node::TableAlign(TableAlign{align: vec![], position: None}), - "align", - ":---,---:", - Node::TableAlign(TableAlign{align: vec![TableAlignKind::Left, TableAlignKind::Right], position: None}) - )] - #[case( - Node::TableAlign(TableAlign{align: vec![TableAlignKind::Left], position: None}), - "unknown", - "ignored", - Node::TableAlign(TableAlign{align: vec![TableAlignKind::Left], position: None}) - )] - #[case( - Node::MdxFlowExpression(MdxFlowExpression{value: "old".into(), position: None}), - attr_keys::VALUE, - "new_expr", - Node::MdxFlowExpression(MdxFlowExpression{value: "new_expr".into(), position: None}) - )] - #[case( - Node::MdxFlowExpression(MdxFlowExpression{value: "expr".into(), position: None}), - "unknown", - "ignored", - Node::MdxFlowExpression(MdxFlowExpression{value: "expr".into(), position: None}) - )] - #[case( - Node::MdxTextExpression(MdxTextExpression{value: "old".into(), position: None}), - attr_keys::VALUE, - "new_expr", - Node::MdxTextExpression(MdxTextExpression{value: "new_expr".into(), position: None}) - )] - #[case( - Node::MdxTextExpression(MdxTextExpression{value: "expr".into(), position: None}), - "unknown", - "ignored", - Node::MdxTextExpression(MdxTextExpression{value: "expr".into(), position: None}) - )] - #[case( - Node::MdxJsEsm(MdxJsEsm{value: "import x".into(), position: None}), - attr_keys::VALUE, - "import y", - Node::MdxJsEsm(MdxJsEsm{value: "import y".into(), position: None}) - )] - #[case( - Node::MdxJsEsm(MdxJsEsm{value: "import x".into(), position: None}), - "unknown", - "ignored", - Node::MdxJsEsm(MdxJsEsm{value: "import x".into(), position: None}) - )] - #[case( - Node::MdxJsxFlowElement(MdxJsxFlowElement{name: Some("div".to_string()), attributes: Vec::new(), children: Vec::new(), position: None}), - attr_keys::NAME, - "section", - Node::MdxJsxFlowElement(MdxJsxFlowElement{name: Some("section".to_string()), attributes: Vec::new(), children: Vec::new(), position: None}) - )] - #[case( - Node::MdxJsxFlowElement(MdxJsxFlowElement{name: None, attributes: Vec::new(), children: Vec::new(), position: None}), - attr_keys::NAME, - "main", - Node::MdxJsxFlowElement(MdxJsxFlowElement{name: Some("main".to_string()), attributes: Vec::new(), children: Vec::new(), position: None}) - )] - #[case( - Node::MdxJsxFlowElement(MdxJsxFlowElement{name: Some("div".to_string()), attributes: Vec::new(), children: Vec::new(), position: None}), - "unknown", - "ignored", - Node::MdxJsxFlowElement(MdxJsxFlowElement{name: Some("div".to_string()), attributes: Vec::new(), children: Vec::new(), position: None}) - )] - #[case( - Node::MdxJsxTextElement(MdxJsxTextElement{name: Some("span".into()), attributes: Vec::new(), children: Vec::new(), position: None}), - attr_keys::NAME, - "b", - Node::MdxJsxTextElement(MdxJsxTextElement{name: Some("b".into()), attributes: Vec::new(), children: Vec::new(), position: None}) - )] - #[case( - Node::MdxJsxTextElement(MdxJsxTextElement{name: None, attributes: Vec::new(), children: Vec::new(), position: None}), - attr_keys::NAME, - "i", - Node::MdxJsxTextElement(MdxJsxTextElement{name: Some("i".into()), attributes: Vec::new(), children: Vec::new(), position: None}) - )] - #[case( - Node::MdxJsxTextElement(MdxJsxTextElement{name: Some("span".into()), attributes: Vec::new(), children: Vec::new(), position: None}), - "unknown", - "ignored", - Node::MdxJsxTextElement(MdxJsxTextElement{name: Some("span".into()), attributes: Vec::new(), children: Vec::new(), position: None}) - )] - fn test_set_attr(#[case] mut node: Node, #[case] attr: &str, #[case] value: &str, #[case] expected: Node) { - node.set_attr(attr, value); - assert_eq!(node, expected); - } - - #[rstest] - #[case(AttrValue::String("test".to_string()), AttrValue::String("test".to_string()), true)] - #[case(AttrValue::String("test".to_string()), AttrValue::String("other".to_string()), false)] - #[case(AttrValue::Integer(42), AttrValue::Integer(42), true)] - #[case(AttrValue::Integer(42), AttrValue::Integer(0), false)] - #[case(AttrValue::Boolean(true), AttrValue::Boolean(true), true)] - #[case(AttrValue::Boolean(true), AttrValue::Boolean(false), false)] - #[case(AttrValue::String("42".to_string()), AttrValue::Integer(42), false)] - #[case(AttrValue::Boolean(false), AttrValue::Integer(0), false)] - fn test_attr_value_eq(#[case] a: AttrValue, #[case] b: AttrValue, #[case] expected: bool) { - assert_eq!(a == b, expected); - } - - #[rstest] - #[case(AttrValue::String("test".to_string()), "test")] - #[case(AttrValue::Integer(42), "42")] - #[case(AttrValue::Boolean(true), "true")] - fn test_attr_value_as_str(#[case] value: AttrValue, #[case] expected: &str) { - assert_eq!(&value.as_string(), expected); - } - - #[rstest] - #[case(AttrValue::Integer(42), Some(42))] - #[case(AttrValue::String("42".to_string()), Some(42))] - #[case(AttrValue::Boolean(false), Some(0))] - fn test_attr_value_as_i64(#[case] value: AttrValue, #[case] expected: Option<i64>) { - assert_eq!(value.as_i64(), expected); - } - - // --- Callout tests --- - - #[cfg(feature = "callout")] - #[rstest] - // basic render - #[case( - Node::Callout(Callout { fold: None, kind: "NOTE".to_string(), title: None, - values: vec![Node::Text(Text { value: "content".to_string(), position: None })], - position: None }), - "> [!NOTE]\n> content" - )] - // with title - #[case( - Node::Callout(Callout { fold: None, kind: "WARNING".to_string(), title: Some("Heads up".to_string()), - values: vec![Node::Text(Text { value: "watch out".to_string(), position: None })], - position: None }), - "> [!WARNING] Heads up\n> watch out" - )] - // empty body - #[case( - Node::Callout(Callout { fold: None, kind: "TIP".to_string(), title: None, values: vec![], position: None }), - "> [!TIP]" - )] - // multiline body: single Text with embedded '\n' — each line prefixed with "> " - #[case( - Node::Callout(Callout { fold: None, kind: "INFO".to_string(), title: None, - values: vec![Node::Text(Text { value: "line one\nline two".to_string(), position: None })], - position: None }), - "> [!INFO]\n> line one\n> line two" - )] - // two separate position-less Text values are concatenated inline (no implicit newline) - #[case( - Node::Callout(Callout { fold: None, kind: "INFO".to_string(), title: None, - values: vec![ - Node::Text(Text { value: "part a".to_string(), position: None }), - Node::Text(Text { value: "part b".to_string(), position: None }), - ], - position: None }), - "> [!INFO]\n> part apart b" - )] - fn test_callout_render(#[case] node: Node, #[case] expected: &str) { - assert_eq!(node.to_string_with(&RenderOptions::default()), expected); - } - - #[cfg(feature = "callout")] - #[rstest] - #[case(Node::Callout(Callout { fold: None, kind: "NOTE".to_string(), title: None, values: vec![], position: None }), "kind", Some(AttrValue::String("NOTE".to_string())))] - #[case(Node::Callout(Callout { fold: None, kind: "WARNING".to_string(), title: Some("Title".to_string()), values: vec![], position: None }), "title", Some(AttrValue::String("Title".to_string())))] - #[case(Node::Callout(Callout { fold: None, kind: "TIP".to_string(), title: None, values: vec![], position: None }), "title", None)] - #[case(Node::Callout(Callout { fold: None, kind: "NOTE".to_string(), title: None, values: vec![Node::Text(Text { value: "body".to_string(), position: None })], position: None }), "value", Some(AttrValue::String("body".to_string())))] - #[case(Node::Callout(Callout { fold: None, kind: "NOTE".to_string(), title: None, values: vec![Node::Text(Text { value: "body".to_string(), position: None })], position: None }), "children", Some(AttrValue::Array(vec![Node::Text(Text { value: "body".to_string(), position: None })])))] - fn test_callout_attr(#[case] node: Node, #[case] attr: &str, #[case] expected: Option<AttrValue>) { - assert_eq!(node.attr(attr), expected); - } - - #[cfg(feature = "callout")] - #[rstest] - #[case( - Node::Callout(Callout { fold: None, kind: "NOTE".to_string(), title: None, values: vec![], position: None }), - "kind", "WARNING", - Node::Callout(Callout { fold: None, kind: "WARNING".to_string(), title: None, values: vec![], position: None }) - )] - // set_attr stores kind as-is without case conversion - #[case( - Node::Callout(Callout { fold: None, kind: "NOTE".to_string(), title: None, values: vec![], position: None }), - "kind", "tip", - Node::Callout(Callout { fold: None, kind: "tip".to_string(), title: None, values: vec![], position: None }) - )] - #[case( - Node::Callout(Callout { fold: None, kind: "NOTE".to_string(), title: None, values: vec![], position: None }), - "title", "My title", - Node::Callout(Callout { fold: None, kind: "NOTE".to_string(), title: Some("My title".to_string()), values: vec![], position: None }) - )] - // empty string clears title - #[case( - Node::Callout(Callout { fold: None, kind: "NOTE".to_string(), title: Some("old".to_string()), values: vec![], position: None }), - "title", "", - Node::Callout(Callout { fold: None, kind: "NOTE".to_string(), title: None, values: vec![], position: None }) - )] - fn test_callout_set_attr(#[case] mut node: Node, #[case] attr: &str, #[case] value: &str, #[case] expected: Node) { - node.set_attr(attr, value); - assert_eq!(node, expected); - } - - #[cfg(feature = "callout")] - #[rstest] - // with_value changes first body node - #[case( - Node::Callout(Callout { fold: None, kind: "NOTE".to_string(), title: None, - values: vec![Node::Text(Text { value: "old".to_string(), position: None })], - position: None }), - "new", - Node::Callout(Callout { fold: None, kind: "NOTE".to_string(), title: None, - values: vec![Node::Text(Text { value: "new".to_string(), position: None })], - position: None }) - )] - fn test_callout_with_value(#[case] node: Node, #[case] value: &str, #[case] expected: Node) { - assert_eq!(node.clone().with_value(value), expected); - assert_eq!(node.into_with_value(value), expected); - } - - #[cfg(feature = "callout")] - #[rstest] - #[case(Node::Callout(Callout { fold: None, kind: "NOTE".to_string(), title: None, values: vec![], position: None }), "callout")] - fn test_callout_name(#[case] node: Node, #[case] expected: &str) { - assert_eq!(node.name(), expected); - } - - #[cfg(feature = "callout")] - #[rstest] - #[case( - Node::Callout(Callout { fold: None, kind: "NOTE".to_string(), title: None, - values: vec![Node::Text(Text { value: "body".to_string(), position: None })], - position: None }), - "body" - )] - fn test_callout_value(#[case] node: Node, #[case] expected: &str) { - assert_eq!(node.value(), expected); - } - - #[cfg(feature = "callout")] - #[rstest] - #[case(Node::Callout(Callout { fold: None, kind: "NOTE".to_string(), title: None, values: vec![], position: None }), true)] - #[case(Node::Blockquote(Blockquote { values: vec![], position: None }), false)] - #[case(Node::Text(Text { value: "test".to_string(), position: None }), false)] - fn test_is_callout(#[case] node: Node, #[case] expected: bool) { - assert_eq!(node.is_callout(), expected); - } - - #[cfg(feature = "callout")] - #[test] - fn test_callout_end_to_end() { - use crate::Markdown; - // plain blockquote stays as Blockquote - let plain = Markdown::from_markdown_str("> just a quote").unwrap(); - assert!(plain.nodes[0].is_blockquote()); - - // [!NOTE] is parsed as Callout - let note = Markdown::from_markdown_str("> [!NOTE]\n> body").unwrap(); - assert!(note.nodes[0].is_callout()); - assert_eq!(note.nodes[0].attr("kind"), Some(AttrValue::String("NOTE".to_string()))); - - // [!WARNING] with title - let warn = Markdown::from_markdown_str("> [!WARNING] Watch out\n> content").unwrap(); - assert!(warn.nodes[0].is_callout()); - assert_eq!( - warn.nodes[0].attr("title"), - Some(AttrValue::String("Watch out".to_string())) - ); - } - - // --- Embed tests --- - - #[cfg(feature = "embed")] - #[rstest] - // plain embed - #[case("![[target]]", vec![Node::Embed(Embed { target: "target".to_string(), display: None, position: None })])] - // embed with display hint (image size) - #[case("![[image.png|400]]", vec![Node::Embed(Embed { target: "image.png".to_string(), display: Some("400".to_string()), position: None })])] - // embed with section reference in target - #[case("![[note#Heading]]", vec![Node::Embed(Embed { target: "note#Heading".to_string(), display: None, position: None })])] - // embed at start of text - #[case("![[note]] rest", vec![ - Node::Embed(Embed { target: "note".to_string(), display: None, position: None }), - Node::Text(Text { value: " rest".to_string(), position: None }), - ])] - // embed in middle - #[case("before ![[note]] after", vec![ - Node::Text(Text { value: "before ".to_string(), position: None }), - Node::Embed(Embed { target: "note".to_string(), display: None, position: None }), - Node::Text(Text { value: " after".to_string(), position: None }), - ])] - // multiple embeds - #[case("![[a]] and ![[b]]", vec![ - Node::Embed(Embed { target: "a".to_string(), display: None, position: None }), - Node::Text(Text { value: " and ".to_string(), position: None }), - Node::Embed(Embed { target: "b".to_string(), display: None, position: None }), - ])] - // multibyte target - #[case("![[ノート]]", vec![Node::Embed(Embed { target: "ノート".to_string(), display: None, position: None })])] - // no embed — plain text unchanged - #[case("plain text", vec![Node::Text(Text { value: "plain text".to_string(), position: None })])] - // unclosed embed — treated as plain text - #[case("![[unclosed", vec![Node::Text(Text { value: "![[unclosed".to_string(), position: None })])] - // lone ! before non-embed — treated as plain text - #[case("! not an embed", vec![Node::Text(Text { value: "! not an embed".to_string(), position: None })])] - fn test_parse_embeds_in_text(#[case] input: &str, #[case] expected: Vec<Node>) { - let mut result = Vec::new(); - Node::parse_embeds_into(input, None, &mut result); - assert_eq!(result, expected); - } - - #[cfg(feature = "embed")] - #[rstest] - // plain embed renders without display - #[case( - Node::Embed(Embed { target: "note.md".to_string(), display: None, position: None }), - "![[note.md]]" - )] - // embed with display hint - #[case( - Node::Embed(Embed { target: "image.png".to_string(), display: Some("400".to_string()), position: None }), - "![[image.png|400]]" - )] - // embed with section reference - #[case( - Node::Embed(Embed { target: "note#Intro".to_string(), display: None, position: None }), - "![[note#Intro]]" - )] - fn test_embed_render(#[case] node: Node, #[case] expected: &str) { - assert_eq!(node.to_string_with(&RenderOptions::default()), expected); - } - - #[cfg(feature = "embed")] - #[rstest] - #[case(Node::Embed(Embed { target: "note".to_string(), display: None, position: None }), "url", Some(AttrValue::String("note".to_string())))] - // value returns target when no display - #[case(Node::Embed(Embed { target: "note".to_string(), display: None, position: None }), "value", Some(AttrValue::String("note".to_string())))] - // url always returns target regardless of display - #[case(Node::Embed(Embed { target: "note".to_string(), display: Some("400".to_string()), position: None }), "url", Some(AttrValue::String("note".to_string())))] - // value returns display when present - #[case(Node::Embed(Embed { target: "note".to_string(), display: Some("400".to_string()), position: None }), "value", Some(AttrValue::String("400".to_string())))] - // unknown attr returns None - #[case(Node::Embed(Embed { target: "note".to_string(), display: None, position: None }), "unknown", None)] - fn test_embed_attr(#[case] node: Node, #[case] attr: &str, #[case] expected: Option<AttrValue>) { - assert_eq!(node.attr(attr), expected); - } - - #[cfg(feature = "embed")] - #[rstest] - // set url changes target - #[case( - Node::Embed(Embed { target: "old".to_string(), display: None, position: None }), - "url", "new", - Node::Embed(Embed { target: "new".to_string(), display: None, position: None }) - )] - // set value changes display - #[case( - Node::Embed(Embed { target: "note".to_string(), display: None, position: None }), - "value", "800", - Node::Embed(Embed { target: "note".to_string(), display: Some("800".to_string()), position: None }) - )] - // empty value clears display - #[case( - Node::Embed(Embed { target: "note".to_string(), display: Some("400".to_string()), position: None }), - "value", "", - Node::Embed(Embed { target: "note".to_string(), display: None, position: None }) - )] - fn test_embed_set_attr(#[case] mut node: Node, #[case] attr: &str, #[case] value: &str, #[case] expected: Node) { - node.set_attr(attr, value); - assert_eq!(node, expected); - } - - #[cfg(feature = "embed")] - #[rstest] - // no display: with_value sets target - #[case( - Node::Embed(Embed { target: "old".to_string(), display: None, position: None }), - "new", - Node::Embed(Embed { target: "new".to_string(), display: None, position: None }) - )] - // with display: with_value sets display - #[case( - Node::Embed(Embed { target: "note".to_string(), display: Some("400".to_string()), position: None }), - "800", - Node::Embed(Embed { target: "note".to_string(), display: Some("800".to_string()), position: None }) - )] - fn test_embed_with_value(#[case] node: Node, #[case] value: &str, #[case] expected: Node) { - assert_eq!(node.clone().with_value(value), expected); - assert_eq!(node.into_with_value(value), expected); - } - - #[cfg(feature = "embed")] - #[rstest] - #[case(Node::Embed(Embed { target: "note".to_string(), display: None, position: None }), "embed")] - fn test_embed_name(#[case] node: Node, #[case] expected: &str) { - assert_eq!(node.name(), expected); - } - - #[cfg(feature = "embed")] - #[rstest] - // value returns display when present - #[case(Node::Embed(Embed { target: "note".to_string(), display: Some("400".to_string()), position: None }), "400")] - // value returns target when no display - #[case(Node::Embed(Embed { target: "note".to_string(), display: None, position: None }), "note")] - fn test_embed_value(#[case] node: Node, #[case] expected: &str) { - assert_eq!(node.value(), expected); - } - - #[cfg(feature = "embed")] - #[rstest] - #[case(Node::Embed(Embed { target: "note".to_string(), display: None, position: None }), true)] - #[case(Node::Text(Text { value: "test".to_string(), position: None }), false)] - #[case(Node::Image(Image { alt: "".to_string(), url: "img.png".to_string(), title: None, position: None }), false)] - fn test_is_embed(#[case] node: Node, #[case] expected: bool) { - assert_eq!(node.is_embed(), expected); - } - - #[cfg(feature = "embed")] - #[test] - fn test_embed_end_to_end() { - use crate::Markdown; - let md = Markdown::from_markdown_str("See ![[note.md]] for details.").unwrap(); - let embed = md.nodes.iter().find(|n| n.is_embed()); - assert!(embed.is_some()); - assert_eq!( - embed.unwrap().attr("url"), - Some(AttrValue::String("note.md".to_string())) - ); - } - - #[cfg(all(feature = "callout", feature = "wikilink"))] - #[rstest] - // callout body containing a wikilink - #[case("> [!NOTE]\n> See [[note]]", 1, "> [!NOTE]\n> See [[note]]\n")] - fn test_callout_with_wikilink(#[case] input: &str, #[case] expected_nodes: usize, #[case] expected_output: &str) { - use crate::Markdown; - let md = Markdown::from_markdown_str(input).unwrap(); - assert_eq!(md.nodes.len(), expected_nodes); - assert_eq!(md.to_string(), expected_output); - } - - #[cfg(feature = "callout")] - #[rstest] - // callout body with bold - #[case("> [!NOTE]\n> **important**", 1, "> [!NOTE]\n> **important**\n")] - // callout body with inline code - #[case("> [!NOTE]\n> Use `code`", 1, "> [!NOTE]\n> Use `code`\n")] - // multiple callouts in one document - #[case( - "> [!NOTE]\n> first\n\n> [!WARNING]\n> second", - 2, - "> [!NOTE]\n> first\n\n> [!WARNING]\n> second\n" - )] - // callout after heading - #[case("# Title\n\n> [!NOTE]\n> body", 2, "# Title\n\n> [!NOTE]\n> body\n")] - // callout body with a multi-item list — second item must not be over-indented - #[case("> [!NOTE]\n> - item 1\n> - item 2", 1, "> [!NOTE]\n> - item 1\n> - item 2\n")] - fn test_callout_combined_with_other_nodes( - #[case] input: &str, - #[case] expected_nodes: usize, - #[case] expected_output: &str, - ) { - use crate::Markdown; - let md = Markdown::from_markdown_str(input).unwrap(); - assert_eq!(md.nodes.len(), expected_nodes); - assert_eq!(md.to_string(), expected_output); - } - - #[cfg(feature = "embed")] - #[rstest] - // embed inside list item - #[case("- ![[note.md]]", 1, "- ![[note.md]]\n")] - // embed inside heading - #[case("# See ![[note]]", 1, "# See ![[note]]\n")] - // embed inside bold - #[case("**![[note]]**", 1, "**![[note]]**\n")] - fn test_embed_combined_with_other_nodes( - #[case] input: &str, - #[case] expected_nodes: usize, - #[case] expected_output: &str, - ) { - use crate::Markdown; - let md = Markdown::from_markdown_str(input).unwrap(); - assert_eq!(md.nodes.len(), expected_nodes); - assert_eq!(md.to_string(), expected_output); - } - - #[cfg(all(feature = "callout", feature = "embed"))] - #[rstest] - // wikilink inside callout body + embed as separate node - #[case("> [!NOTE]\n> [[link]]\n\n![[embed]]", 2, "> [!NOTE]\n> [[link]]\n\n![[embed]]\n")] - fn test_callout_and_embed_together( - #[case] input: &str, - #[case] expected_nodes: usize, - #[case] expected_output: &str, - ) { - use crate::Markdown; - let md = Markdown::from_markdown_str(input).unwrap(); - assert_eq!(md.nodes.len(), expected_nodes); - assert_eq!(md.to_string(), expected_output); - } - - #[cfg(all(feature = "embed", feature = "wikilink"))] - #[test] - fn test_embed_does_not_conflict_with_wikilink() { - use crate::Markdown; - // ![[embed]] must become Embed, not WikiLink with preceding "!" - let md = Markdown::from_markdown_str("![[embed]] and [[link]]").unwrap(); - let embed = md.nodes.iter().find(|n| n.is_embed()); - let wikilink = md.nodes.iter().find(|n| n.is_wikilink()); - assert!(embed.is_some(), "expected Embed node"); - assert!(wikilink.is_some(), "expected WikiLink node"); - } - - #[rstest] - #[case::heading_marker("## not a heading", "\\## not a heading")] - #[case::heading_seven_hashes_still_escaped("####### x", "\\####### x")] - #[case::emphasis_star("*not emphasis*", "\\*not emphasis\\*")] - #[case::emphasis_underscore("_not emphasis_", "\\_not emphasis\\_")] - #[case::code_span("`not code`", "\\`not code\\`")] - #[case::link_brackets("[not a link](x)", "\\[not a link\\](x)")] - #[case::list_marker("- not a list item", "\\- not a list item")] - #[case::blockquote_marker("> not a quote", "\\> not a quote")] - #[case::table_pipe("| not a table |", "\\| not a table \\|")] - #[case::strikethrough("~not strikethrough~", "\\~not strikethrough\\~")] - #[case::math("$not math$", "\\$not math\\$")] - #[case::plus_list_marker("+ not a list item", "\\+ not a list item")] - #[case::ordered_list_dot("5. not a list", "5\\. not a list")] - #[case::ordered_list_paren("5) not a list", "5\\) not a list")] - #[case::setext_h1_underline("Foo\n===", "Foo\n\\===")] - #[case::hash_not_at_line_start("a#b", "a#b")] - #[case::dash_not_at_line_start("a-b", "a-b")] - #[case::plus_not_at_line_start("a+b", "a+b")] - #[case::gt_not_at_line_start("a>b", "a>b")] - #[case::eq_not_at_line_start("a=b", "a=b")] - #[case::dot_not_after_leading_digits("a5. b", "a5. b")] - #[case::dot_after_broken_digit_run("5x3. b", "5x3. b")] - #[case::escapes_after_embedded_newline("a\n# b", "a\n\\# b")] - #[case::plain_text_unchanged("plain text", "plain text")] - #[case::literal_backslash_is_doubled("a\\b", "a\\\\b")] - #[case::empty_string("", "")] - #[case::bang_before_bracket_only_bracket_escaped("![bar]", "!\\[bar\\]")] - #[case::bang_at_value_end_not_escaped("foo!", "foo!")] - #[case::consecutive_newlines_become_entity("foo\n\nbar", "foo&#10;\nbar")] - #[case::three_consecutive_newlines("foo\n\n\nbar", "foo&#10;&#10;\nbar")] - #[case::single_newline_unchanged("foo\nbar", "foo\nbar")] - #[case::leading_tab_becomes_entity("\tfoo", "&#9;foo")] - #[case::tab_not_at_line_start_unchanged("foo\tbar", "foo\tbar")] - #[case::angle_bracket_escaped("<br/> not a tag", "\\<br/> not a tag")] - // `&` is deliberately left unescaped: see KNOWN_FAILURES in gfm_roundtrip_fidelity.rs (#591). - #[case::ampersand_not_escaped("a&b", "a&b")] - fn test_escape_text(#[case] input: &str, #[case] expected: &str) { - assert_eq!(escape_text(input.to_string()), expected); - } - - #[rstest] - #[case::escaped_when_parsed_from_source( - Node::Text(Text { - value: "*a*".to_string(), - position: Some(Position { - start: Point { line: 1, column: 1 }, - end: Point { line: 1, column: 4 }, - }), - }), - "\\*a\\*" - )] - #[case::untouched_when_built_programmatically( - Node::Text(Text { value: "*a*".to_string(), position: None }), - "*a*" - )] - fn test_text_escaping_gated_by_position(#[case] node: Node, #[case] expected: &str) { - assert_eq!(node.to_string_with(&RenderOptions::default()), expected); - } - - #[rstest] - #[case::bare_safe("https://example.com/x", UrlSurroundStyle::None, "https://example.com/x")] - #[case::space_forces_angle("my uri", UrlSurroundStyle::None, "<my uri>")] - #[case::empty_forces_angle("", UrlSurroundStyle::None, "<>")] - #[case::unbalanced_paren_escaped("b)c", UrlSurroundStyle::None, "b\\)c")] - #[case::both_parens_escaped("foo(and(bar)", UrlSurroundStyle::None, "foo\\(and\\(bar\\)")] - #[case::forced_angle_style("plain", UrlSurroundStyle::Angle, "<plain>")] - #[case::angle_escapes_inner_angle_and_backslash("foo\\>", UrlSurroundStyle::None, "<foo\\\\\\>>")] - fn test_render_link_destination(#[case] url: &str, #[case] style: UrlSurroundStyle, #[case] expected: &str) { - assert_eq!(render_link_destination(url, &style), expected); - } - - #[rstest] - #[case::double_quote_escaped(r#"a "b" c"#, TitleSurroundStyle::Double, r#""a \"b\" c""#)] - #[case::single_quote_escaped("a 'b' c", TitleSurroundStyle::Single, "'a \\'b\\' c'")] - #[case::paren_escapes_both("a (b) c", TitleSurroundStyle::Paren, "(a \\(b\\) c)")] - #[case::backslash_escaped("a\\b", TitleSurroundStyle::Double, "\"a\\\\b\"")] - fn test_render_link_title(#[case] title: &str, #[case] style: TitleSurroundStyle, #[case] expected: &str) { - assert_eq!(render_link_title(title, &style), expected); - } - - #[rstest] - #[case::plain("plain", "plain")] - #[case::brackets("[a][b]", "\\[a\\]\\[b\\]")] - #[case::backslash("a\\b", "a\\\\b")] - fn test_escape_label(#[case] input: &str, #[case] expected: &str) { - assert_eq!(escape_label(input), expected); - } - - #[rstest] - #[case::shortcut_kept_when_ident_matches_label_present("foo", "foo", Some("foo".to_string()), "[foo]")] - #[case::shortcut_kept_when_ident_matches_label_absent("foo", "foo", None, "[foo]")] - #[case::full_form_when_ident_differs("foo", "bar", Some("bar".to_string()), "[foo][bar]")] - #[case::full_form_when_plain_needs_broad_escaping("foo*bar", "foo*bar", Some("foo*bar".to_string()), "[foo*bar][foo*bar]")] - fn test_link_ref_shortcut_vs_full( - #[case] text: &str, - #[case] ident: &str, - #[case] label: Option<String>, - #[case] expected: &str, - ) { - let node = Node::LinkRef(LinkRef { - ident: ident.to_string(), - values: vec![text.to_string().into()], - label, - position: None, - }); - assert_eq!(node.to_string_with(&RenderOptions::default()), expected); - } - - #[test] - fn test_link_ref_display_escaping_does_not_leak_into_matching_label() { - // The display text (first bracket) is escaped for safety; the matching key - // (second bracket) must stay the definition-compatible, narrowly-escaped label. - let node = Node::LinkRef(LinkRef { - ident: "foo*bar".to_string(), - values: vec![Node::Text(Text { - value: "foo*bar".to_string(), - position: Some(Position { - start: Point { line: 1, column: 1 }, - end: Point { line: 1, column: 8 }, - }), - })], - label: Some("foo*bar".to_string()), - position: None, - }); - assert_eq!(node.to_string_with(&RenderOptions::default()), "[foo\\*bar][foo*bar]"); - } - - #[rstest] - #[case::shortcut_kept_when_ident_matches("foo bar", "foo bar", "![foo bar]")] - #[case::full_form_when_ident_differs("foo bar", "foo*bar*", "![foo bar][foo*bar*]")] - fn test_image_ref_shortcut_vs_full(#[case] alt: &str, #[case] ident: &str, #[case] expected: &str) { - let node = Node::ImageRef(ImageRef { - alt: alt.to_string(), - ident: ident.to_string(), - label: None, - position: None, - }); - assert_eq!(node.to_string_with(&RenderOptions::default()), expected); - } - - #[rstest] - #[case::no_newline_unchanged("abc", 2, "abc")] - #[case::continuation_indented("a\nb", 2, "a\n b")] - #[case::empty_lines_untouched("a\n\nb", 2, "a\n\n b")] - #[case::first_line_never_indented(" a\nb", 3, " a\n b")] - fn test_indent_continuation(#[case] content: &str, #[case] width: usize, #[case] expected: &str) { - assert_eq!(indent_continuation(content, width), expected); - } - - #[rstest] - #[case::no_newline_still_shifted("abc", 3, " abc")] - #[case::zero_delta_unchanged("a\n b", 0, "a\n b")] - #[case::first_line_also_shifted("a\nb", 2, " a\n b")] - #[case::negative_delta_clamped_at_zero(" a\n b", -5, "a\nb")] - fn test_reindent_all_lines(#[case] content: &str, #[case] delta: isize, #[case] expected: &str) { - assert_eq!(reindent_all_lines(content, delta), expected); - } - - #[rstest] - #[case::unordered_bullet(false, 0, None, 2)] - #[case::ordered_single_digit(true, 0, None, 3)] - #[case::ordered_double_digit(true, 9, None, 4)] - #[case::ordered_with_start(true, 0, Some(10), 4)] - fn test_list_own_prefix_width( - #[case] ordered: bool, - #[case] index: usize, - #[case] start: Option<u32>, - #[case] expected: usize, - ) { - assert_eq!(list_own_prefix_width(ordered, index, start), expected); - } - - #[rstest] - #[case::no_backticks("", 1)] - #[case::single_run("foo`bar", 2)] - #[case::longer_run("foo``bar", 3)] - fn test_code_span_fence(#[case] value: &str, #[case] expected_len: usize) { - assert_eq!(code_span_fence(value), "`".repeat(expected_len)); - } - - #[rstest] - #[case::no_backticks_minimum_three("plain", "", 3)] - #[case::body_run_escalates("aaa\n```", "", 4)] - fn test_code_fence(#[case] value: &str, #[case] info: &str, #[case] expected_len: usize) { - assert_eq!(code_fence(value, info), "`".repeat(expected_len)); - } - - #[rstest] - #[case::info_backtick_switches_to_tilde("plain", "a`b", 3)] - #[case::info_backtick_run_switches_to_tilde("plain", "a``b", 3)] - #[case::body_tilde_run_escalates_within_tilde_fence("aaa\n~~~", "a`b", 4)] - fn test_code_fence_tilde(#[case] value: &str, #[case] info: &str, #[case] expected_len: usize) { - assert_eq!(code_fence(value, info), "~".repeat(expected_len)); - } - - proptest! { - // Every backslash escape_text inserts precedes exactly the character it - // protects, and pre-existing backslashes are doubled, so blindly dropping - // each backslash and keeping the character after it must recover the - // original string, for any input. Tabs and repeated newlines go through a - // separate entity-based path (see test_escape_text), excluded here. - #[test] - fn escape_text_is_losslessly_invertible( - s in prop::collection::vec(any::<char>(), 0..60) - .prop_map(|cs| cs.into_iter().collect::<String>()) - .prop_filter("excludes entity-escaped tab/newline-run cases", |s| { - !s.contains('\t') && !s.contains("\n\n") - }) - ) { - let escaped = escape_text(s.clone()); - prop_assert_eq!(naive_unescape(&escaped), s); - } - - // Rendering must never panic, whatever the heading text (arbitrary Unicode, - // possibly ending in a run of `#`) — this is a UTF-8 char-boundary regression - // guard for the trailing-`#` escape logic. - #[test] - fn heading_render_never_panics_on_unicode_text( - body in prop::collection::vec(any::<char>(), 0..20).prop_map(|cs| cs.into_iter().collect::<String>()), - hashes in 0usize..5, - depth in 1u8..=6, - ) { - let value = format!("{}{}", body, "#".repeat(hashes)); - let node = Node::Heading(Heading { depth, values: vec![value.into()], position: None }); - let _ = node.to_string_with(&RenderOptions::default()); - } - - #[test] - fn escape_label_is_losslessly_invertible( - s in prop::collection::vec(any::<char>(), 0..40).prop_map(|cs| cs.into_iter().collect::<String>()) - ) { - let escaped = escape_label(&s); - prop_assert_eq!(naive_unescape(&escaped), s); - } - - // A Link's destination, rendered through the real writer and re-parsed by the - // real markdown parser, must recover the original URL string — the concrete - // bug class from the GFM spec-suite fidelity check (spaces/parens/angle - // brackets silently corrupting or breaking the link on round-trip). - #[test] - fn link_url_round_trips_through_parser( - url in prop::collection::vec( - prop::sample::select(vec!['a', 'b', '1', ' ', '(', ')', '<', '>', '\\']), - 1..12, - ).prop_map(|cs| cs.into_iter().collect::<String>()) - ) { - let doc = format!("[text]({})\n", render_link_destination(&url, &UrlSurroundStyle::None)); - let parsed = crate::Markdown::from_markdown_str(&doc).unwrap(); - let link = parsed.nodes.iter().find_map(|n| match n { - Node::Link(l) => Some(l.url.as_str().to_string()), - _ => None, - }); - prop_assert_eq!(link, Some(url)); - } - - } - - fn naive_unescape(s: &str) -> String { - let mut result = String::with_capacity(s.len()); - let mut chars = s.chars(); - while let Some(c) = chars.next() { - if c == '\\' - && let Some(next) = chars.next() - { - result.push(next); - continue; - } - result.push(c); - } - result - } -} diff --git a/crates/mq-markdown/src/node/attr.rs b/crates/mq-markdown/src/node/attr.rs new file mode 100644 index 000000000..ad5429b87 --- /dev/null +++ b/crates/mq-markdown/src/node/attr.rs @@ -0,0 +1,464 @@ +//! Reading and writing node attributes by name. + +use super::*; + +impl Node { + /// Returns the value of the specified attribute, if present. + /// + /// `line`/`end_line` are handled here, not per-variant below, since `position` is common + /// to every node. + pub fn attr(&self, attr: &str) -> Option<AttrValue> { + match attr { + attr_keys::LINE => return self.position().map(|p| AttrValue::Integer(p.start.line as i64)), + attr_keys::END_LINE => return self.position().map(|p| AttrValue::Integer(p.end.line as i64)), + _ => {} + } + + match self { + Node::Footnote(Footnote { ident, values, .. }) => match attr { + attr_keys::IDENT => Some(AttrValue::String(ident.clone())), + attr_keys::VALUE => Some(AttrValue::String(values_to_string(values, &RenderOptions::default()))), + attr_keys::VALUES | attr_keys::CHILDREN => Some(AttrValue::Array(values.clone())), + _ => None, + }, + Node::Html(Html { value, .. }) => match attr { + attr_keys::VALUE => Some(AttrValue::String(value.clone())), + _ => None, + }, + Node::Text(Text { value, .. }) => match attr { + attr_keys::VALUE => Some(AttrValue::String(value.clone())), + _ => None, + }, + Node::Code(Code { + value, + lang, + meta, + fence, + .. + }) => match attr { + attr_keys::VALUE => Some(AttrValue::String(value.clone())), + attr_keys::LANG => lang.clone().map(AttrValue::String), + attr_keys::META => meta.clone().map(AttrValue::String), + attr_keys::FENCE => Some(AttrValue::Boolean(*fence)), + _ => None, + }, + Node::CodeInline(CodeInline { value, .. }) => match attr { + attr_keys::VALUE => Some(AttrValue::String(value.to_string())), + _ => None, + }, + Node::MathInline(MathInline { value, .. }) => match attr { + attr_keys::VALUE => Some(AttrValue::String(value.to_string())), + _ => None, + }, + Node::Math(Math { value, .. }) => match attr { + attr_keys::VALUE => Some(AttrValue::String(value.clone())), + _ => None, + }, + Node::Yaml(Yaml { value, .. }) => match attr { + attr_keys::VALUE => Some(AttrValue::String(value.clone())), + _ => None, + }, + Node::Toml(Toml { value, .. }) => match attr { + attr_keys::VALUE => Some(AttrValue::String(value.clone())), + _ => None, + }, + Node::Image(Image { alt, url, title, .. }) => match attr { + attr_keys::ALT => Some(AttrValue::String(alt.clone())), + attr_keys::URL => Some(AttrValue::String(url.clone())), + attr_keys::TITLE => title.clone().map(AttrValue::String), + _ => None, + }, + Node::ImageRef(ImageRef { alt, ident, label, .. }) => match attr { + attr_keys::ALT => Some(AttrValue::String(alt.clone())), + attr_keys::IDENT => Some(AttrValue::String(ident.clone())), + attr_keys::LABEL => label.clone().map(AttrValue::String), + _ => None, + }, + Node::Link(Link { url, title, values, .. }) => match attr { + attr_keys::URL => Some(AttrValue::String(url.as_str().to_string())), + attr_keys::TITLE => title.as_ref().map(|t| AttrValue::String(t.to_value())), + attr_keys::VALUE => Some(AttrValue::String(values_to_string(values, &RenderOptions::default()))), + attr_keys::VALUES | attr_keys::CHILDREN => Some(AttrValue::Array(values.clone())), + _ => None, + }, + #[cfg(feature = "wikilink")] + Node::WikiLink(WikiLink { target, text, .. }) => match attr { + attr_keys::URL => Some(AttrValue::String(target.clone())), + attr_keys::VALUE => Some(AttrValue::String(text.clone().unwrap_or_else(|| target.clone()))), + _ => None, + }, + #[cfg(feature = "callout")] + Node::Callout(Callout { + kind, + fold, + title, + values, + .. + }) => match attr { + attr_keys::KIND => Some(AttrValue::String(kind.clone())), + attr_keys::FOLD => fold.map(|fold| AttrValue::String(fold.to_string())), + attr_keys::TITLE => title.clone().map(AttrValue::String), + attr_keys::VALUE => Some(AttrValue::String(values_to_string(values, &RenderOptions::default()))), + attr_keys::VALUES | attr_keys::CHILDREN => Some(AttrValue::Array(values.clone())), + _ => None, + }, + #[cfg(feature = "embed")] + Node::Embed(Embed { target, display, .. }) => match attr { + attr_keys::URL => Some(AttrValue::String(target.clone())), + attr_keys::VALUE => Some(AttrValue::String(display.clone().unwrap_or_else(|| target.clone()))), + _ => None, + }, + Node::LinkRef(LinkRef { ident, label, .. }) => match attr { + attr_keys::IDENT => Some(AttrValue::String(ident.clone())), + attr_keys::LABEL => label.clone().map(AttrValue::String), + _ => None, + }, + Node::FootnoteRef(FootnoteRef { ident, label, .. }) => match attr { + attr_keys::IDENT => Some(AttrValue::String(ident.clone())), + attr_keys::LABEL => label.clone().map(AttrValue::String), + _ => None, + }, + Node::Definition(Definition { + ident, + url, + title, + label, + .. + }) => match attr { + attr_keys::IDENT => Some(AttrValue::String(ident.clone())), + attr_keys::URL => Some(AttrValue::String(url.as_str().to_string())), + attr_keys::TITLE => title.as_ref().map(|t| AttrValue::String(t.to_value())), + attr_keys::LABEL => label.clone().map(AttrValue::String), + _ => None, + }, + Node::Heading(Heading { depth, values, .. }) => match attr { + attr_keys::DEPTH | attr_keys::LEVEL => Some(AttrValue::Integer(*depth as i64)), + attr_keys::VALUE => Some(AttrValue::String(values_to_string(values, &RenderOptions::default()))), + attr_keys::VALUES | attr_keys::CHILDREN => Some(AttrValue::Array(values.clone())), + _ => None, + }, + Node::List(List { + index, + level, + ordered, + checked, + values, + .. + }) => match attr { + attr_keys::INDEX => Some(AttrValue::Integer(*index as i64)), + attr_keys::LEVEL => Some(AttrValue::Integer(*level as i64)), + attr_keys::ORDERED => Some(AttrValue::Boolean(*ordered)), + attr_keys::CHECKED => checked.map(AttrValue::Boolean), + attr_keys::VALUE => Some(AttrValue::String(values_to_string(values, &RenderOptions::default()))), + attr_keys::VALUES | attr_keys::CHILDREN => Some(AttrValue::Array(values.clone())), + _ => None, + }, + Node::TableCell(TableCell { + column, row, values, .. + }) => match attr { + attr_keys::COLUMN => Some(AttrValue::Integer(*column as i64)), + attr_keys::ROW => Some(AttrValue::Integer(*row as i64)), + attr_keys::VALUE => Some(AttrValue::String(values_to_string(values, &RenderOptions::default()))), + attr_keys::VALUES | attr_keys::CHILDREN => Some(AttrValue::Array(values.clone())), + _ => None, + }, + Node::TableAlign(TableAlign { align, .. }) => match attr { + attr_keys::ALIGN => Some(AttrValue::String( + align.iter().map(|a| a.to_string()).collect::<Vec<_>>().join(","), + )), + _ => None, + }, + Node::MdxFlowExpression(MdxFlowExpression { value, .. }) => match attr { + attr_keys::VALUE => Some(AttrValue::String(value.to_string())), + _ => None, + }, + Node::MdxTextExpression(MdxTextExpression { value, .. }) => match attr { + attr_keys::VALUE => Some(AttrValue::String(value.to_string())), + _ => None, + }, + Node::MdxJsEsm(MdxJsEsm { value, .. }) => match attr { + attr_keys::VALUE => Some(AttrValue::String(value.to_string())), + _ => None, + }, + Node::MdxJsxFlowElement(MdxJsxFlowElement { name, children, .. }) => match attr { + attr_keys::NAME => name.clone().map(AttrValue::String), + attr_keys::VALUES | attr_keys::CHILDREN => Some(AttrValue::Array(children.clone())), + _ => None, + }, + Node::MdxJsxTextElement(MdxJsxTextElement { name, children, .. }) => match attr { + attr_keys::NAME => name.as_ref().map(|n| AttrValue::String(n.to_string())), + attr_keys::VALUES | attr_keys::CHILDREN => Some(AttrValue::Array(children.clone())), + _ => None, + }, + Node::Strong(Strong { values, .. }) + | Node::Blockquote(Blockquote { values, .. }) + | Node::Delete(Delete { values, .. }) + | Node::Emphasis(Emphasis { values, .. }) + | Node::TableRow(TableRow { values, .. }) + | Node::Fragment(Fragment { values, .. }) => match attr { + attr_keys::VALUE => Some(AttrValue::String(values_to_string(values, &RenderOptions::default()))), + attr_keys::VALUES | attr_keys::CHILDREN => Some(AttrValue::Array(values.clone())), + _ => None, + }, + Node::Break(_) | Node::HorizontalRule(_) | Node::Empty => None, + } + } + + /// Sets the value of the specified attribute for the node, if supported. + pub fn set_attr(&mut self, attr: &str, value: impl Into<AttrValue>) { + let value = value.into(); + let value_str = value.as_string(); + + match self { + Node::Footnote(f) => { + if attr == attr_keys::IDENT { + f.ident = value_str; + } + } + Node::Html(h) => { + if attr == attr_keys::VALUE { + h.value = value_str; + } + } + Node::Text(t) => { + if attr == attr_keys::VALUE { + t.value = value_str; + } + } + Node::Code(c) => match attr { + attr_keys::VALUE => { + c.value = value_str; + } + attr_keys::LANG | "language" => { + c.lang = if value_str.is_empty() { None } else { Some(value_str) }; + } + attr_keys::META => { + c.meta = if value_str.is_empty() { None } else { Some(value_str) }; + } + attr_keys::FENCE => { + c.fence = match value { + AttrValue::Boolean(b) => b, + _ => value_str == "true", + }; + } + _ => (), + }, + Node::CodeInline(ci) => { + if attr == attr_keys::VALUE { + ci.value = value_str.into(); + } + } + Node::MathInline(mi) => { + if attr == attr_keys::VALUE { + mi.value = value_str.into(); + } + } + Node::Math(m) => { + if attr == attr_keys::VALUE { + m.value = value_str; + } + } + Node::Yaml(y) => { + if attr == attr_keys::VALUE { + y.value = value_str; + } + } + Node::Toml(t) => { + if attr == attr_keys::VALUE { + t.value = value_str; + } + } + Node::Image(i) => match attr { + attr_keys::ALT => { + i.alt = value_str; + } + attr_keys::URL => { + i.url = value_str; + } + attr_keys::TITLE => { + i.title = if value_str.is_empty() { None } else { Some(value_str) }; + } + _ => (), + }, + Node::ImageRef(i) => match attr { + attr_keys::ALT => { + i.alt = value_str; + } + attr_keys::IDENT => { + i.ident = value_str; + } + attr_keys::LABEL => { + i.label = if value_str.is_empty() { None } else { Some(value_str) }; + } + _ => (), + }, + Node::Link(l) => match attr { + attr_keys::URL => { + l.url = Url::new(value_str); + } + attr_keys::TITLE => { + l.title = if value_str.is_empty() { + None + } else { + Some(Title::new(value_str)) + }; + } + _ => (), + }, + Node::LinkRef(l) => match attr { + attr_keys::IDENT => { + l.ident = value_str; + } + attr_keys::LABEL => { + l.label = if value_str.is_empty() { None } else { Some(value_str) }; + } + _ => (), + }, + Node::FootnoteRef(f) => match attr { + attr_keys::IDENT => { + f.ident = value_str; + } + attr_keys::LABEL => { + f.label = if value_str.is_empty() { None } else { Some(value_str) }; + } + _ => (), + }, + Node::Definition(d) => match attr { + attr_keys::IDENT => { + d.ident = value_str; + } + attr_keys::URL => { + d.url = Url::new(value_str); + } + attr_keys::TITLE => { + d.title = if value_str.is_empty() { + None + } else { + Some(Title::new(value_str)) + }; + } + attr_keys::LABEL => { + d.label = if value_str.is_empty() { None } else { Some(value_str) }; + } + _ => (), + }, + Node::Heading(h) => match attr { + attr_keys::DEPTH | attr_keys::LEVEL => { + h.depth = match value { + AttrValue::Integer(i) => i as u8, + _ => value_str.parse::<u8>().unwrap_or(h.depth), + }; + } + _ => (), + }, + Node::List(l) => match attr { + attr_keys::INDEX => { + l.index = match value { + AttrValue::Integer(i) => i as usize, + _ => value_str.parse::<usize>().unwrap_or(l.index), + }; + } + attr_keys::LEVEL => { + l.level = match value { + AttrValue::Integer(i) => i as u8, + _ => value_str.parse::<u8>().unwrap_or(l.level), + }; + } + attr_keys::ORDERED => { + l.ordered = match value { + AttrValue::Boolean(b) => b, + _ => value_str == "true", + }; + } + attr_keys::CHECKED => { + l.checked = if value_str.is_empty() { + None + } else { + Some(match value { + AttrValue::Boolean(b) => b, + _ => value_str == "true", + }) + }; + } + _ => (), + }, + Node::TableCell(c) => match attr { + attr_keys::COLUMN => { + c.column = match value { + AttrValue::Integer(i) => i as usize, + _ => value_str.parse::<usize>().unwrap_or(c.column), + }; + } + attr_keys::ROW => { + c.row = match value { + AttrValue::Integer(i) => i as usize, + _ => value_str.parse::<usize>().unwrap_or(c.row), + }; + } + _ => (), + }, + Node::TableAlign(th) => { + if attr == attr_keys::ALIGN { + th.align = value_str.split(',').map(|s| s.trim().into()).collect(); + } + } + Node::MdxFlowExpression(m) => { + if attr == attr_keys::VALUE { + m.value = value_str.into(); + } + } + Node::MdxTextExpression(m) => { + if attr == attr_keys::VALUE { + m.value = value_str.into(); + } + } + Node::MdxJsEsm(m) => { + if attr == attr_keys::VALUE { + m.value = value_str.into(); + } + } + Node::MdxJsxFlowElement(m) => { + if attr == attr_keys::NAME { + m.name = if value_str.is_empty() { None } else { Some(value_str) }; + } + } + Node::MdxJsxTextElement(m) => { + if attr == attr_keys::NAME { + m.name = if value_str.is_empty() { + None + } else { + Some(value_str.into()) + }; + } + } + Node::Delete(_) + | Node::Blockquote(_) + | Node::Emphasis(_) + | Node::Strong(_) + | Node::TableRow(_) + | Node::Break(_) + | Node::HorizontalRule(_) + | Node::Fragment(_) + | Node::Empty => (), + #[cfg(feature = "wikilink")] + Node::WikiLink(w) => match attr { + attr_keys::URL => w.target = value_str, + attr_keys::VALUE => w.text = if value_str.is_empty() { None } else { Some(value_str) }, + _ => (), + }, + #[cfg(feature = "callout")] + Node::Callout(c) => match attr { + attr_keys::KIND => c.kind = value_str, + attr_keys::FOLD => c.fold = value_str.chars().next().filter(|c| matches!(c, '+' | '-')), + attr_keys::TITLE => c.title = if value_str.is_empty() { None } else { Some(value_str) }, + _ => (), + }, + #[cfg(feature = "embed")] + Node::Embed(e) => match attr { + attr_keys::URL => e.target = value_str, + attr_keys::VALUE => e.display = if value_str.is_empty() { None } else { Some(value_str) }, + _ => (), + }, + } + } +} diff --git a/crates/mq-markdown/src/node/expand.rs b/crates/mq-markdown/src/node/expand.rs new file mode 100644 index 000000000..55600b328 --- /dev/null +++ b/crates/mq-markdown/src/node/expand.rs @@ -0,0 +1,339 @@ +//! Turning Obsidian links and inline links written in text into nodes. + +use super::*; + +impl Node { + /// Returns true if any `Text` descendant contains `[[`, covering both + /// embed (`![[`) and wikilink (`[[`) patterns. + #[cfg(any(feature = "embed", feature = "wikilink"))] + fn values_contain_links(values: &[Node]) -> bool { + values.iter().any(|n| match n { + Node::Text(t) => t.value.contains("[["), + Node::Heading(h) => Self::values_contain_links(&h.values), + Node::Blockquote(b) => Self::values_contain_links(&b.values), + #[cfg(feature = "callout")] + Node::Callout(c) => Self::values_contain_links(&c.values), + Node::List(l) => Self::values_contain_links(&l.values), + Node::Strong(s) => Self::values_contain_links(&s.values), + Node::Emphasis(e) => Self::values_contain_links(&e.values), + Node::Delete(d) => Self::values_contain_links(&d.values), + Node::TableCell(tc) => Self::values_contain_links(&tc.values), + Node::TableRow(tr) => Self::values_contain_links(&tr.values), + Node::Footnote(f) => Self::values_contain_links(&f.values), + _ => false, + }) + } + + /// Shared tree walker used by all expand functions. Recurses into container + /// nodes and delegates `Text` node processing to `parse_into`. + #[cfg(any(feature = "embed", feature = "wikilink"))] + fn expand_with(nodes: Vec<Node>, parse_into: fn(&str, Option<Position>, &mut Vec<Node>)) -> Vec<Node> { + let mut result = Vec::with_capacity(nodes.len()); + for node in nodes { + Self::expand_with_into(node, &mut result, parse_into); + } + result + } + + #[cfg(any(feature = "embed", feature = "wikilink"))] + fn expand_with_into(node: Node, out: &mut Vec<Node>, parse_into: fn(&str, Option<Position>, &mut Vec<Node>)) { + match node { + Node::Text(Text { ref value, .. }) if !value.contains("[[") => out.push(node), + Node::Text(Text { value, position }) => parse_into(&value, position, out), + Node::Heading(mut h) => { + if Self::values_contain_links(&h.values) { + h.values = Self::expand_with(h.values, parse_into); + } + out.push(Node::Heading(h)); + } + Node::Blockquote(mut b) => { + if Self::values_contain_links(&b.values) { + b.values = Self::expand_with(b.values, parse_into); + } + out.push(Node::Blockquote(b)); + } + #[cfg(feature = "callout")] + Node::Callout(mut c) => { + if Self::values_contain_links(&c.values) { + c.values = Self::expand_with(c.values, parse_into); + } + out.push(Node::Callout(c)); + } + Node::List(mut l) => { + if Self::values_contain_links(&l.values) { + l.values = Self::expand_with(l.values, parse_into); + } + out.push(Node::List(l)); + } + Node::Strong(mut s) => { + if Self::values_contain_links(&s.values) { + s.values = Self::expand_with(s.values, parse_into); + } + out.push(Node::Strong(s)); + } + Node::Emphasis(mut e) => { + if Self::values_contain_links(&e.values) { + e.values = Self::expand_with(e.values, parse_into); + } + out.push(Node::Emphasis(e)); + } + Node::Delete(mut d) => { + if Self::values_contain_links(&d.values) { + d.values = Self::expand_with(d.values, parse_into); + } + out.push(Node::Delete(d)); + } + Node::TableCell(mut tc) => { + if Self::values_contain_links(&tc.values) { + tc.values = Self::expand_with(tc.values, parse_into); + } + out.push(Node::TableCell(tc)); + } + Node::TableRow(mut tr) => { + if Self::values_contain_links(&tr.values) { + tr.values = Self::expand_with(tr.values, parse_into); + } + out.push(Node::TableRow(tr)); + } + Node::Footnote(mut f) => { + if Self::values_contain_links(&f.values) { + f.values = Self::expand_with(f.values, parse_into); + } + out.push(Node::Footnote(f)); + } + other => out.push(other), + } + } + + /// Recursively scans a node list and converts `![[target]]` / `![[target|display]]` + /// patterns inside `Text` nodes into `Embed` nodes. + #[cfg(feature = "embed")] + pub fn expand_embeds(nodes: Vec<Node>) -> Vec<Node> { + Self::expand_with(nodes, Self::parse_embeds_into) + } + + /// Splits a text string on `![[...]]` patterns, pushing a mix of `Text` + /// and `Embed` nodes into `out`. + #[cfg(feature = "embed")] + pub(super) fn parse_embeds_into(text: &str, position: Option<Position>, out: &mut Vec<Node>) { + let mut last_end = 0; + let mut search_from = 0; + let mut found_any = false; + + while let Some(bang_rel) = text[search_from..].find("![[") { + let bang = search_from + bang_rel; + let inner_start = bang + 3; + + let Some(close_rel) = text[inner_start..].find("]]") else { + search_from = bang + 1; + continue; + }; + let close = inner_start + close_rel; + let content = &text[inner_start..close]; + + if content.contains('[') || content.contains(']') { + search_from = bang + 1; + continue; + } + + found_any = true; + if last_end < bang { + out.push(Node::Text(Text { + value: text[last_end..bang].to_string(), + position: position.clone(), + })); + } + let (target, display) = if let Some(pipe) = content.find('|') { + ( + content[..pipe].trim().to_string(), + Some(content[pipe + 1..].trim().to_string()), + ) + } else { + (content.trim().to_string(), None) + }; + out.push(Node::Embed(Embed { + target, + display, + position: position.clone(), + })); + last_end = close + 2; + search_from = close + 2; + } + + if !found_any { + out.push(Node::Text(Text { + value: text.to_string(), + position, + })); + return; + } + + if last_end < text.len() { + out.push(Node::Text(Text { + value: text[last_end..].to_string(), + position, + })); + } + } + + /// Recursively scans a node list and converts `[[target]]` / `[[target|text]]` + /// patterns inside `Text` nodes into `WikiLink` nodes. + #[cfg(feature = "wikilink")] + pub fn expand_wikilinks(nodes: Vec<Node>) -> Vec<Node> { + Self::expand_with(nodes, Self::parse_wikilinks_into) + } + + /// Splits a text string on `[[...]]` patterns, pushing a mix of `Text` + /// and `WikiLink` nodes into `out`. + #[cfg(feature = "wikilink")] + fn parse_wikilinks_into(text: &str, position: Option<Position>, out: &mut Vec<Node>) { + let mut last_end = 0; + let mut search_from = 0; + let mut found_any = false; + + while let Some(open_rel) = text[search_from..].find("[[") { + let open = search_from + open_rel; + let inner_start = open + 2; + + let Some(close_rel) = text[inner_start..].find("]]") else { + break; + }; + let close = inner_start + close_rel; + let content = &text[inner_start..close]; + + if content.contains('[') || content.contains(']') { + // nested brackets — skip past this '[' and retry + search_from = open + 1; + continue; + } + + found_any = true; + if last_end < open { + out.push(Node::Text(Text { + value: text[last_end..open].to_string(), + position: position.clone(), + })); + } + let (target, text_part) = if let Some(pipe) = content.find('|') { + (content[..pipe].trim(), Some(content[pipe + 1..].trim())) + } else { + (content.trim(), None) + }; + out.push(Node::WikiLink(WikiLink { + target: target.to_string(), + text: text_part.map(|t| t.to_string()), + position: position.clone(), + })); + last_end = close + 2; + search_from = close + 2; + } + + if !found_any { + out.push(Node::Text(Text { + value: text.to_string(), + position, + })); + return; + } + + if last_end < text.len() { + out.push(Node::Text(Text { + value: text[last_end..].to_string(), + position, + })); + } + } + + /// Splits a text string on `[[...]]` patterns, returning a mix of `Text` + /// and `WikiLink` nodes. Used only in tests. + #[cfg(all(feature = "wikilink", test))] + pub(super) fn parse_wikilinks_in_text(text: &str, position: Option<Position>) -> Vec<Node> { + let mut result = Vec::new(); + Self::parse_wikilinks_into(text, position, &mut result); + result + } + + /// Recursively scans a node list and converts both `![[target]]` (embed) and + /// `[[target]]` (wikilink) patterns in a single tree traversal. More efficient + /// than running `expand_embeds` followed by `expand_wikilinks` separately, which + /// traverses the tree and scans each `Text` node twice. + #[cfg(all(feature = "embed", feature = "wikilink"))] + pub fn expand_inline_links(nodes: Vec<Node>) -> Vec<Node> { + Self::expand_with(nodes, Self::parse_inline_links_into) + } + + /// Splits a text string on `![[...]]` and `[[...]]` patterns in a single pass, + /// pushing a mix of `Text`, `Embed`, and `WikiLink` nodes into `out`. + /// A `[[` immediately preceded by `!` is treated as an embed; otherwise as a wikilink. + #[cfg(all(feature = "embed", feature = "wikilink"))] + pub(super) fn parse_inline_links_into(text: &str, position: Option<Position>, out: &mut Vec<Node>) { + let mut last_end = 0; + let mut search_from = 0; + let mut found_any = false; + + while let Some(open_rel) = text[search_from..].find("[[") { + let open = search_from + open_rel; + let inner_start = open + 2; + + let Some(close_rel) = text[inner_start..].find("]]") else { + break; + }; + let close = inner_start + close_rel; + let content = &text[inner_start..close]; + + if content.contains('[') || content.contains(']') { + search_from = open + 1; + continue; + } + + let is_embed = text.as_bytes()[..open].last() == Some(&b'!'); + let token_start = if is_embed { open - 1 } else { open }; + + found_any = true; + if last_end < token_start { + out.push(Node::Text(Text { + value: text[last_end..token_start].to_string(), + position: position.clone(), + })); + } + + let (target, opt_part) = if let Some(pipe) = content.find('|') { + (content[..pipe].trim(), Some(content[pipe + 1..].trim())) + } else { + (content.trim(), None) + }; + + if is_embed { + out.push(Node::Embed(Embed { + target: target.to_string(), + display: opt_part.map(|s| s.to_string()), + position: position.clone(), + })); + } else { + out.push(Node::WikiLink(WikiLink { + target: target.to_string(), + text: opt_part.map(|s| s.to_string()), + position: position.clone(), + })); + } + + last_end = close + 2; + search_from = close + 2; + } + + if !found_any { + out.push(Node::Text(Text { + value: text.to_string(), + position, + })); + return; + } + + if last_end < text.len() { + out.push(Node::Text(Text { + value: text[last_end..].to_string(), + position, + })); + } + } +} diff --git a/crates/mq-markdown/src/node/render.rs b/crates/mq-markdown/src/node/render.rs new file mode 100644 index 000000000..a22807944 --- /dev/null +++ b/crates/mq-markdown/src/node/render.rs @@ -0,0 +1,1273 @@ +//! Rendering nodes back to Markdown. + +use super::*; + +impl Node { + pub fn to_string_with(&self, options: &RenderOptions) -> String { + self.render_with_theme(options, &ColorTheme::PLAIN) + } + + /// Returns a colored string representation of this node using ANSI escape codes. + #[cfg(feature = "color")] + pub fn to_colored_string_with(&self, options: &RenderOptions) -> String { + self.render_with_theme(options, &ColorTheme::COLORED) + } + + pub(crate) fn render_with_theme(&self, options: &RenderOptions, theme: &ColorTheme<'_>) -> String { + match self { + Self::List(List { + level, + checked, + values, + ordered, + index, + start, + marker, + .. + }) => { + let marker = if *ordered { + let delimiter = if *marker == Some(ListMarker::Paren) { ')' } else { '.' }; + format!("{}{}", start.unwrap_or(1) as usize + *index, delimiter) + } else if let Some(style) = &options.list_style { + style.to_string() + } else { + marker.and_then(ListMarker::style).unwrap_or_default().to_string() + }; + let checkbox = (*checked).map(|it| if it { "[x] " } else { "[ ] " }).unwrap_or(""); + let prefix_width = *level as usize * 2 + list_own_prefix_width(*ordered, *index, *start); + // Lines after the first are indented to the content of the item, whatever column they + // had in the source, which is not reliable after an escape. + let content = render_values_block(values, options, theme); + let content = indent_continuation(&content, prefix_width); + let (ms, me) = &theme.list_marker; + format!( + "{}{}{}{} {}{}", + " ".repeat(*level as usize), + ms, + marker, + me, + checkbox, + content + ) + } + Self::TableRow(TableRow { values, .. }) => { + let (ts, te) = &theme.table_separator; + let cells = values + .iter() + .map(|cell| cell.render_with_theme(options, theme)) + .collect::<Vec<_>>() + .join("|"); + format!("{}|{}{}|", ts, te, cells) + } + Self::TableCell(TableCell { values, .. }) => render_cell_values(values, options, theme), + Self::TableAlign(TableAlign { align, .. }) => { + let (ts, te) = &theme.table_separator; + format!("{}|{}|{}", ts, align.iter().map(|a| a.to_string()).join("|"), te) + } + Self::Blockquote(Blockquote { values, .. }) => { + let (bs, be) = &theme.blockquote_marker; + render_values_block(values, options, theme) + .split('\n') + .map(|line| format!("{}> {}{}", bs, be, line)) + .join("\n") + } + #[cfg(feature = "callout")] + Self::Callout(Callout { + kind, + fold, + title, + values, + .. + }) => { + let (bs, be) = &theme.blockquote_marker; + let fold = fold.map(String::from).unwrap_or_default(); + let header = match title.as_deref() { + Some(t) if !t.is_empty() => format!("[!{}]{} {}", kind, fold, t), + _ => format!("[!{}]{}", kind, fold), + }; + let header_line = format!("{}> {}{}", bs, be, header); + if values.is_empty() { + return header_line; + } + let body = render_values_block(values, options, theme); + if body.trim().is_empty() { + header_line + } else { + let body_lines = body + .split('\n') + .map(|line| format!("{}> {}{}", bs, be, line)) + .join("\n"); + format!("{}\n{}", header_line, body_lines) + } + } + #[cfg(feature = "embed")] + Self::Embed(Embed { target, display, .. }) => match display.as_deref() { + Some(d) if !d.is_empty() => format!("![[{}|{}]]", target, d), + _ => format!("![[{}]]", target), + }, + Self::Code(Code { + value, + lang, + fence, + meta, + position, + }) => { + let (cs, ce) = &theme.code; + if lang.is_some() || *fence { + let meta = meta.as_deref().map(|meta| format!(" {}", meta)).unwrap_or_default(); + let info = format!("{}{}", lang.as_deref().unwrap_or(""), meta); + // Empty body skips the content line so it doesn't gain a blank one. + let fence_str = code_fence(value, &info); + if value.is_empty() { + // Blank lines are not in the value, but the lines of the block tell of them. + let blank = position + .as_ref() + .map_or(0, |p| p.end.line.saturating_sub(p.start.line).saturating_sub(1)); + format!("{}{}{}\n{}{}{}", cs, fence_str, info, "\n".repeat(blank), fence_str, ce) + } else { + format!("{}{}{}\n{}\n{}{}", cs, fence_str, info, value, fence_str, ce) + } + } else { + value.lines().map(|line| format!("{} {}{}", cs, line, ce)).join("\n") + } + } + Self::Definition(Definition { + ident, + label, + url, + title, + .. + }) => { + let (us, ue) = &theme.link_url; + format!( + "[{}]: {}{}{}{}", + definition_label(label.as_deref(), ident), + us, + url.to_string_with(options), + ue, + title + .as_ref() + .map(|title| format!(" {}", title.to_string_with(options))) + .unwrap_or_default() + ) + } + Self::Delete(Delete { values, .. }) => { + let (ds, de) = &theme.delete; + format!("{}~~{}~~{}", ds, render_values(values, options, theme), de) + } + Self::Emphasis(Emphasis { values, .. }) => { + // A lone nested Emphasis child needs the other delimiter, or adjacent + // `*` `*` pairs fuse into `**` and reparse as Strong instead. + let delim = if matches!(values.as_slice(), [Self::Emphasis(_)]) { + "_" + } else { + "*" + }; + render_emphasis(values, delim, options, theme) + } + Self::Footnote(Footnote { values, ident, .. }) => { + format!( + "[^{}]: {}", + escape_brackets(ident), + render_values(values, options, theme) + ) + } + Self::FootnoteRef(FootnoteRef { label, ident, .. }) => { + format!("[^{}]", reference_label(label.as_deref(), ident)) + } + Self::Heading(Heading { depth, values, .. }) => { + let (hs, he) = &theme.heading; + let text = render_values(values, options, theme); + // A line ending at either end is not part of the heading text, so it is written as a + // character reference. + let text = { + let inner = text.trim_matches('\n'); + let leading = text.len() - text.trim_start_matches('\n').len(); + let trailing = text.len() - text.trim_end_matches('\n').len(); + if inner.is_empty() || (leading == 0 && trailing == 0) { + text.clone() + } else { + format!("{}{}{}", "&#10;".repeat(leading), inner, "&#10;".repeat(trailing)) + } + }; + // Multi-line content must stay setext for depths 1-2; ATX has no setext form. + if text.contains('\n') && matches!(depth, 1 | 2) { + let underline = if *depth == 1 { "===" } else { "---" }; + format!("{}{}\n{}{}", hs, text, underline, he) + } else { + // A trailing `#` run reads back as an ATX closing sequence and gets + // stripped as syntax rather than kept as heading text. + let text = text.replace('\n', " "); + let text = match text.rfind(|c: char| c != '#') { + Some(i) => { + let hash_start = i + text[i..].chars().next().map_or(1, char::len_utf8); + if hash_start < text.len() { + format!("{}\\{}", &text[..hash_start], &text[hash_start..]) + } else { + text + } + } + None if !text.is_empty() => format!("\\{text}"), + _ => text, + }; + format!("{}{} {}{}", hs, "#".repeat(*depth as usize), text, he) + } + } + Self::Html(Html { value, .. }) => { + let (hs, he) = &theme.html; + format!("{}{}{}", hs, value, he) + } + Self::Image(Image { alt, url, title, .. }) => { + let (is, ie) = &theme.image; + format!( + "{}![{}]({}{}){}", + is, + escape_label(alt), + render_link_destination(url, &options.link_url_style), + title + .as_deref() + .map(|it| format!(" {}", render_link_title(it, &options.link_title_style))) + .unwrap_or_default(), + ie + ) + } + // Shortcut is safe only if `alt` normalizes back to `ident`; `ident` is raw + // (already correctly escaped), so escape_label would double it up. + Self::ImageRef(ImageRef { alt, ident, label, .. }) => { + let (is, ie) = &theme.image; + let mismatched = crate::parser::normalize(alt) != ident.as_str(); + if mismatched || needs_broad_escaping(alt) { + format!( + "{}![{}][{}]{}", + is, + escape_label(alt), + reference_label(label.as_deref(), ident), + ie + ) + } else { + format!("{}![{}]{}", is, escape_label(alt), ie) + } + } + Self::CodeInline(CodeInline { value, .. }) => { + let (cs, ce) = &theme.code_inline; + let fence = code_span_fence(value); + // Padding avoids fusing with an edge backtick and protects a genuine + // leading+trailing space from the parser's own space-stripping rule. + let all_spaces = value.chars().all(|c| c == ' '); + if value.starts_with('`') + || value.ends_with('`') + || (!all_spaces && value.starts_with(' ') && value.ends_with(' ')) + { + format!("{}{} {} {}{}", cs, fence, value, fence, ce) + } else { + format!("{}{}{}{}{}", cs, fence, value, fence, ce) + } + } + Self::MathInline(MathInline { value, .. }) => { + let (ms, me) = &theme.math; + let fence = math_span_fence(value); + // Padding keeps a dollar sign at an end from fusing with the fence, and a genuine space at + // both ends from being removed when it is read. + let all_spaces = value.chars().all(|c| c == ' '); + if value.starts_with('$') + || value.ends_with('$') + || (!all_spaces && value.starts_with(' ') && value.ends_with(' ')) + { + format!("{}{} {} {}{}", ms, fence, value, fence, me) + } else { + format!("{}{}{}{}{}", ms, fence, value, fence, me) + } + } + Self::Link(Link { url, title, values, .. }) => { + let (ls, le) = &theme.link; + if title.is_none() + && let Some(target) = autolink_target(url.as_str(), values) + { + return format!("{ls}<{target}>{le}"); + } + format!( + "{}[{}]({}{}){}", + ls, + render_values(values, options, theme), + url.to_string_with(options), + title + .as_ref() + .map(|title| format!(" {}", title.to_string_with(options))) + .unwrap_or_default(), + le + ) + } + #[cfg(feature = "wikilink")] + Self::WikiLink(WikiLink { target, text, .. }) => { + let (ls, le) = &theme.link; + match text.as_deref() { + Some(t) if t != target.as_str() => format!("{}[[{}|{}]]{}", ls, target, t, le), + _ => format!("{}[[{}]]{}", ls, target, le), + } + } + // Same reasoning as ImageRef, plus the same broad-escaping fallback. + Self::LinkRef(LinkRef { + values, ident, label, .. + }) => { + let (ls, le) = &theme.link; + let rendered = render_values(values, options, theme); + let plain = values_to_value(values); + // The label of a shortcut reference is its text as written, markup included. + let written = render_values(values, options, &ColorTheme::PLAIN); + let mismatched = crate::parser::normalize(&written) != ident.as_str(); + + if mismatched || needs_broad_escaping(&plain) { + format!( + "{}[{}][{}]{}", + ls, + rendered, + reference_label(label.as_deref(), ident), + le + ) + } else { + format!("{}[{}]{}", ls, rendered, le) + } + } + Self::Math(Math { value, .. }) => { + let (ms, me) = &theme.math; + let fence = "$".repeat((longest_run(value, '$') + 1).max(2)); + format!("{}{}\n{}\n{}{}", ms, fence, value, fence, me) + } + // Only escape text parsed from real markdown (has a position); programmatic + // values like JSON output or attr lookups must stay untouched. + Self::Text(Text { value, position }) => { + if position.is_some() { + escape_text_with(value.clone(), options.mdx) + } else { + value.clone() + } + } + Self::MdxFlowExpression(mdx_flow_expression) => { + format!("{{{}}}", indent_expression(&mdx_flow_expression.value)) + } + Self::MdxJsxFlowElement(mdx_jsx_flow_element) => { + let name = mdx_jsx_flow_element.name.as_deref().unwrap_or_default(); + let attributes = if mdx_jsx_flow_element.attributes.is_empty() { + "".to_string() + } else { + format!( + " {}", + mdx_jsx_flow_element + .attributes + .iter() + .map(Self::mdx_attribute_content_to_string) + .join(" ") + ) + }; + + if mdx_jsx_flow_element.children.is_empty() { + // A fragment has no self-closing form. + if name.is_empty() { + format!("<{}></>", attributes) + } else { + format!("<{}{} />", name, attributes,) + } + } else { + // Each tag and the children sit on their own lines, or the children would read as + // inline content of the tag. + let children = render_values_block(&mdx_jsx_flow_element.children, options, theme); + // The lines of an expression are indented as they are written, which indenting them + // again would add to each time they are read. + let children = if mdx_jsx_flow_element.children.iter().any(has_multiline_expression) { + children + } else { + indent_continuation(&format!(" {}", children), 2) + }; + format!("<{}{}>\n{}\n</{}>", name, attributes, children, name) + } + } + Self::MdxJsxTextElement(mdx_jsx_text_element) => { + let name = mdx_jsx_text_element.name.as_deref().unwrap_or_default(); + let attributes = if mdx_jsx_text_element.attributes.is_empty() { + "".to_string() + } else { + format!( + " {}", + mdx_jsx_text_element + .attributes + .iter() + .map(Self::mdx_attribute_content_to_string) + .join(" ") + ) + }; + + if mdx_jsx_text_element.children.is_empty() { + // A fragment has no self-closing form. + if name.is_empty() { + format!("<{}></>", attributes) + } else { + format!("<{}{} />", name, attributes,) + } + } else { + format!( + "<{}{}>{}</{}>", + name, + attributes, + render_values(&mdx_jsx_text_element.children, options, theme), + name + ) + } + } + Self::MdxTextExpression(mdx_text_expression) => { + format!("{{{}}}", indent_expression(&mdx_text_expression.value)) + } + Self::MdxJsEsm(mdxjs_esm) => mdxjs_esm.value.to_string(), + Self::Strong(Strong { values, .. }) => { + let (ss, se) = &theme.strong; + let last = values.len().saturating_sub(1); + let content = values + .iter() + .enumerate() + .map(|(index, value)| match value { + // Emphasis at an end uses the other delimiter, or `*` and `**` fuse into `***`. + // An underscore does not open or close next to a letter or digit, so those keep `*`. + Self::Emphasis(Emphasis { values: inner, .. }) + if (index == 0 || index == last) + && !index + .checked_sub(1) + .is_some_and(|before| ends_with_word_char(&values[before])) + && !values.get(index + 1).is_some_and(starts_with_word_char) => + { + render_emphasis(inner, "_", options, theme) + } + value => value.render_with_theme(options, theme), + }) + .collect::<String>(); + format!("{}**{}**{}", ss, content, se) + } + Self::Yaml(Yaml { value, .. }) => { + let (fs, fe) = &theme.frontmatter; + format!("{}---\n{}\n---{}", fs, value, fe) + } + Self::Toml(Toml { value, .. }) => { + let (fs, fe) = &theme.frontmatter; + format!("{}+++\n{}\n+++{}", fs, value, fe) + } + Self::Break(_) => "\\\n".to_string(), + Self::HorizontalRule(HorizontalRule { marker, .. }) => { + let (hs, he) = &theme.horizontal_rule; + let marker = match marker { + Some(HorizontalRuleMarker::Dash) => '-', + Some(HorizontalRuleMarker::Underscore) => '_', + _ => '*', + }; + format!("{hs}{marker}{marker}{marker}{he}") + } + Self::Fragment(Fragment { values }) => values + .iter() + .filter(|value| !value.is_empty() && !value.is_empty_fragment()) + .map(|value| value.render_with_theme(options, theme)) + .join("\n"), + Self::Empty => String::new(), + } + } + + fn mdx_attribute_content_to_string(attr: &MdxAttributeContent) -> SmolStr { + match attr { + MdxAttributeContent::Expression(value) => format!("{{{}}}", indent_expression(value)).into(), + MdxAttributeContent::Property(property) => match &property.value { + Some(value) => match value { + MdxAttributeValue::Expression(value) => { + format!("{}={{{}}}", property.name, indent_expression(value)).into() + } + MdxAttributeValue::Literal(literal) => { + format!("{}={}", property.name, quote_attribute_value(literal)).into() + } + }, + None => property.name.clone(), + }, + } + } +} + +/// Whether `node` or a node in it is an expression of more than one line. +fn has_multiline_expression(node: &Node) -> bool { + match node { + Node::MdxFlowExpression(expression) => expression.value.contains('\n'), + Node::MdxTextExpression(expression) => expression.value.contains('\n'), + Node::MdxJsxTextElement(MdxJsxTextElement { attributes, .. }) + | Node::MdxJsxFlowElement(MdxJsxFlowElement { attributes, .. }) + if attributes.iter().any(|attribute| match attribute { + MdxAttributeContent::Expression(value) => value.contains('\n'), + MdxAttributeContent::Property(MdxJsxAttribute { + value: Some(MdxAttributeValue::Expression(value)), + .. + }) => value.contains('\n'), + _ => false, + }) => + { + true + } + other => other.children().iter().any(has_multiline_expression), + } +} + +/// Quotes the value of a JSX attribute with the quote it does not contain. Character references that the +/// value holds as text are written with their `&` as a reference. +fn quote_attribute_value(value: &str) -> String { + let mut escaped = String::with_capacity(value.len()); + for (index, c) in value.char_indices() { + if c == '&' && starts_reference(&value[index + 1..]) { + escaped.push_str("&amp;"); + } else { + escaped.push(c); + } + } + if !escaped.contains('"') { + format!("\"{escaped}\"") + } else if !escaped.contains('\'') { + format!("'{escaped}'") + } else { + format!("\"{}\"", escaped.replace('"', "&quot;")) + } +} + +/// The content of an expression as it is written: the lines after the first are indented, as the +/// indentation of up to two characters is removed from them when they are read. +fn indent_expression(value: &str) -> String { + value + .split('\n') + .enumerate() + .map(|(index, line)| { + if index == 0 || line.is_empty() || line == "\r" { + line.to_string() + } else { + format!(" {line}") + } + }) + .collect::<Vec<_>>() + .join("\n") +} + +pub(crate) fn values_to_string(values: &[Node], options: &RenderOptions) -> String { + render_values(values, options, &ColorTheme::PLAIN) +} + +/// Indents every line, the first included, by `width` spaces, except empty ones. +pub(crate) fn indent_lines(content: &str, width: usize) -> String { + let pad = " ".repeat(width); + content + .split('\n') + .map(|line| { + if line.is_empty() { + line.to_string() + } else { + format!("{pad}{line}") + } + }) + .collect::<Vec<_>>() + .join("\n") +} + +/// Indents every line after the first by `width` spaces, except empty ones. +fn indent_continuation(content: &str, width: usize) -> String { + let Some((first, rest)) = content.split_once('\n') else { + return content.to_string(); + }; + let pad = " ".repeat(width); + let mut result = String::with_capacity(content.len() + width * 2); + result.push_str(first); + for line in rest.split('\n') { + result.push('\n'); + if !line.is_empty() { + result.push_str(&pad); + result.push_str(line); + } + } + result +} + +fn render_emphasis(values: &[Node], delim: &str, options: &RenderOptions, theme: &ColorTheme<'_>) -> String { + let (es, ee) = &theme.emphasis; + format!("{es}{delim}{}{delim}{ee}", render_values(values, options, theme)) +} + +/// Renders the content of a table cell. A pipe that is not escaped yet, as in code, would end the cell. +pub(crate) fn render_cell_values(values: &[Node], options: &RenderOptions, theme: &ColorTheme<'_>) -> String { + let rendered = render_values(values, options, theme); + if !rendered.contains('|') { + return rendered; + } + let mut result = String::with_capacity(rendered.len() + 4); + let mut escaped = false; + for c in rendered.chars() { + if c == '|' && !escaped { + result.push('\\'); + } + escaped = c == '\\' && !escaped; + result.push(c); + } + result +} + +fn ends_with_word_char(node: &Node) -> bool { + matches!(node, Node::Text(Text { value, .. }) if value.ends_with(|c: char| c.is_alphanumeric())) +} + +fn starts_with_word_char(node: &Node) -> bool { + matches!(node, Node::Text(Text { value, .. }) if value.starts_with(|c: char| c.is_alphanumeric())) +} + +/// Renders `value`, escaping a `!` that ends its text when `next` starts a link, which would make an image. +pub(crate) fn render_before( + value: &Node, + next: Option<&Node>, + options: &RenderOptions, + theme: &ColorTheme<'_>, +) -> String { + let mut rendered = value.render_with_theme(options, theme); + let starts_link = match next { + Some(Node::Link(_) | Node::LinkRef(_)) => true, + #[cfg(feature = "wikilink")] + Some(Node::WikiLink(_)) => true, + _ => false, + }; + if starts_link && matches!(value, Node::Text(Text { position: Some(_), .. })) && rendered.ends_with('!') { + rendered.insert(rendered.len() - 1, '\\'); + } + rendered +} + +pub(crate) fn render_values(values: &[Node], options: &RenderOptions, theme: &ColorTheme<'_>) -> String { + let mut pre_position: Option<Position> = None; + values + .iter() + .enumerate() + .map(|(index, value)| { + let rendered = || render_before(value, values.get(index + 1), options, theme); + if let Some(pos) = value.position() { + let new_line_count = pre_position + .as_ref() + .map(|p: &Position| pos.start.line.saturating_sub(p.end.line)) + .unwrap_or_default(); + + let space = if new_line_count > 0 + && pre_position + .as_ref() + .map(|p| pos.start.line > p.end.line) + .unwrap_or_default() + { + " ".repeat(pos.start.column.saturating_sub(1)) + } else { + "".to_string() + }; + + pre_position = Some(pos); + + if space.is_empty() { + format!("{}{}", "\n".repeat(new_line_count), rendered()) + } else { + format!( + "{}{}", + "\n".repeat(new_line_count), + rendered().lines().map(|line| format!("{}{}", space, line)).join("\n") + ) + } + } else { + pre_position = None; + rendered() + } + }) + .collect::<String>() +} + +/// Like `render_values` but without column-offset–based indentation. +/// +/// Inside blockquotes and callouts, node positions include the `> ` prefix offset, +/// so applying column-based spacing causes double-indentation. This variant preserves +/// blank lines between values (via newline counting) but ignores the column position. +pub(crate) fn render_values_block(values: &[Node], options: &RenderOptions, theme: &ColorTheme<'_>) -> String { + let is_table_part = |node: &Node| matches!(node, Node::TableCell(_) | Node::TableAlign(_)); + let mut result = String::new(); + let mut pre_position: Option<Position> = None; + let mut index = 0; + + while index < values.len() { + // The cells of a table are laid out together, as at the top level. + let (rendered, position, next) = if is_table_part(&values[index]) { + let end = values[index..] + .iter() + .position(|node| !is_table_part(node)) + .map_or(values.len(), |offset| index + offset); + let run = &values[index..end]; + let table = crate::Markdown { + nodes: run.to_vec(), + options: options.clone(), + }; + let position = match (run[0].position(), run[run.len() - 1].position()) { + (Some(first), Some(last)) => Some(Position { + start: first.start, + end: last.end, + }), + _ => None, + }; + ( + table.render_with_theme(theme).trim_end_matches('\n').to_string(), + position, + end, + ) + } else { + let value = &values[index]; + ( + render_before(value, values.get(index + 1), options, theme), + value.position(), + index + 1, + ) + }; + index = next; + + if let Some(pos) = position { + let new_line_count = pre_position + .as_ref() + .map(|p: &Position| pos.start.line.saturating_sub(p.end.line)) + .unwrap_or_default(); + pre_position = Some(pos); + result.push_str(&"\n".repeat(new_line_count)); + } else { + pre_position = None; + } + result.push_str(&rendered); + } + result +} + +/// Renders a link/image destination, auto-upgrading to `<...>` (escaping `\ < >`) +/// when the bare form would be unsafe (empty, or contains whitespace/control/`< >`); +/// otherwise bare, escaping `\ ( )` since unescaped parens must balance. +pub(super) fn render_link_destination(url: &str, style: &UrlSurroundStyle) -> String { + let needs_angle = url.is_empty() + || url + .chars() + .any(|c| c.is_whitespace() || c.is_control() || c == '<' || c == '>'); + + if matches!(style, UrlSurroundStyle::Angle) || needs_angle { + let mut result = String::with_capacity(url.len() + 2); + result.push('<'); + for c in url.chars() { + if matches!(c, '\\' | '<' | '>') { + result.push('\\'); + } + result.push(c); + } + result.push('>'); + result + } else { + let mut result = String::with_capacity(url.len()); + for c in url.chars() { + if matches!(c, '\\' | '(' | ')') { + result.push('\\'); + } + result.push(c); + } + result + } +} + +/// Renders a link/image title, escaping `\` and the style's delimiter(s) so the +/// title's own quote/paren characters can't end it early. +pub(super) fn render_link_title(title: &str, style: &TitleSurroundStyle) -> String { + let (open, close, delims): (char, char, &[char]) = match style { + TitleSurroundStyle::Double => ('"', '"', &['"']), + TitleSurroundStyle::Single => ('\'', '\'', &['\'']), + TitleSurroundStyle::Paren => ('(', ')', &['(', ')']), + }; + let mut result = String::with_capacity(title.len() + 2); + result.push(open); + for c in title.chars() { + if c == '\\' || delims.contains(&c) { + result.push('\\'); + } + result.push(c); + } + result.push(close); + result +} + +/// Escapes `\ [ ]` in reference labels/idents and image alt text: plain strings +/// (not walked as inline nodes) that still sit inside `[...]` syntax. +/// True if `s` needs escaping beyond `escape_label`'s narrow `\ [ ]` set — unsafe +/// for a reference's implicit shortcut form, since the definition's label won't match. +fn needs_broad_escaping(s: &str) -> bool { + s.bytes().any(|b| { + matches!( + b, + b'`' | b'*' | b'_' | b'|' | b'~' | b'$' | b'#' | b'-' | b'+' | b'>' | b'=' | b'.' | b')' | b'!' + ) + }) +} + +/// Shifts every line's leading indentation by `delta` spaces (never below zero), +/// leaving each line's own content untouched. +pub(crate) fn reindent_all_lines(content: &str, delta: isize) -> String { + if delta == 0 { + return content.to_string(); + } + content + .split('\n') + .map(|line| { + if line.is_empty() { + line.to_string() + } else { + let leading = line.bytes().take_while(|&b| b == b' ').count(); + let new_leading = (leading as isize + delta).max(0) as usize; + format!("{}{}", " ".repeat(new_leading), &line[leading..]) + } + }) + .collect::<Vec<_>>() + .join("\n") +} + +/// The width of a list item's own marker and trailing space (not including ancestor indentation), +/// e.g. 2 for `"- "`, 3 for `"1. "`, 4 for `"10. "`. A checkbox is not part of it, as the content of +/// the item starts at the checkbox, so children are indented to the marker alone. +pub(crate) fn list_own_prefix_width(ordered: bool, index: usize, start: Option<u32>) -> usize { + let marker_len = if ordered { + (start.unwrap_or(1) as usize + index).to_string().chars().count() + 1 + } else { + 1 + }; + marker_len + 1 +} + +/// The length of the longest run of `ch` in `s`. +fn longest_run(s: &str, ch: char) -> usize { + s.split('\n') + .flat_map(|line| line.split(|c| c != ch).map(str::len)) + .max() + .unwrap_or(0) +} + +/// Longest fence-char run in `value` + 1 (min 3); switches to tilde when `info` +/// has a backtick, since a backtick-fenced info string can't contain one. +fn code_fence(value: &str, info: &str) -> String { + if info.contains('`') { + let len = (longest_run(value, '~') + 1).max(3); + "~".repeat(len) + } else { + let len = (longest_run(value, '`') + 1).max(3); + "`".repeat(len) + } +} + +/// The label to write in a definition: the identifier as it was written when the label is only that +/// without its escapes and character references, which would not read back as the same identifier. +fn definition_label(label: Option<&str>, ident: &str) -> String { + match label { + Some(label) if crate::parser::normalize(label) == ident => escape_label(label), + Some(label) if crate::parser::normalize(label) == crate::parser::normalize(&crate::parser::unescape(ident)) => { + ident.to_string() + } + Some(label) => escape_label(label), + None => ident.to_string(), + } +} + +/// The label to write in the second brackets of a full reference. The label as written is kept when it +/// still resolves to `ident`. +fn reference_label(label: Option<&str>, ident: &str) -> String { + match label { + Some(label) if crate::parser::normalize(label) == ident => escape_label(label), + _ => ident.to_string(), + } +} + +/// Picks an inline code span delimiter one backtick longer than the longest +/// backtick run in `value`, so no run inside it can be mistaken for the closing. +fn code_span_fence(value: &str) -> String { + let mut max_run = 0; + let mut cur = 0; + for c in value.chars() { + if c == '`' { + cur += 1; + max_run = max_run.max(cur); + } else { + cur = 0; + } + } + "`".repeat(max_run + 1) +} + +/// The shortest run of dollar signs, from one, that is not the length of a run in `value`, so that no +/// run inside a math span can be mistaken for the closing. +fn math_span_fence(value: &str) -> String { + let mut lengths = std::collections::HashSet::new(); + let mut current = 0; + for c in value.chars() { + if c == '$' { + current += 1; + } else if current > 0 { + lengths.insert(current); + current = 0; + } + } + if current > 0 { + lengths.insert(current); + } + "$".repeat((1..).find(|length| !lengths.contains(length)).unwrap_or(1)) +} + +/// Escapes the brackets that are not escaped yet, for an identifier that keeps the escapes it was written with. +fn escape_brackets(s: &str) -> String { + let mut result = String::with_capacity(s.len()); + let mut chars = s.chars().peekable(); + while let Some(c) = chars.next() { + match c { + '\\' if chars.peek().is_some_and(char::is_ascii_punctuation) => { + result.push(c); + result.extend(chars.next()); + } + '[' | ']' => { + result.push('\\'); + result.push(c); + } + _ => result.push(c), + } + } + result +} + +fn escape_label(s: &str) -> String { + if !s.bytes().any(|b| matches!(b, b'\\' | b'[' | b']')) { + return s.to_string(); + } + let mut result = String::with_capacity(s.len()); + for c in s.chars() { + if matches!(c, '\\' | '[' | ']') { + result.push('\\'); + } + result.push(c); + } + result +} + +/// The text of an autolink for a link whose only content is its own destination, when writing that +/// text as a link label would need escapes. A URL in a label is read as it is, so `\_` would stay. +fn autolink_target(url: &str, values: &[Node]) -> Option<String> { + let [Node::Text(Text { value, .. })] = values else { + return None; + }; + let target = if value == url { + url + } else if url.strip_prefix("mailto:") == Some(value.as_str()) && crate::parser::is_autolink_email(value) { + value.as_str() + } else { + return None; + }; + let plain = !target + .chars() + .any(|c| c.is_whitespace() || c.is_control() || matches!(c, '<' | '>')); + let scheme = target.split_once(':').is_some_and(|(scheme, _)| { + (2..=32).contains(&scheme.len()) + && scheme.starts_with(|c: char| c.is_ascii_alphabetic()) + && scheme + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '+' | '.' | '-')) + }); + let email = value != url; + (plain && (scheme || email) && escape_text(target.to_string()) != target).then(|| target.to_string()) +} + +/// Whether `rest`, the text after an `&`, reads as the rest of a character reference. +fn starts_reference(rest: &str) -> bool { + let name = match rest.strip_prefix('#') { + Some(number) => match number.strip_prefix(['x', 'X']) { + Some(hex) => hex + .split_once(';') + .map(|(digits, _)| (digits, char::is_ascii_hexdigit as fn(&char) -> bool)), + None => number + .split_once(';') + .map(|(digits, _)| (digits, char::is_ascii_digit as fn(&char) -> bool)), + }, + None => rest + .split_once(';') + .map(|(name, _)| (name, char::is_ascii_alphanumeric as fn(&char) -> bool)), + }; + name.is_some_and(|(name, is_valid)| !name.is_empty() && name.chars().all(|c| is_valid(&c))) +} + +fn is_email_local(c: char) -> bool { + c.is_ascii_alphanumeric() || matches!(c, '.' | '-' | '_' | '+') +} + +/// Whether `rest`, the text after an `@`, starts with a domain that makes an email address of the text +/// before it. +fn is_email_domain(rest: &str) -> bool { + let domain = rest + .split(|c: char| !(c.is_ascii_alphanumeric() || matches!(c, '.' | '-' | '_'))) + .next() + .unwrap_or_default(); + domain + .split_once('.') + .is_some_and(|(label, after)| !label.is_empty() && after.starts_with(|c: char| c.is_ascii_alphanumeric())) +} + +/// Re-escapes markdown-significant characters so plain text can't be reinterpreted as +/// syntax on re-parse. `# - + > =` only unsafe at line start, `. )` only after leading digits; +/// a leading `\t` and runs of 2+ `\n` (only possible via decoded entities) use `&#N;` instead. +fn escape_text(value: String) -> String { + escape_text_with(value, false) +} + +/// Whether `text` starts with the `import ` or `export ` that makes a block ESM in MDX. +fn starts_esm(text: &str) -> bool { + text.starts_with("import ") || text.starts_with("export ") +} + +/// `escape_text`, and for MDX also the braces and the `import` or `export` that starts a line. +fn escape_text_with(value: String, mdx: bool) -> String { + // Skip the copy when nothing needs escaping; byte scan avoids UTF-8 decoding. + let needs_escaping = value.bytes().any(|b| { + matches!( + b, + b'\\' + | b'`' + | b'*' + | b'_' + | b'[' + | b']' + | b'|' + | b'~' + | b'$' + | b'#' + | b'-' + | b'+' + | b'>' + | b'<' + | b'=' + | b'.' + | b')' + | b'\n' + | b'\t' + | b'&' + | b'@' + ) + }); + let mdx_special = mdx && (value.contains('{') || value.contains("import ") || value.contains("export ")); + if !needs_escaping && !mdx_special { + return value; + } + + let mut result = String::with_capacity(value.len()); + let mut at_line_start = true; + // True while in an unbroken digit run since line start (ordered-list marker number). + let mut leading_digits = false; + let mut chars = value + .char_indices() + .map(|(index, c)| (c, &value[index + c.len_utf8()..])) + .peekable(); + let mut previous = None; + + while let Some((c, rest)) = chars.next() { + match c { + '\n' if chars.peek().is_some_and(|(next, _)| *next == '\n') => { + result.push_str("&#10;"); + at_line_start = true; + leading_digits = false; + continue; + } + '\t' if at_line_start => { + result.push_str("&#9;"); + at_line_start = false; + leading_digits = false; + continue; + } + '\\' | '`' | '*' | '_' | '[' | ']' | '|' | '~' | '$' | '<' => result.push('\\'), + '{' if mdx => result.push('\\'), + // The first letter as a character reference keeps the line from starting an ESM block. + 'i' | 'e' if mdx && at_line_start && starts_esm(&value[value.len() - rest.len() - 1..]) => { + result.push_str(if c == 'i' { "&#105;" } else { "&#101;" }); + at_line_start = false; + leading_digits = false; + previous = Some(c); + continue; + } + '#' | '-' | '+' | '>' | '=' if at_line_start => result.push('\\'), + '.' | ')' if leading_digits => result.push('\\'), + '&' if starts_reference(rest) => result.push('\\'), + '@' if previous.is_some_and(is_email_local) && is_email_domain(rest) => result.push('\\'), + _ => {} + } + result.push(c); + previous = Some(c); + + let is_digit = c.is_ascii_digit(); + leading_digits = if at_line_start { + is_digit + } else { + leading_digits && is_digit + }; + at_line_start = c == '\n'; + } + + result +} + +#[cfg(test)] +mod tests { + use super::*; + use proptest::prelude::*; + use rstest::rstest; + + #[rstest] + #[case::heading_marker("## not a heading", "\\## not a heading")] + #[case::heading_seven_hashes_still_escaped("####### x", "\\####### x")] + #[case::emphasis_star("*not emphasis*", "\\*not emphasis\\*")] + #[case::emphasis_underscore("_not emphasis_", "\\_not emphasis\\_")] + #[case::code_span("`not code`", "\\`not code\\`")] + #[case::link_brackets("[not a link](x)", "\\[not a link\\](x)")] + #[case::list_marker("- not a list item", "\\- not a list item")] + #[case::blockquote_marker("> not a quote", "\\> not a quote")] + #[case::table_pipe("| not a table |", "\\| not a table \\|")] + #[case::strikethrough("~not strikethrough~", "\\~not strikethrough\\~")] + #[case::math("$not math$", "\\$not math\\$")] + #[case::plus_list_marker("+ not a list item", "\\+ not a list item")] + #[case::ordered_list_dot("5. not a list", "5\\. not a list")] + #[case::ordered_list_paren("5) not a list", "5\\) not a list")] + #[case::setext_h1_underline("Foo\n===", "Foo\n\\===")] + #[case::hash_not_at_line_start("a#b", "a#b")] + #[case::dash_not_at_line_start("a-b", "a-b")] + #[case::plus_not_at_line_start("a+b", "a+b")] + #[case::gt_not_at_line_start("a>b", "a>b")] + #[case::eq_not_at_line_start("a=b", "a=b")] + #[case::dot_not_after_leading_digits("a5. b", "a5. b")] + #[case::dot_after_broken_digit_run("5x3. b", "5x3. b")] + #[case::escapes_after_embedded_newline("a\n# b", "a\n\\# b")] + #[case::plain_text_unchanged("plain text", "plain text")] + #[case::literal_backslash_is_doubled("a\\b", "a\\\\b")] + #[case::empty_string("", "")] + #[case::bang_before_bracket_only_bracket_escaped("![bar]", "!\\[bar\\]")] + #[case::bang_at_value_end_not_escaped("foo!", "foo!")] + #[case::consecutive_newlines_become_entity("foo\n\nbar", "foo&#10;\nbar")] + #[case::three_consecutive_newlines("foo\n\n\nbar", "foo&#10;&#10;\nbar")] + #[case::single_newline_unchanged("foo\nbar", "foo\nbar")] + #[case::leading_tab_becomes_entity("\tfoo", "&#9;foo")] + #[case::tab_not_at_line_start_unchanged("foo\tbar", "foo\tbar")] + #[case::angle_bracket_escaped("<br/> not a tag", "\\<br/> not a tag")] + // `&` is deliberately left unescaped: see KNOWN_FAILURES in gfm_roundtrip_fidelity.rs (#591). + #[case::ampersand_not_escaped("a&b", "a&b")] + fn test_escape_text(#[case] input: &str, #[case] expected: &str) { + assert_eq!(escape_text(input.to_string()), expected); + } + + #[rstest] + #[case::plain("plain", "plain")] + #[case::brackets("[a][b]", "\\[a\\]\\[b\\]")] + #[case::backslash("a\\b", "a\\\\b")] + fn test_escape_label(#[case] input: &str, #[case] expected: &str) { + assert_eq!(escape_label(input), expected); + } + + #[rstest] + #[case::no_newline_unchanged("abc", 2, "abc")] + #[case::continuation_indented("a\nb", 2, "a\n b")] + #[case::empty_lines_untouched("a\n\nb", 2, "a\n\n b")] + #[case::first_line_never_indented(" a\nb", 3, " a\n b")] + fn test_indent_continuation(#[case] content: &str, #[case] width: usize, #[case] expected: &str) { + assert_eq!(indent_continuation(content, width), expected); + } + + #[rstest] + #[case::no_backticks("", 1)] + #[case::single_run("foo`bar", 2)] + #[case::longer_run("foo``bar", 3)] + fn test_code_span_fence(#[case] value: &str, #[case] expected_len: usize) { + assert_eq!(code_span_fence(value), "`".repeat(expected_len)); + } + + #[rstest] + #[case::no_backticks_minimum_three("plain", "", 3)] + #[case::body_run_escalates("aaa\n```", "", 4)] + fn test_code_fence(#[case] value: &str, #[case] info: &str, #[case] expected_len: usize) { + assert_eq!(code_fence(value, info), "`".repeat(expected_len)); + } + + #[rstest] + #[case::info_backtick_switches_to_tilde("plain", "a`b", 3)] + #[case::info_backtick_run_switches_to_tilde("plain", "a``b", 3)] + #[case::body_tilde_run_escalates_within_tilde_fence("aaa\n~~~", "a`b", 4)] + fn test_code_fence_tilde(#[case] value: &str, #[case] info: &str, #[case] expected_len: usize) { + assert_eq!(code_fence(value, info), "~".repeat(expected_len)); + } + + proptest! { + // Every backslash escape_text inserts precedes exactly the character it + // protects, and pre-existing backslashes are doubled, so blindly dropping + // each backslash and keeping the character after it must recover the + // original string, for any input. Tabs and repeated newlines go through a + // separate entity-based path (see test_escape_text), excluded here. + #[test] + fn escape_text_is_losslessly_invertible( + s in prop::collection::vec(any::<char>(), 0..60) + .prop_map(|cs| cs.into_iter().collect::<String>()) + .prop_filter("excludes entity-escaped tab/newline-run cases", |s| { + !s.contains('\t') && !s.contains("\n\n") + }) + ) { + let escaped = escape_text(s.clone()); + prop_assert_eq!(naive_unescape(&escaped), s); + } + + // Rendering must never panic, whatever the heading text (arbitrary Unicode, + // possibly ending in a run of `#`) — this is a UTF-8 char-boundary regression + // guard for the trailing-`#` escape logic. + #[test] + fn heading_render_never_panics_on_unicode_text( + body in prop::collection::vec(any::<char>(), 0..20).prop_map(|cs| cs.into_iter().collect::<String>()), + hashes in 0usize..5, + depth in 1u8..=6, + ) { + let value = format!("{}{}", body, "#".repeat(hashes)); + let node = Node::Heading(Heading { depth, values: vec![value.into()], position: None }); + let _ = node.to_string_with(&RenderOptions::default()); + } + + #[test] + fn escape_label_is_losslessly_invertible( + s in prop::collection::vec(any::<char>(), 0..40).prop_map(|cs| cs.into_iter().collect::<String>()) + ) { + let escaped = escape_label(&s); + prop_assert_eq!(naive_unescape(&escaped), s); + } + + // A Link's destination, rendered through the real writer and re-parsed by the + // real markdown parser, must recover the original URL string — the concrete + // bug class from the GFM spec-suite fidelity check (spaces/parens/angle + // brackets silently corrupting or breaking the link on round-trip). + #[test] + fn link_url_round_trips_through_parser( + url in prop::collection::vec( + prop::sample::select(vec!['a', 'b', '1', ' ', '(', ')', '<', '>', '\\']), + 1..12, + ).prop_map(|cs| cs.into_iter().collect::<String>()) + ) { + let doc = format!("[text]({})\n", render_link_destination(&url, &UrlSurroundStyle::None)); + let parsed = crate::Markdown::from_markdown_str(&doc).unwrap(); + let link = parsed.nodes.iter().find_map(|n| match n { + Node::Link(l) => Some(l.url.as_str().to_string()), + _ => None, + }); + prop_assert_eq!(link, Some(url)); + } + + } + + fn naive_unescape(s: &str) -> String { + let mut result = String::with_capacity(s.len()); + let mut chars = s.chars(); + while let Some(c) = chars.next() { + if c == '\\' + && let Some(next) = chars.next() + { + result.push(next); + continue; + } + result.push(c); + } + result + } +} diff --git a/crates/mq-markdown/src/node/tests.rs b/crates/mq-markdown/src/node/tests.rs new file mode 100644 index 000000000..3d6b04440 --- /dev/null +++ b/crates/mq-markdown/src/node/tests.rs @@ -0,0 +1,3032 @@ +use super::*; +use rstest::rstest; + +#[rstest] +#[case::dash('-', Some(ListMarker::Dash))] +#[case::plus('+', Some(ListMarker::Plus))] +#[case::star('*', Some(ListMarker::Star))] +#[case::period('.', Some(ListMarker::Period))] +#[case::paren(')', Some(ListMarker::Paren))] +#[case::underscore('_', None)] +#[case::digit('1', None)] +fn list_marker_from_char(#[case] char: char, #[case] expected: Option<ListMarker>) { + assert_eq!(ListMarker::from_char(char), expected); + assert_eq!(expected.and_then(ListMarker::as_char), expected.map(|_| char)); +} + +#[rstest] +#[case::dash(ListMarker::Dash, Some(ListStyle::Dash))] +#[case::plus(ListMarker::Plus, Some(ListStyle::Plus))] +#[case::star(ListMarker::Star, Some(ListStyle::Star))] +#[case::period(ListMarker::Period, None)] +#[case::paren(ListMarker::Paren, None)] +#[case::continuation(ListMarker::Continuation, None)] +fn list_marker_style(#[case] marker: ListMarker, #[case] expected: Option<ListStyle>) { + assert_eq!(marker.style(), expected); +} + +#[rstest] +#[case::dash(ListStyle::Dash, ListMarker::Dash)] +#[case::plus(ListStyle::Plus, ListMarker::Plus)] +#[case::star(ListStyle::Star, ListMarker::Star)] +fn list_marker_from_style(#[case] style: ListStyle, #[case] expected: ListMarker) { + assert_eq!(ListMarker::from(style), expected); +} + +#[rstest] +#[case::star('*', Some(HorizontalRuleMarker::Star))] +#[case::dash('-', Some(HorizontalRuleMarker::Dash))] +#[case::underscore('_', Some(HorizontalRuleMarker::Underscore))] +#[case::plus('+', None)] +fn horizontal_rule_marker_from_char(#[case] char: char, #[case] expected: Option<HorizontalRuleMarker>) { + assert_eq!(HorizontalRuleMarker::from_char(char), expected); + assert_eq!(expected.map(HorizontalRuleMarker::as_char), expected.map(|_| char)); +} + +#[rstest] +#[case::bullet_dash(false, Some(ListMarker::Dash), "- a")] +#[case::bullet_plus(false, Some(ListMarker::Plus), "+ a")] +#[case::bullet_star(false, Some(ListMarker::Star), "* a")] +#[case::bullet_unknown(false, None, "- a")] +#[case::bullet_with_an_ordered_marker(false, Some(ListMarker::Paren), "- a")] +#[case::ordered_period(true, Some(ListMarker::Period), "1. a")] +#[case::ordered_paren(true, Some(ListMarker::Paren), "1) a")] +#[case::ordered_unknown(true, None, "1. a")] +#[case::ordered_with_a_bullet_marker(true, Some(ListMarker::Star), "1. a")] +fn list_renders_its_marker(#[case] ordered: bool, #[case] marker: Option<ListMarker>, #[case] expected: &str) { + let node = Node::List(List { + index: 0, + level: 0, + checked: None, + values: vec![Node::Text(Text { + value: "a".into(), + position: None, + })], + ordered, + start: None, + spread: false, + marker, + position: None, + }); + assert_eq!( + node.render_with_theme(&RenderOptions::default(), &ColorTheme::PLAIN), + expected + ); +} + +#[rstest] +#[case::star(Some(HorizontalRuleMarker::Star), "***")] +#[case::dash(Some(HorizontalRuleMarker::Dash), "---")] +#[case::underscore(Some(HorizontalRuleMarker::Underscore), "___")] +#[case::unknown(None, "***")] +fn horizontal_rule_renders_its_marker(#[case] marker: Option<HorizontalRuleMarker>, #[case] expected: &str) { + let node = Node::HorizontalRule(HorizontalRule { marker, position: None }); + assert_eq!( + node.render_with_theme(&RenderOptions::default(), &ColorTheme::PLAIN), + expected + ); +} + +/// The JSON of the markers is the character they are written with, as it was when they were chars. +#[cfg(feature = "json")] +#[rstest] +#[case::dash(ListMarker::Dash, "\"-\"")] +#[case::plus(ListMarker::Plus, "\"+\"")] +#[case::star(ListMarker::Star, "\"*\"")] +#[case::period(ListMarker::Period, "\".\"")] +#[case::paren(ListMarker::Paren, "\")\"")] +fn list_marker_json(#[case] marker: ListMarker, #[case] expected: &str) { + assert_eq!(serde_json::to_string(&marker).unwrap(), expected); + assert_eq!(serde_json::from_str::<ListMarker>(expected).unwrap(), marker); +} + +#[cfg(feature = "json")] +#[rstest] +#[case::star(HorizontalRuleMarker::Star, "\"*\"")] +#[case::dash(HorizontalRuleMarker::Dash, "\"-\"")] +#[case::underscore(HorizontalRuleMarker::Underscore, "\"_\"")] +fn horizontal_rule_marker_json(#[case] marker: HorizontalRuleMarker, #[case] expected: &str) { + assert_eq!(serde_json::to_string(&marker).unwrap(), expected); + assert_eq!(serde_json::from_str::<HorizontalRuleMarker>(expected).unwrap(), marker); +} + +#[cfg(feature = "json")] +#[test] +fn list_json_keeps_the_marker_as_a_string() { + let json = + r#"{"type":"list","values":[],"index":0,"level":0,"ordered":false,"checked":null,"spread":false,"marker":"+"}"#; + let list = serde_json::from_str::<List>(json).unwrap(); + assert_eq!(list.marker, Some(ListMarker::Plus)); + assert!(serde_json::to_string(&list).unwrap().contains(r#""marker":"+""#)); +} + +#[test] +fn map_values_into_transforms_owned_fragments() { + let node = Node::Fragment(Fragment { + values: vec![Node::Text(Text { + value: "before".to_string(), + position: None, + })], + }); + + let mapped = node + .map_values_into(&mut |node| -> Result<Node, std::io::Error> { + Ok(match node { + Node::Text(text) => Node::Text(Text { + value: text.value.to_uppercase(), + position: text.position.clone(), + }), + node => node.clone(), + }) + }) + .unwrap(); + + assert_eq!(mapped.to_string(), "BEFORE"); +} + +#[test] +fn map_values_into_owned_moves_nodes_through_the_callback() { + let node = Node::Fragment(Fragment { + values: vec![Node::Text(Text { + value: "before".to_string(), + position: None, + })], + }); + + let mapped = node + .map_values_into_owned(&mut |node| -> Result<Node, std::io::Error> { + Ok(match node { + Node::Text(mut text) => { + text.value.make_ascii_uppercase(); + Node::Text(text) + } + node => node, + }) + }) + .unwrap(); + + assert_eq!(mapped.to_string(), "BEFORE"); +} + +#[rstest] +#[case::text(Node::Text(Text{value: "".to_string(), position: None}), + "test".to_string(), + Node::Text(Text{value: "test".to_string(), position: None }))] +#[case::blockquote(Node::Blockquote(Blockquote{values: vec!["test".to_string().into()], position: None }), + "test".to_string(), + Node::Blockquote(Blockquote{values: vec!["test".to_string().into()], position: None }))] +#[case::delete(Node::Delete(Delete{values: vec!["test".to_string().into()], position: None }), + "test".to_string(), + Node::Delete(Delete{values: vec!["test".to_string().into()], position: None }))] +#[case::emphasis(Node::Emphasis(Emphasis{values: vec!["test".to_string().into()], position: None }), + "test".to_string(), + Node::Emphasis(Emphasis{values: vec!["test".to_string().into()], position: None }))] +#[case::strong(Node::Strong(Strong{values: vec!["test".to_string().into()], position: None }), + "test".to_string(), + Node::Strong(Strong{values: vec!["test".to_string().into()], position: None }))] +#[case::heading(Node::Heading(Heading {depth: 1, values: vec!["test".to_string().into()], position: None }), + "test".to_string(), + Node::Heading(Heading{depth: 1, values: vec!["test".to_string().into()], position: None }))] +#[case::link(Node::Link(Link {url: Url::new("test".to_string()), values: Vec::new(), title: None, position: None }), + "test".to_string(), + Node::Link(Link{url: Url::new("test".to_string()), values: Vec::new(), title: None, position: None }))] +#[case::image(Node::Image(Image {alt: "test".to_string(), url: "test".to_string(), title: None, position: None }), + "test".to_string(), + Node::Image(Image{alt: "test".to_string(), url: "test".to_string(), title: None, position: None }))] +#[case::code(Node::Code(Code {value: "test".to_string(), lang: None, fence: true, meta: None, position: None }), + "test".to_string(), + Node::Code(Code{value: "test".to_string(), lang: None, fence: true, meta: None, position: None }))] +#[case::footnote_ref(Node::FootnoteRef(FootnoteRef {ident: "test".to_string(), label: None, position: None }), + "test".to_string(), + Node::FootnoteRef(FootnoteRef{ident: "test".to_string(), label: Some("test".to_string()), position: None }))] +#[case::footnote(Node::Footnote(Footnote {ident: "test".to_string(), values: Vec::new(), position: None }), + "test".to_string(), + Node::Footnote(Footnote{ident: "test".to_string(), values: Vec::new(), position: None }))] +#[case::list(Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: vec!["test".to_string().into()], position: None }), + "test".to_string(), + Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: vec!["test".to_string().into()], position: None }))] +#[case::list(Node::List(List{ marker: None,start: None, spread: false, index: 1, level: 1, checked: Some(true), ordered: false, values: vec!["test".to_string().into()], position: None }), + "test".to_string(), + Node::List(List{ marker: None,start: None, spread: false, index: 1, level: 1, checked: Some(true), ordered: false, values: vec!["test".to_string().into()], position: None }))] +#[case::list(Node::List(List{ marker: None,start: None, spread: false, index: 2, level: 2, checked: Some(false), ordered: false, values: vec!["test".to_string().into()], position: None }), + "test".to_string(), + Node::List(List{ marker: None,start: None, spread: false, index: 2, level: 2, checked: Some(false), ordered: false, values: vec!["test".to_string().into()], position: None }))] +#[case::code_inline(Node::CodeInline(CodeInline{ value: "t".into(), position: None }), + "test".to_string(), + Node::CodeInline(CodeInline{ value: "test".into(), position: None }))] +#[case::math_inline(Node::MathInline(MathInline{ value: "t".into(), position: None }), + "test".to_string(), + Node::MathInline(MathInline{ value: "test".into(), position: None }))] +#[case::toml(Node::Toml(Toml{ value: "t".to_string(), position: None }), + "test".to_string(), + Node::Toml(Toml{ value: "test".to_string(), position: None }))] +#[case::yaml(Node::Yaml(Yaml{ value: "t".to_string(), position: None }), + "test".to_string(), + Node::Yaml(Yaml{ value: "test".to_string(), position: None }))] +#[case::html(Node::Html(Html{ value: "t".to_string(), position: None }), + "test".to_string(), + Node::Html(Html{ value: "test".to_string(), position: None }))] +#[case::table_row(Node::TableRow(TableRow{ values: vec![ + Node::TableCell(TableCell{values: vec!["test1".to_string().into()], row:0, column:1, position: None}), + Node::TableCell(TableCell{values: vec!["test2".to_string().into()], row:0, column:2, position: None}) + ] + , position: None }), + "test3,test4".to_string(), + Node::TableRow(TableRow{ values: vec![ + Node::TableCell(TableCell{values: vec!["test3".to_string().into()], row:0, column:1, position: None}), + Node::TableCell(TableCell{values: vec!["test4".to_string().into()], row:0, column:2, position: None}) + ] + , position: None }))] +#[case::table_cell(Node::TableCell(TableCell{values: vec!["test1".to_string().into()], row:0, column:1, position: None}), + "test2".to_string(), + Node::TableCell(TableCell{values: vec!["test2".to_string().into()], row:0, column:1, position: None}),)] +#[case::link_ref(Node::LinkRef(LinkRef{ident: "test2".to_string(), values: vec![attr_keys::VALUE.to_string().into()], label: Some("test2".to_string()), position: None}), + "test2".to_string(), + Node::LinkRef(LinkRef{ident: "test2".to_string(), values: vec![attr_keys::VALUE.to_string().into()], label: Some("test2".to_string()), position: None}),)] +#[case::image_ref(Node::ImageRef(ImageRef{alt: attr_keys::ALT.to_string(), ident: "test1".to_string(), label: None, position: None}), + "test2".to_string(), + Node::ImageRef(ImageRef{alt: attr_keys::ALT.to_string(), ident: "test2".to_string(), label: Some("test2".to_string()), position: None}),)] +#[case::definition(Node::Definition(Definition{ url: Url::new(attr_keys::URL.to_string()), title: None, ident: "test1".to_string(), label: None, position: None}), + "test2".to_string(), + Node::Definition(Definition{url: Url::new("test2".to_string()), title: None, ident: "test1".to_string(), label: None, position: None}),)] +#[case::break_(Node::Break(Break{ position: None}), + "test".to_string(), + Node::Break(Break{position: None}))] +#[case::horizontal_rule(Node::HorizontalRule(HorizontalRule{ marker: None, position: None}), + "test".to_string(), + Node::HorizontalRule(HorizontalRule{ marker: None,position: None}))] +#[case::mdx_flow_expression(Node::MdxFlowExpression(MdxFlowExpression{value: "test".into(), position: None}), + "updated".to_string(), + Node::MdxFlowExpression(MdxFlowExpression{value: "updated".into(), position: None}))] +#[case::mdx_text_expression(Node::MdxTextExpression(MdxTextExpression{value: "test".into(), position: None}), + "updated".to_string(), + Node::MdxTextExpression(MdxTextExpression{value: "updated".into(), position: None}))] +#[case::mdx_js_esm(Node::MdxJsEsm(MdxJsEsm{value: "test".into(), position: None}), + "updated".to_string(), + Node::MdxJsEsm(MdxJsEsm{value: "updated".into(), position: None}))] +#[case(Node::MdxJsxFlowElement(MdxJsxFlowElement{ + name: Some("div".to_string()), + attributes: Vec::new(), + children: vec!["test".to_string().into()], + position: None + }), + "updated".to_string(), + Node::MdxJsxFlowElement(MdxJsxFlowElement{ + name: Some("div".to_string()), + attributes: Vec::new(), + children: vec!["updated".to_string().into()], + position: None + }))] +#[case::mdx_jsx_text_element(Node::MdxJsxTextElement(MdxJsxTextElement{ + name: Some("span".into()), + attributes: Vec::new(), + children: vec!["test".to_string().into()], + position: None + }), + "updated".to_string(), + Node::MdxJsxTextElement(MdxJsxTextElement{ + name: Some("span".into()), + attributes: Vec::new(), + children: vec!["updated".to_string().into()], + position: None + }))] +#[case(Node::Math(Math{ value: "x^2".to_string(), position: None }), + "test".to_string(), + Node::Math(Math{ value: "test".to_string(), position: None }))] +fn test_with_value(#[case] node: Node, #[case] input: String, #[case] expected: Node) { + assert_eq!(node.clone().with_value(input.as_str()), expected); + assert_eq!(node.into_with_value(input.as_str()), expected); +} + +#[rstest] +#[case(Node::Blockquote(Blockquote{values: vec![ + Node::Text(Text{value: "first".to_string(), position: None}), + Node::Text(Text{value: "second".to_string(), position: None}) +], position: None}), + "new", + 0, + Node::Blockquote(Blockquote{values: vec![ + Node::Text(Text{value: "new".to_string(), position: None}), + Node::Text(Text{value: "second".to_string(), position: None}) + ], position: None}))] +#[case(Node::Blockquote(Blockquote{values: vec![ + Node::Text(Text{value: "first".to_string(), position: None}), + Node::Text(Text{value: "second".to_string(), position: None}) +], position: None}), + "new", + 1, + Node::Blockquote(Blockquote{values: vec![ + Node::Text(Text{value: "first".to_string(), position: None}), + Node::Text(Text{value: "new".to_string(), position: None}) + ], position: None}))] +#[case(Node::Delete(Delete{values: vec![ + Node::Text(Text{value: "first".to_string(), position: None}), + Node::Text(Text{value: "second".to_string(), position: None}) +], position: None}), + "new", + 0, + Node::Delete(Delete{values: vec![ + Node::Text(Text{value: "new".to_string(), position: None}), + Node::Text(Text{value: "second".to_string(), position: None}) + ], position: None}))] +#[case(Node::Emphasis(Emphasis{values: vec![ + Node::Text(Text{value: "first".to_string(), position: None}), + Node::Text(Text{value: "second".to_string(), position: None}) +], position: None}), + "new", + 1, + Node::Emphasis(Emphasis{values: vec![ + Node::Text(Text{value: "first".to_string(), position: None}), + Node::Text(Text{value: "new".to_string(), position: None}) + ], position: None}))] +#[case(Node::Strong(Strong{values: vec![ + Node::Text(Text{value: "first".to_string(), position: None}), + Node::Text(Text{value: "second".to_string(), position: None}) +], position: None}), + "new", + 0, + Node::Strong(Strong{values: vec![ + Node::Text(Text{value: "new".to_string(), position: None}), + Node::Text(Text{value: "second".to_string(), position: None}) + ], position: None}))] +#[case(Node::Heading(Heading{depth: 1, values: vec![ + Node::Text(Text{value: "first".to_string(), position: None}), + Node::Text(Text{value: "second".to_string(), position: None}) +], position: None}), + "new", + 1, + Node::Heading(Heading{depth: 1, values: vec![ + Node::Text(Text{value: "first".to_string(), position: None}), + Node::Text(Text{value: "new".to_string(), position: None}) + ], position: None}))] +#[case(Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: vec![ + Node::Text(Text{value: "first".to_string(), position: None}), + Node::Text(Text{value: "second".to_string(), position: None}) +], position: None}), + "new", + 0, + Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: vec![ + Node::Text(Text{value: "new".to_string(), position: None}), + Node::Text(Text{value: "second".to_string(), position: None}) + ], position: None}))] +#[case(Node::TableCell(TableCell{column: 0, row: 0, values: vec![ + Node::Text(Text{value: "first".to_string(), position: None}), + Node::Text(Text{value: "second".to_string(), position: None}) +], position: None}), + "new", + 1, + Node::TableCell(TableCell{column: 0, row: 0, values: vec![ + Node::Text(Text{value: "first".to_string(), position: None}), + Node::Text(Text{value: "new".to_string(), position: None}) + ], position: None}))] +#[case(Node::Text(Text{value: "plain text".to_string(), position: None}), + "new", + 0, + Node::Text(Text{value: "plain text".to_string(), position: None}))] +#[case(Node::Code(Code{value: "code".to_string(), lang: None, fence: true, meta: None, position: None}), + "new", + 0, + Node::Code(Code{value: "code".to_string(), lang: None, fence: true, meta: None, position: None}))] +#[case(Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 1, checked: Some(true), ordered: false, values: vec![ + Node::Text(Text{value: "first".to_string(), position: None}) +], position: None}), + "new", + 0, + Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 1, checked: Some(true), ordered: false, values: vec![ + Node::Text(Text{value: "new".to_string(), position: None}) + ], position: None}))] +#[case(Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 1, checked: None, ordered: false, values: vec![ + Node::Text(Text{value: "first".to_string(), position: None}) +], position: None}), + "new", + 2, + Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 1, checked: None, ordered: false, values: vec![ + Node::Text(Text{value: "first".to_string(), position: None}) + ], position: None}))] +#[case::link_ref(Node::LinkRef(LinkRef{ident: "id".to_string(), values: vec![ + Node::Text(Text{value: "first".to_string(), position: None}), + Node::Text(Text{value: "second".to_string(), position: None}) + ], label: None, position: None}), "new", 0, Node::LinkRef(LinkRef{ident: "id".to_string(), values: vec![ + Node::Text(Text{value: "new".to_string(), position: None}), + Node::Text(Text{value: "second".to_string(), position: None}) + ], label: None, position: None}))] +#[case(Node::MdxJsxFlowElement(MdxJsxFlowElement{ + name: Some("div".to_string()), + attributes: Vec::new(), + children: vec![ + Node::Text(Text{value: "first".to_string(), position: None}), + Node::Text(Text{value: "second".to_string(), position: None}) + ], + position: None + }), + "new", + 0, + Node::MdxJsxFlowElement(MdxJsxFlowElement{ + name: Some("div".to_string()), + attributes: Vec::new(), + children: vec![ + Node::Text(Text{value: "new".to_string(), position: None}), + Node::Text(Text{value: "second".to_string(), position: None}) + ], + position: None + }))] +#[case(Node::MdxJsxFlowElement(MdxJsxFlowElement{ + name: Some("div".to_string()), + attributes: Vec::new(), + children: vec![ + Node::Text(Text{value: "first".to_string(), position: None}), + Node::Text(Text{value: "second".to_string(), position: None}) + ], + position: None + }), + "new", + 1, + Node::MdxJsxFlowElement(MdxJsxFlowElement{ + name: Some("div".to_string()), + attributes: Vec::new(), + children: vec![ + Node::Text(Text{value: "first".to_string(), position: None}), + Node::Text(Text{value: "new".to_string(), position: None}) + ], + position: None + }))] +#[case(Node::MdxJsxTextElement(MdxJsxTextElement{ + name: Some("span".into()), + attributes: Vec::new(), + children: vec![ + Node::Text(Text{value: "first".to_string(), position: None}), + Node::Text(Text{value: "second".to_string(), position: None}) + ], + position: None + }), + "new", + 0, + Node::MdxJsxTextElement(MdxJsxTextElement{ + name: Some("span".into()), + attributes: Vec::new(), + children: vec![ + Node::Text(Text{value: "new".to_string(), position: None}), + Node::Text(Text{value: "second".to_string(), position: None}) + ], + position: None + }))] +#[case(Node::MdxJsxTextElement(MdxJsxTextElement{ + name: Some("span".into()), + attributes: Vec::new(), + children: vec![ + Node::Text(Text{value: "first".to_string(), position: None}), + Node::Text(Text{value: "second".to_string(), position: None}) + ], + position: None + }), + "new", + 1, + Node::MdxJsxTextElement(MdxJsxTextElement{ + name: Some("span".into()), + attributes: Vec::new(), + children: vec![ + Node::Text(Text{value: "first".to_string(), position: None}), + Node::Text(Text{value: "new".to_string(), position: None}) + ], + position: None + }))] +fn test_with_children_value(#[case] node: Node, #[case] value: &str, #[case] index: usize, #[case] expected: Node) { + assert_eq!(node.clone().with_children_value(value, index), expected); + assert_eq!(node.into_with_children_value(value, index), expected); +} + +#[rstest] +#[case(Node::Text(Text{value: "test".to_string(), position: None }), + "test".to_string())] +#[case(Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 2, checked: None, ordered: false, values: vec!["test".to_string().into()], position: None}), + " - test".to_string())] +fn test_display(#[case] node: Node, #[case] expected: String) { + assert_eq!(node.to_string_with(&RenderOptions::default()), expected); +} + +#[rstest] +#[case(Node::Text(Text{value: "test".to_string(), position: None}), true)] +#[case(Node::CodeInline(CodeInline{value: "test".into(), position: None}), false)] +#[case(Node::MathInline(MathInline{value: "test".into(), position: None}), false)] +fn test_is_text(#[case] node: Node, #[case] expected: bool) { + assert_eq!(node.is_text(), expected); +} + +#[rstest] +#[case(Node::CodeInline(CodeInline{value: "test".into(), position: None}), true)] +#[case(Node::Text(Text{value: "test".to_string(), position: None}), false)] +fn test_is_inline_code(#[case] node: Node, #[case] expected: bool) { + assert_eq!(node.is_inline_code(), expected); +} + +#[rstest] +#[case(Node::MathInline(MathInline{value: "test".into(), position: None}), true)] +#[case(Node::Text(Text{value: "test".to_string(), position: None}), false)] +fn test_is_inline_math(#[case] node: Node, #[case] expected: bool) { + assert_eq!(node.is_inline_math(), expected); +} + +#[rstest] +#[case(Node::Strong(Strong{values: vec!["test".to_string().into()], position: None}), true)] +#[case(Node::Text(Text{value: "test".to_string(), position: None}), false)] +fn test_is_strong(#[case] node: Node, #[case] expected: bool) { + assert_eq!(node.is_strong(), expected); +} + +#[rstest] +#[case(Node::Delete(Delete{values: vec!["test".to_string().into()], position: None}), true)] +#[case(Node::Text(Text{value: "test".to_string(), position: None}), false)] +fn test_is_delete(#[case] node: Node, #[case] expected: bool) { + assert_eq!(node.is_delete(), expected); +} + +#[rstest] +#[case(Node::Link(Link{url: Url::new("test".to_string()), values: Vec::new(), title: None, position: None}), true)] +#[case(Node::Text(Text{value: "test".to_string(), position: None}), false)] +fn test_is_link(#[case] node: Node, #[case] expected: bool) { + assert_eq!(node.is_link(), expected); +} + +#[rstest] +#[case(Node::LinkRef(LinkRef{ident: "test".to_string(), values: Vec::new(), label: None, position: None}), true)] +#[case(Node::Text(Text{value: "test".to_string(), position: None}), false)] +fn test_is_link_ref(#[case] node: Node, #[case] expected: bool) { + assert_eq!(node.is_link_ref(), expected); +} + +#[rstest] +#[case(Node::Image(Image{alt: attr_keys::ALT.to_string(), url: "test".to_string(), title: None, position: None}), true)] +#[case(Node::Text(Text{value: "test".to_string(), position: None}), false)] +fn test_is_image(#[case] node: Node, #[case] expected: bool) { + assert_eq!(node.is_image(), expected); +} + +#[rstest] +#[case(Node::ImageRef(ImageRef{alt: attr_keys::ALT.to_string(), ident: "test".to_string(), label: None, position: None}), true)] +#[case(Node::Text(Text{value: "test".to_string(), position: None}), false)] +fn test_is_image_ref(#[case] node: Node, #[case] expected: bool) { + assert_eq!(node.is_image_ref(), expected); +} + +#[rstest] +#[case(Node::Code(Code{value: "code".to_string(), lang: Some("rust".to_string()), fence: true, meta: None, position: None}), true, Some("rust".into()))] +#[case(Node::Code(Code{value: "code".to_string(), lang: Some("rust".to_string()), fence: true, meta: None, position: None}), false, Some("python".into()))] +#[case(Node::Code(Code{value: "code".to_string(), lang: None, fence: true, meta: None, position: None}), true, None)] +#[case(Node::Code(Code{value: "code".to_string(), lang: None, fence: false, meta: None, position: None}), true, None)] +#[case(Node::Text(Text{value: "test".to_string(), position: None}), false, None)] +fn test_is_code(#[case] node: Node, #[case] expected: bool, #[case] lang: Option<SmolStr>) { + assert_eq!(node.is_code(lang), expected); +} + +#[rstest] +#[case(Node::Heading(Heading{depth: 1, values: vec!["test".to_string().into()], position: None}), true, Some(1))] +#[case(Node::Heading(Heading{depth: 2, values: vec!["test".to_string().into()], position: None}), false, Some(1))] +#[case(Node::Heading(Heading{depth: 1, values: vec!["test".to_string().into()], position: None}), true, None)] +#[case(Node::Text(Text{value: "test".to_string(), position: None}), false, None)] +fn test_is_heading(#[case] node: Node, #[case] expected: bool, #[case] depth: Option<u8>) { + assert_eq!(node.is_heading(depth), expected); +} + +#[rstest] +#[case(Node::HorizontalRule(HorizontalRule{ marker: None,position: None}), true)] +#[case(Node::Text(Text{value: "test".to_string(), position: None}), false)] +fn test_is_horizontal_rule(#[case] node: Node, #[case] expected: bool) { + assert_eq!(node.is_horizontal_rule(), expected); +} + +#[rstest] +#[case(Node::Blockquote(Blockquote{values: vec!["test".to_string().into()], position: None}), true)] +#[case(Node::Text(Text{value: "test".to_string(), position: None}), false)] +fn test_is_blockquote(#[case] node: Node, #[case] expected: bool) { + assert_eq!(node.is_blockquote(), expected); +} + +#[cfg(feature = "wikilink")] +#[rstest] +#[case(Node::WikiLink(WikiLink{target: "target".to_string(), text: None, position: None}), true)] +#[case(Node::WikiLink(WikiLink{target: "Three laws of motion".to_string(), text: Some("Newton".to_string()), position: None}), true)] +#[case(Node::Link(Link{url: Url::new("https://example.com".to_string()), values: Vec::new(), title: None, position: None}), false)] +#[case(Node::Link(Link{url: Url::new("relative.md".to_string()), values: Vec::new(), title: None, position: None}), false)] +#[case(Node::Text(Text{value: "test".to_string(), position: None}), false)] +#[case(Node::Text(Text{value: "[[target]]".to_string(), position: None}), false)] +fn test_is_wikilink(#[case] node: Node, #[case] expected: bool) { + assert_eq!(node.is_wikilink(), expected); +} + +#[cfg(feature = "wikilink")] +#[rstest] +#[case(Node::WikiLink(WikiLink{target: "target".to_string(), text: None, position: None}), true)] +#[case(Node::WikiLink(WikiLink{target: "Three laws of motion".to_string(), text: Some("Newton".to_string()), position: None}), true)] +#[case(Node::Link(Link{url: Url::new("https://example.com".to_string()), values: Vec::new(), title: None, position: None}), true)] +#[case(Node::Link(Link{url: Url::new("relative.md".to_string()), values: Vec::new(), title: None, position: None}), true)] +#[case(Node::Text(Text{value: "test".to_string(), position: None}), false)] +fn test_is_link_includes_wikilink(#[case] node: Node, #[case] expected: bool) { + assert_eq!(node.is_link(), expected); +} + +#[cfg(feature = "wikilink")] +#[rstest] +// no wikilinks — returns original text node unchanged +#[case("plain text", vec![Node::Text(Text{value: "plain text".to_string(), position: None})])] +// only a wikilink +#[case("[[target]]", vec![Node::WikiLink(WikiLink{target: "target".to_string(), text: None, position: None})])] +// wikilink with display text +#[case("[[target|display]]", vec![Node::WikiLink(WikiLink{target: "target".to_string(), text: Some("display".to_string()), position: None})])] +// wikilink with spaces in target +#[case("[[Three laws of motion]]", vec![Node::WikiLink(WikiLink{target: "Three laws of motion".to_string(), text: None, position: None})])] +// wikilink with .md extension +#[case("[[target.md]]", vec![Node::WikiLink(WikiLink{target: "target.md".to_string(), text: None, position: None})])] +// wikilink at start +#[case("[[target]] after", vec![ + Node::WikiLink(WikiLink{target: "target".to_string(), text: None, position: None}), + Node::Text(Text{value: " after".to_string(), position: None}), +])] +// wikilink at end +#[case("before [[target]]", vec![ + Node::Text(Text{value: "before ".to_string(), position: None}), + Node::WikiLink(WikiLink{target: "target".to_string(), text: None, position: None}), +])] +// wikilink in middle +#[case("before [[target]] after", vec![ + Node::Text(Text{value: "before ".to_string(), position: None}), + Node::WikiLink(WikiLink{target: "target".to_string(), text: None, position: None}), + Node::Text(Text{value: " after".to_string(), position: None}), +])] +// multiple wikilinks +#[case("[[a]] and [[b]]", vec![ + Node::WikiLink(WikiLink{target: "a".to_string(), text: None, position: None}), + Node::Text(Text{value: " and ".to_string(), position: None}), + Node::WikiLink(WikiLink{target: "b".to_string(), text: None, position: None}), +])] +// unclosed [[ treated as plain text +#[case("[[unclosed", vec![Node::Text(Text{value: "[[unclosed".to_string(), position: None})])] +// nested brackets invalid — treated as plain text +#[case("[[in[ner]]]", vec![Node::Text(Text{value: "[[in[ner]]]".to_string(), position: None})])] +// multibyte characters in surrounding text +#[case("日本語 [[ターゲット]] テキスト", vec![ + Node::Text(Text{value: "日本語 ".to_string(), position: None}), + Node::WikiLink(WikiLink{target: "ターゲット".to_string(), text: None, position: None}), + Node::Text(Text{value: " テキスト".to_string(), position: None}), +])] +// multibyte characters inside wikilink target and display text +#[case("[[ページ|表示名]]", vec![ + Node::WikiLink(WikiLink{target: "ページ".to_string(), text: Some("表示名".to_string()), position: None}), +])] +fn test_parse_wikilinks_in_text(#[case] input: &str, #[case] expected: Vec<Node>) { + let result = Node::parse_wikilinks_in_text(input, None); + assert_eq!(result, expected); +} + +#[cfg(feature = "wikilink")] +#[rstest] +#[case(Node::WikiLink(WikiLink{target: "target".to_string(), text: None, position: None}), "url", Some(AttrValue::String("target".to_string())))] +#[case(Node::WikiLink(WikiLink{target: "target".to_string(), text: None, position: None}), "value", Some(AttrValue::String("target".to_string())))] +#[case(Node::WikiLink(WikiLink{target: "target".to_string(), text: Some("display".to_string()), position: None}), "url", Some(AttrValue::String("target".to_string())))] +#[case(Node::WikiLink(WikiLink{target: "target".to_string(), text: Some("display".to_string()), position: None}), "value", Some(AttrValue::String("display".to_string())))] +#[case(Node::WikiLink(WikiLink{target: "target".to_string(), text: None, position: None}), "title", None)] +fn test_wikilink_attr(#[case] node: Node, #[case] attr: &str, #[case] expected: Option<AttrValue>) { + assert_eq!(node.attr(attr), expected); +} + +#[cfg(feature = "wikilink")] +#[rstest] +// no display text: with_value sets target +#[case( + Node::WikiLink(WikiLink{target: "target".to_string(), text: None, position: None}), + "new-target", + Node::WikiLink(WikiLink{target: "new-target".to_string(), text: None, position: None}) +)] +// with display text: with_value sets text, not target +#[case( + Node::WikiLink(WikiLink{target: "page".to_string(), text: Some("Display Text".to_string()), position: None}), + "DISPLAY TEXT", + Node::WikiLink(WikiLink{target: "page".to_string(), text: Some("DISPLAY TEXT".to_string()), position: None}) +)] +fn test_wikilink_with_value(#[case] node: Node, #[case] value: &str, #[case] expected: Node) { + assert_eq!(node.clone().with_value(value), expected); + assert_eq!(node.into_with_value(value), expected); +} + +#[cfg(feature = "wikilink")] +#[rstest] +// footnote with no wikilinks — values unchanged +#[case( + Node::Footnote(Footnote{ident: "1".to_string(), values: vec![Node::Text(Text{value: "plain text".to_string(), position: None})], position: None}), + vec![Node::Footnote(Footnote{ident: "1".to_string(), values: vec![Node::Text(Text{value: "plain text".to_string(), position: None})], position: None})] +)] +// footnote with a wikilink in text — expanded to WikiLink node +#[case( + Node::Footnote(Footnote{ident: "1".to_string(), values: vec![Node::Text(Text{value: "[[target]]".to_string(), position: None})], position: None}), + vec![Node::Footnote(Footnote{ident: "1".to_string(), values: vec![Node::WikiLink(WikiLink{target: "target".to_string(), text: None, position: None})], position: None})] +)] +// footnote with wikilink and display text +#[case( + Node::Footnote(Footnote{ident: "2".to_string(), values: vec![Node::Text(Text{value: "[[target|display]]".to_string(), position: None})], position: None}), + vec![Node::Footnote(Footnote{ident: "2".to_string(), values: vec![Node::WikiLink(WikiLink{target: "target".to_string(), text: Some("display".to_string()), position: None})], position: None})] +)] +// footnote with mixed text and wikilink +#[case( + Node::Footnote(Footnote{ident: "3".to_string(), values: vec![Node::Text(Text{value: "see [[note]]".to_string(), position: None})], position: None}), + vec![Node::Footnote(Footnote{ident: "3".to_string(), values: vec![ + Node::Text(Text{value: "see ".to_string(), position: None}), + Node::WikiLink(WikiLink{target: "note".to_string(), text: None, position: None}), + ], position: None})] +)] +fn test_expand_wikilinks_footnote(#[case] input: Node, #[case] expected: Vec<Node>) { + let result = Node::expand_wikilinks(vec![input]); + assert_eq!(result, expected); +} + +#[cfg(all(feature = "embed", feature = "wikilink"))] +#[rstest] +// embed only +#[case("![[note.md]]", vec![ + Node::Embed(Embed{target: "note.md".to_string(), display: None, position: None}), +])] +// wikilink only +#[case("[[target]]", vec![ + Node::WikiLink(WikiLink{target: "target".to_string(), text: None, position: None}), +])] +// embed and wikilink in same text +#[case("![[embed]] and [[link]]", vec![ + Node::Embed(Embed{target: "embed".to_string(), display: None, position: None}), + Node::Text(Text{value: " and ".to_string(), position: None}), + Node::WikiLink(WikiLink{target: "link".to_string(), text: None, position: None}), +])] +// wikilink before embed +#[case("[[link]] and ![[embed]]", vec![ + Node::WikiLink(WikiLink{target: "link".to_string(), text: None, position: None}), + Node::Text(Text{value: " and ".to_string(), position: None}), + Node::Embed(Embed{target: "embed".to_string(), display: None, position: None}), +])] +// embed with display hint +#[case("![[image.png|400]]", vec![ + Node::Embed(Embed{target: "image.png".to_string(), display: Some("400".to_string()), position: None}), +])] +// wikilink with display text +#[case("[[page|display]]", vec![ + Node::WikiLink(WikiLink{target: "page".to_string(), text: Some("display".to_string()), position: None}), +])] +// surrounding text +#[case("before ![[note]] after [[link]] end", vec![ + Node::Text(Text{value: "before ".to_string(), position: None}), + Node::Embed(Embed{target: "note".to_string(), display: None, position: None}), + Node::Text(Text{value: " after ".to_string(), position: None}), + Node::WikiLink(WikiLink{target: "link".to_string(), text: None, position: None}), + Node::Text(Text{value: " end".to_string(), position: None}), +])] +// plain text with no links +#[case("just plain text", vec![ + Node::Text(Text{value: "just plain text".to_string(), position: None}), +])] +// multibyte characters +#[case("日本語 ![[ファイル]] と [[ページ]]", vec![ + Node::Text(Text{value: "日本語 ".to_string(), position: None}), + Node::Embed(Embed{target: "ファイル".to_string(), display: None, position: None}), + Node::Text(Text{value: " と ".to_string(), position: None}), + Node::WikiLink(WikiLink{target: "ページ".to_string(), text: None, position: None}), +])] +fn test_parse_inline_links_into(#[case] input: &str, #[case] expected: Vec<Node>) { + let mut result = Vec::new(); + Node::parse_inline_links_into(input, None, &mut result); + assert_eq!(result, expected); +} + +#[rstest] +#[case(Node::Html(Html{value: "<div>test</div>".to_string(), position: None}), true)] +#[case(Node::Text(Text{value: "test".to_string(), position: None}), false)] +fn test_is_html(#[case] node: Node, #[case] expected: bool) { + assert_eq!(node.is_html(), expected); +} + +#[rstest] +#[case(Node::node_values( + &Node::Strong(Strong{values: vec!["test".to_string().into()], position: None})), + vec!["test".to_string().into()])] +#[case(Node::node_values( + &Node::Text(Text{value: "test".to_string(), position: None})), + vec!["test".to_string().into()])] +#[case(Node::node_values( + &Node::Blockquote(Blockquote{values: vec!["test".to_string().into()], position: None})), + vec!["test".to_string().into()])] +#[case(Node::node_values( + &Node::Delete(Delete{values: vec!["test".to_string().into()], position: None})), + vec!["test".to_string().into()])] +#[case(Node::node_values( + &Node::Emphasis(Emphasis{values: vec!["test".to_string().into()], position: None})), + vec!["test".to_string().into()])] +#[case(Node::node_values( + &Node::Heading(Heading{depth: 1, values: vec!["test".to_string().into()], position: None})), + vec!["test".to_string().into()])] +#[case(Node::node_values( + &Node::List(List{ marker: None,values: vec!["test".to_string().into()], ordered: false, level: 1, checked: Some(false), index: 0, start: None, spread: false, position: None})), + vec!["test".to_string().into()])] +fn test_node_value(#[case] actual: Vec<Node>, #[case] expected: Vec<Node>) { + assert_eq!(actual, expected); +} + +#[rstest] +#[case(Node::Footnote(Footnote{ident: "test".to_string(), values: Vec::new(), position: None}), true)] +#[case(Node::Text(Text{value: "test".to_string(), position: None}), false)] +fn test_is_footnote(#[case] node: Node, #[case] expected: bool) { + assert_eq!(node.is_footnote(), expected); +} + +#[rstest] +#[case(Node::FootnoteRef(FootnoteRef{ident: "test".to_string(), label: None, position: None}), true)] +#[case(Node::Text(Text{value: "test".to_string(), position: None}), false)] +fn test_is_footnote_ref(#[case] node: Node, #[case] expected: bool) { + assert_eq!(node.is_footnote_ref(), expected); +} + +#[rstest] +#[case(Node::Math(Math{value: "x^2".to_string(), position: None}), true)] +#[case(Node::Text(Text{value: "test".to_string(), position: None}), false)] +fn test_is_math(#[case] node: Node, #[case] expected: bool) { + assert_eq!(node.is_math(), expected); +} + +#[rstest] +#[case(Node::Break(Break{position: None}), true)] +#[case(Node::Text(Text{value: "test".to_string(), position: None}), false)] +fn test_is_break(#[case] node: Node, #[case] expected: bool) { + assert_eq!(node.is_break(), expected); +} + +#[rstest] +#[case(Node::Yaml(Yaml{value: "key: value".to_string(), position: None}), true)] +#[case(Node::Text(Text{value: "test".to_string(), position: None}), false)] +fn test_is_yaml(#[case] node: Node, #[case] expected: bool) { + assert_eq!(node.is_yaml(), expected); +} + +#[rstest] +#[case(Node::Toml(Toml{value: "key = \"value\"".to_string(), position: None}), true)] +#[case(Node::Text(Text{value: "test".to_string(), position: None}), false)] +fn test_is_toml(#[case] node: Node, #[case] expected: bool) { + assert_eq!(node.is_toml(), expected); +} + +#[rstest] +#[case(Node::Definition(Definition{ident: attr_keys::IDENT.to_string(), url: Url::new(attr_keys::URL.to_string()), title: None, label: None, position: None}), true)] +#[case(Node::Text(Text{value: "test".to_string(), position: None}), false)] +fn test_is_definition(#[case] node: Node, #[case] expected: bool) { + assert_eq!(node.is_definition(), expected); +} + +#[rstest] +#[case(Node::Emphasis(Emphasis{values: vec!["test".to_string().into()], position: None}), true)] +#[case(Node::Text(Text{value: "test".to_string(), position: None}), false)] +fn test_is_emphasis(#[case] node: Node, #[case] expected: bool) { + assert_eq!(node.is_emphasis(), expected); +} + +#[rstest] +#[case(Node::MdxFlowExpression(MdxFlowExpression{value: "test".into(), position: None}), true)] +#[case(Node::Text(Text{value: "test".to_string(), position: None}), false)] +fn test_is_mdx_flow_expression(#[case] node: Node, #[case] expected: bool) { + assert_eq!(node.is_mdx_flow_expression(), expected); +} + +#[rstest] +#[case(Node::MdxTextExpression(MdxTextExpression{value: "test".into(), position: None}), true)] +#[case(Node::Text(Text{value: "test".to_string(), position: None}), false)] +fn test_is_mdx_text_expression(#[case] node: Node, #[case] expected: bool) { + assert_eq!(node.is_mdx_text_expression(), expected); +} + +#[rstest] +#[case(Node::MdxJsxFlowElement(MdxJsxFlowElement{name: None, attributes: Vec::new(), children: Vec::new(), position: None}), true)] +#[case(Node::Text(Text{value: "test".to_string(), position: None}), false)] +fn test_is_mdx_jsx_flow_element(#[case] node: Node, #[case] expected: bool) { + assert_eq!(node.is_mdx_jsx_flow_element(), expected); +} + +#[rstest] +#[case(Node::MdxJsxTextElement(MdxJsxTextElement{name: None, attributes: Vec::new(), children: Vec::new(), position: None}), true)] +#[case(Node::Text(Text{value: "test".to_string(), position: None}), false)] +fn test_is_mdx_jsx_text_element(#[case] node: Node, #[case] expected: bool) { + assert_eq!(node.is_mdx_jsx_text_element(), expected); +} + +#[rstest] +#[case(Node::MdxJsEsm(MdxJsEsm{value: "test".into(), position: None}), true)] +#[case(Node::Text(Text{value: "test".to_string(), position: None}), false)] +fn test_is_msx_js_esm(#[case] node: Node, #[case] expected: bool) { + assert_eq!(node.is_mdx_js_esm(), expected); +} + +#[rstest] +#[case::text(Node::Text(Text{value: "test".to_string(), position: None }), RenderOptions::default(), "test")] +#[case::list(Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 2, checked: None, ordered: false, values: vec!["test".to_string().into()], position: None}), RenderOptions::default(), " - test")] +#[case::list(Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 1, checked: None, ordered: false, values: vec!["test".to_string().into()], position: None}), RenderOptions { list_style: Some(ListStyle::Plus), ..Default::default() }, " + test")] +#[case::list(Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 1, checked: Some(true), ordered: false, values: vec!["test".to_string().into()], position: None}), RenderOptions { list_style: Some(ListStyle::Star), ..Default::default() }, " * [x] test")] +#[case::list(Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 1, checked: Some(false), ordered: false, values: vec!["test".to_string().into()], position: None}), RenderOptions::default(), " - [ ] test")] +#[case::list(Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 1, checked: None, ordered: true, values: vec!["test".to_string().into()], position: None}), RenderOptions::default(), " 1. test")] +#[case::list(Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 1, checked: Some(false), ordered: true, values: vec!["test".to_string().into()], position: None}), RenderOptions::default(), " 1. [ ] test")] +#[case::table_row(Node::TableRow(TableRow{values: vec![Node::TableCell(TableCell{column: 0, row: 0, values: vec!["test".to_string().into()], position: None})], position: None}), RenderOptions::default(), "|test|")] +#[case::table_row(Node::TableRow(TableRow{values: vec![Node::TableCell(TableCell{column: 0, row: 0, values: vec!["test".to_string().into()], position: None})], position: None}), RenderOptions::default(), "|test|")] +#[case::table_cell(Node::TableCell(TableCell{column: 0, row: 0, values: vec!["test".to_string().into()], position: None}), RenderOptions::default(), "test")] +#[case::table_cell(Node::TableCell(TableCell{column: 0, row: 0, values: vec!["test".to_string().into()], position: None}), RenderOptions::default(), "test")] +#[case::table_align(Node::TableAlign(TableAlign{align: vec![TableAlignKind::Left, TableAlignKind::Right, TableAlignKind::Center, TableAlignKind::None], position: None}), RenderOptions::default(), "|:---|---:|:---:|---|")] +#[case::block_quote(Node::Blockquote(Blockquote{values: vec!["test".to_string().into()], position: None}), RenderOptions::default(), "> test")] +#[case::block_quote(Node::Blockquote(Blockquote{values: vec!["test\ntest2".to_string().into()], position: None}), RenderOptions::default(), "> test\n> test2")] +#[case::code(Node::Code(Code{value: "code".to_string(), lang: Some("rust".to_string()), fence: true, meta: None, position: None}), RenderOptions::default(), "```rust\ncode\n```")] +#[case::code(Node::Code(Code{value: "code".to_string(), lang: None, fence: true, meta: None, position: None}), RenderOptions::default(), "```\ncode\n```")] +#[case::code(Node::Code(Code{value: "code".to_string(), lang: None, fence: false, meta: None, position: None}), RenderOptions::default(), " code")] +#[case::code(Node::Code(Code{value: "code".to_string(), lang: Some("rust".to_string()), fence: true, meta: Some("meta".to_string()), position: None}), RenderOptions::default(), "```rust meta\ncode\n```")] +#[case::code_empty_body_no_blank_line(Node::Code(Code{value: "".to_string(), lang: None, fence: true, meta: None, position: None}), RenderOptions::default(), "```\n```")] +#[case::code_fence_escalates_past_backtick_run_in_body(Node::Code(Code{value: "aaa\n```".to_string(), lang: None, fence: true, meta: None, position: None}), RenderOptions::default(), "````\naaa\n```\n````")] +#[case::code_fence_escalates_past_longer_run(Node::Code(Code{value: "`````".to_string(), lang: None, fence: true, meta: None, position: None}), RenderOptions::default(), "``````\n`````\n``````")] +#[case::definition(Node::Definition(Definition{ident: "id".to_string(), url: Url::new(attr_keys::URL.to_string()), title: None, label: Some(attr_keys::LABEL.to_string()), position: None}), RenderOptions::default(), "[label]: url")] +#[case::definition(Node::Definition(Definition{ident: "id".to_string(), url: Url::new(attr_keys::URL.to_string()), title: Some(Title::new(attr_keys::TITLE.to_string())), label: Some(attr_keys::LABEL.to_string()), position: None}), RenderOptions::default(), "[label]: url \"title\"")] +#[case::definition(Node::Definition(Definition{ident: "id".to_string(), url: Url::new("".to_string()), title: None, label: Some(attr_keys::LABEL.to_string()), position: None}), RenderOptions::default(), "[label]: <>")] +#[case::delete(Node::Delete(Delete{values: vec!["test".to_string().into()], position: None}), RenderOptions::default(), "~~test~~")] +#[case::emphasis(Node::Emphasis(Emphasis{values: vec!["test".to_string().into()], position: None}), RenderOptions::default(), "*test*")] +#[case::emphasis_nested_in_emphasis_alternates_delimiter(Node::Emphasis(Emphasis{values: vec![Node::Emphasis(Emphasis{values: vec!["foo".to_string().into()], position: None})], position: None}), RenderOptions::default(), "_*foo*_")] +#[case::footnote(Node::Footnote(Footnote{ident: "id".to_string(), values: vec![attr_keys::LABEL.to_string().into()], position: None}), RenderOptions::default(), "[^id]: label")] +#[case::footnote_ref(Node::FootnoteRef(FootnoteRef{ident: attr_keys::LABEL.to_string(), label: Some(attr_keys::LABEL.to_string()), position: None}), RenderOptions::default(), "[^label]")] +#[case::heading(Node::Heading(Heading{depth: 1, values: vec!["test".to_string().into()], position: None}), RenderOptions::default(), "# test")] +#[case::heading(Node::Heading(Heading{depth: 3, values: vec!["test".to_string().into()], position: None}), RenderOptions::default(), "### test")] +#[case::heading_multiline_h1_stays_setext(Node::Heading(Heading{depth: 1, values: vec!["Foo\nBar".to_string().into()], position: None}), RenderOptions::default(), "Foo\nBar\n===")] +#[case::heading_multiline_h2_stays_setext(Node::Heading(Heading{depth: 2, values: vec!["Foo\nBar".to_string().into()], position: None}), RenderOptions::default(), "Foo\nBar\n---")] +#[case::heading_multiline_h3_joins_with_space(Node::Heading(Heading{depth: 3, values: vec!["Foo\nBar".to_string().into()], position: None}), RenderOptions::default(), "### Foo Bar")] +#[case::heading_trailing_hash_escaped(Node::Heading(Heading{depth: 1, values: vec!["foo #".to_string().into()], position: None}), RenderOptions::default(), "# foo \\#")] +#[case::heading_trailing_hash_run_escaped(Node::Heading(Heading{depth: 3, values: vec!["foo ###".to_string().into()], position: None}), RenderOptions::default(), "### foo \\###")] +#[case::heading_trailing_hash_after_multibyte_char(Node::Heading(Heading{depth: 1, values: vec!["foo あ#".to_string().into()], position: None}), RenderOptions::default(), "# foo あ\\#")] +#[case::heading_no_trailing_hash_unaffected(Node::Heading(Heading{depth: 1, values: vec!["foo bar".to_string().into()], position: None}), RenderOptions::default(), "# foo bar")] +#[case::html(Node::Html(Html{value: "<div>test</div>".to_string(), position: None}), RenderOptions::default(), "<div>test</div>")] +#[case::image(Node::Image(Image{alt: attr_keys::ALT.to_string(), url: attr_keys::URL.to_string(), title: None, position: None}), RenderOptions::default(), "![alt](url)")] +#[case::image(Node::Image(Image{alt: attr_keys::ALT.to_string(), url: "url with space".to_string(), title: Some(attr_keys::TITLE.to_string()), position: None}), RenderOptions::default(), "![alt](<url with space> \"title\")")] +#[case::image_ref(Node::ImageRef(ImageRef{alt: attr_keys::ALT.to_string(), ident: "id".to_string(), label: Some("id".to_string()), position: None}), RenderOptions::default(), "![alt][id]")] +#[case::image_ref(Node::ImageRef(ImageRef{alt: "id".to_string(), ident: "id".to_string(), label: Some("id".to_string()), position: None}), RenderOptions::default(), "![id]")] +#[case::code_inline(Node::CodeInline(CodeInline{value: "code".into(), position: None}), RenderOptions::default(), "`code`")] +#[case::code_inline_trailing_backtick_needs_padding(Node::CodeInline(CodeInline{value: "\\[\\`".into(), position: None}), RenderOptions::default(), "`` \\[\\` ``")] +#[case::code_inline_internal_backtick_escalates_fence(Node::CodeInline(CodeInline{value: "foo ` bar".into(), position: None}), RenderOptions::default(), "``foo ` bar``")] +#[case::code_inline_both_ends_backtick_needs_longer_padded_fence(Node::CodeInline(CodeInline{value: "``".into(), position: None}), RenderOptions::default(), "``` `` ```")] +#[case::code_inline_leading_trailing_space_needs_extra_padding(Node::CodeInline(CodeInline{value: " `` ".into(), position: None}), RenderOptions::default(), "``` `` ```")] +#[case::code_inline_all_spaces_untouched(Node::CodeInline(CodeInline{value: " ".into(), position: None}), RenderOptions::default(), "` `")] +#[case::math_inline(Node::MathInline(MathInline{value: "x^2".into(), position: None}), RenderOptions::default(), "$x^2$")] +#[case::link(Node::Link(Link{url: Url::new(attr_keys::URL.to_string()), title: Some(Title::new(attr_keys::TITLE.to_string())), values: vec![attr_keys::VALUE.to_string().into()], position: None}), RenderOptions::default(), "[value](url \"title\")")] +#[case::link(Node::Link(Link{url: Url::new("".to_string()), title: None, values: vec![attr_keys::VALUE.to_string().into()], position: None}), RenderOptions::default(), "[value](<>)")] +#[case::link(Node::Link(Link{url: Url::new(attr_keys::URL.to_string()), title: None, values: vec![attr_keys::VALUE.to_string().into()], position: None}), RenderOptions::default(), "[value](url)")] +#[case::link_ref(Node::LinkRef(LinkRef{ident: "id".to_string(), values: vec!["id".to_string().into()], label: Some("id".to_string()), position: None}), RenderOptions::default(), "[id]")] +#[case::link_ref(Node::LinkRef(LinkRef{ident: "id".to_string(), values: vec!["open".to_string().into()], label: Some("id".to_string()), position: None}), RenderOptions::default(), "[open][id]")] +#[case::math(Node::Math(Math{value: "x^2".to_string(), position: None}), RenderOptions::default(), "$$\nx^2\n$$")] +#[case::strong(Node::Strong(Strong{values: vec!["test".to_string().into()], position: None}), RenderOptions::default(), "**test**")] +#[case::yaml(Node::Yaml(Yaml{value: "key: value".to_string(), position: None}), RenderOptions::default(), "---\nkey: value\n---")] +#[case::toml(Node::Toml(Toml{value: "key = \"value\"".to_string(), position: None}), RenderOptions::default(), "+++\nkey = \"value\"\n+++")] +#[case::break_(Node::Break(Break{position: None}), RenderOptions::default(), "\\\n")] +#[case::horizontal_rule(Node::HorizontalRule(HorizontalRule{ marker: None,position: None}), RenderOptions::default(), "***")] +#[case::mdx_jsx_flow_element(Node::MdxJsxFlowElement(MdxJsxFlowElement{ + name: Some("div".to_string()), + attributes: vec![ + MdxAttributeContent::Property(MdxJsxAttribute { + name: "className".into(), + value: Some(MdxAttributeValue::Literal("container".into())) + }) + ], + children: vec![ + "content".to_string().into() + ], + position: None +}), RenderOptions::default(), "<div className=\"container\">\n content\n</div>")] +#[case::mdx_jsx_flow_element(Node::MdxJsxFlowElement(MdxJsxFlowElement{ + name: Some("div".to_string()), + attributes: vec![ + MdxAttributeContent::Property(MdxJsxAttribute { + name: "className".into(), + value: Some(MdxAttributeValue::Literal("container".into())) + }) + ], + children: Vec::new(), + position: None +}), RenderOptions::default(), "<div className=\"container\" />")] +#[case::mdx_jsx_flow_element(Node::MdxJsxFlowElement(MdxJsxFlowElement{ + name: Some("div".to_string()), + attributes: Vec::new(), + children: Vec::new(), + position: None +}), RenderOptions::default(), "<div />")] +#[case::mdx_jsx_text_element(Node::MdxJsxTextElement(MdxJsxTextElement{ + name: Some("span".into()), + attributes: vec![ + MdxAttributeContent::Expression("...props".into()) + ], + children: vec![ + "inline".to_string().into() + ], + position: None +}), RenderOptions::default(), "<span {...props}>inline</span>")] +#[case::mdx_jsx_text_element(Node::MdxJsxTextElement(MdxJsxTextElement{ + name: Some("span".into()), + attributes: vec![ + MdxAttributeContent::Expression("...props".into()) + ], + children: vec![ + ], + position: None +}), RenderOptions::default(), "<span {...props} />")] +#[case::mdx_jsx_text_element(Node::MdxJsxTextElement(MdxJsxTextElement{ + name: Some("span".into()), + attributes: vec![ + ], + children: vec![ + ], + position: None +}), RenderOptions::default(), "<span />")] +#[case(Node::MdxTextExpression(MdxTextExpression{ + value: "count + 1".into(), + position: None, +}), RenderOptions::default(), "{count + 1}")] +#[case(Node::MdxJsEsm(MdxJsEsm{ + value: "import React from 'react'".into(), + position: None, +}), RenderOptions::default(), "import React from 'react'")] +#[case::fragment_empty(Node::Fragment(Fragment{values: vec![]}), RenderOptions::default(), "")] +#[case::fragment_single(Node::Fragment(Fragment{values: vec![ + Node::Text(Text{value: "hello".to_string(), position: None}) +]}), RenderOptions::default(), "hello")] +#[case::fragment_multiple(Node::Fragment(Fragment{values: vec![ + Node::Text(Text{value: "hello".to_string(), position: None}), + Node::Text(Text{value: "world".to_string(), position: None}) +]}), RenderOptions::default(), "hello\nworld")] +#[case::fragment_filters_empty(Node::Fragment(Fragment{values: vec![ + Node::Text(Text{value: "hello".to_string(), position: None}), + Node::Empty, + Node::Text(Text{value: "world".to_string(), position: None}) +]}), RenderOptions::default(), "hello\nworld")] +#[case::fragment_all_empty(Node::Fragment(Fragment{values: vec![ + Node::Empty, + Node::Empty, +]}), RenderOptions::default(), "")] +#[case::fragment_empty_text_as_blank_line(Node::Fragment(Fragment{values: vec![ + Node::Text(Text{value: "hello".to_string(), position: None}), + Node::Text(Text{value: "".to_string(), position: None}), + Node::Text(Text{value: "world".to_string(), position: None}) +]}), RenderOptions::default(), "hello\n\nworld")] +#[case::fragment_filters_nested_empty_fragment(Node::Fragment(Fragment{values: vec![ + Node::Fragment(Fragment{values: vec![Node::Empty, Node::Empty]}), + Node::Text(Text{value: "hello".to_string(), position: None}), + Node::Fragment(Fragment{values: vec![Node::Empty]}), +]}), RenderOptions::default(), "hello")] +#[case::fragment_filters_deeply_nested_empty_fragment(Node::Fragment(Fragment{values: vec![ + Node::Fragment(Fragment{values: vec![ + Node::Fragment(Fragment{values: vec![Node::Empty]}), + Node::Empty, + ]}), + Node::Text(Text{value: "hello".to_string(), position: None}), +]}), RenderOptions::default(), "hello")] +#[case::fragment_keeps_nested_fragment_with_blank_line_text(Node::Fragment(Fragment{values: vec![ + Node::Fragment(Fragment{values: vec![Node::Empty]}), + Node::Text(Text{value: "hello".to_string(), position: None}), + Node::Fragment(Fragment{values: vec![Node::Text(Text{value: "".to_string(), position: None})]}), + Node::Text(Text{value: "world".to_string(), position: None}), +]}), RenderOptions::default(), "hello\n\nworld")] +#[cfg_attr(feature = "wikilink", case::wikilink(Node::WikiLink(WikiLink{target: "target".to_string(), text: None, position: None}), RenderOptions::default(), "[[target]]"))] +#[cfg_attr(feature = "wikilink", case::wikilink_with_text(Node::WikiLink(WikiLink{target: "target".to_string(), text: Some("display text".to_string()), position: None}), RenderOptions::default(), "[[target|display text]]"))] +#[cfg_attr(feature = "wikilink", case::wikilink_same_text(Node::WikiLink(WikiLink{target: "target".to_string(), text: Some("target".to_string()), position: None}), RenderOptions::default(), "[[target]]"))] +fn test_to_string_with(#[case] node: Node, #[case] options: RenderOptions, #[case] expected: &str) { + assert_eq!(node.to_string_with(&options), expected); +} + +#[test] +fn test_node_partial_ord() { + let node1 = Node::Text(Text { + value: "test1".to_string(), + position: Some(Position { + start: Point { line: 1, column: 1 }, + end: Point { line: 1, column: 5 }, + }), + }); + + let node2 = Node::Text(Text { + value: "test2".to_string(), + position: Some(Position { + start: Point { line: 1, column: 6 }, + end: Point { line: 1, column: 10 }, + }), + }); + + let node3 = Node::Text(Text { + value: "test3".to_string(), + position: Some(Position { + start: Point { line: 2, column: 1 }, + end: Point { line: 2, column: 5 }, + }), + }); + + assert_eq!(node1.partial_cmp(&node2), Some(std::cmp::Ordering::Less)); + assert_eq!(node2.partial_cmp(&node1), Some(std::cmp::Ordering::Greater)); + + assert_eq!(node1.partial_cmp(&node3), Some(std::cmp::Ordering::Less)); + assert_eq!(node3.partial_cmp(&node1), Some(std::cmp::Ordering::Greater)); + + let node4 = Node::Text(Text { + value: "test4".to_string(), + position: None, + }); + + assert_eq!(node1.partial_cmp(&node4), Some(std::cmp::Ordering::Less)); + assert_eq!(node4.partial_cmp(&node1), Some(std::cmp::Ordering::Greater)); + + let node5 = Node::Text(Text { + value: "test5".to_string(), + position: None, + }); + + assert_eq!(node4.partial_cmp(&node5), Some(std::cmp::Ordering::Equal)); + + let node6 = Node::Code(Code { + value: "code".to_string(), + lang: None, + fence: true, + meta: None, + position: None, + }); + + assert_eq!(node6.partial_cmp(&node4), Some(std::cmp::Ordering::Less)); + assert_eq!(node4.partial_cmp(&node6), Some(std::cmp::Ordering::Greater)); +} + +#[rstest] +#[case(Node::Blockquote(Blockquote{values: Vec::new(), position: None}), "blockquote")] +#[case(Node::Break(Break{position: None}), "break")] +#[case(Node::Definition(Definition{ident: "".to_string(), url: Url::new("".to_string()), title: None, label: None, position: None}), "definition")] +#[case(Node::Delete(Delete{values: Vec::new(), position: None}), "delete")] +#[case(Node::Heading(Heading{depth: 1, values: Vec::new(), position: None}), "h1")] +#[case(Node::Heading(Heading{depth: 2, values: Vec::new(), position: None}), "h2")] +#[case(Node::Heading(Heading{depth: 3, values: Vec::new(), position: None}), "h3")] +#[case(Node::Heading(Heading{depth: 4, values: Vec::new(), position: None}), "h4")] +#[case(Node::Heading(Heading{depth: 5, values: Vec::new(), position: None}), "h5")] +#[case(Node::Heading(Heading{depth: 6, values: Vec::new(), position: None}), "h6")] +#[case(Node::Heading(Heading{depth: 7, values: Vec::new(), position: None}), "h")] +#[case(Node::Emphasis(Emphasis{values: Vec::new(), position: None}), "emphasis")] +#[case(Node::Footnote(Footnote{ident: "".to_string(), values: Vec::new(), position: None}), "footnote")] +#[case(Node::FootnoteRef(FootnoteRef{ident: "".to_string(), label: None, position: None}), "footnoteref")] +#[case(Node::Html(Html{value: "".to_string(), position: None}), "html")] +#[case(Node::Yaml(Yaml{value: "".to_string(), position: None}), "yaml")] +#[case(Node::Toml(Toml{value: "".to_string(), position: None}), "toml")] +#[case(Node::Image(Image{alt: "".to_string(), url: "".to_string(), title: None, position: None}), "image")] +#[case(Node::ImageRef(ImageRef{alt: "".to_string(), ident: "".to_string(), label: None, position: None}), "image_ref")] +#[case(Node::CodeInline(CodeInline{value: "".into(), position: None}), "code_inline")] +#[case(Node::MathInline(MathInline{value: "".into(), position: None}), "math_inline")] +#[case(Node::Link(Link{url: Url::new("".to_string()), title: None, values: Vec::new(), position: None}), "link")] +#[case(Node::LinkRef(LinkRef{ident: "".to_string(), values: Vec::new(), label: None, position: None}), "link_ref")] +#[case(Node::Math(Math{value: "".to_string(), position: None}), "math")] +#[case(Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: Vec::new(), position: None}), "list")] +#[case(Node::TableAlign(TableAlign{align: Vec::new(), position: None}), "table_align")] +#[case(Node::TableRow(TableRow{values: Vec::new(), position: None}), "table_row")] +#[case(Node::TableCell(TableCell{column: 0, row: 0, values: Vec::new(), position: None}), "table_cell")] +#[case(Node::Code(Code{value: "".to_string(), lang: None, fence: true, meta: None, position: None}), "code")] +#[case(Node::Strong(Strong{values: Vec::new(), position: None}), "strong")] +#[case(Node::HorizontalRule(HorizontalRule{ marker: None,position: None}), "Horizontal_rule")] +#[case(Node::MdxFlowExpression(MdxFlowExpression{value: "".into(), position: None}), "mdx_flow_expression")] +#[case(Node::MdxJsxFlowElement(MdxJsxFlowElement{name: None, attributes: Vec::new(), children: Vec::new(), position: None}), "mdx_jsx_flow_element")] +#[case(Node::MdxJsxTextElement(MdxJsxTextElement{name: None, attributes: Vec::new(), children: Vec::new(), position: None}), "mdx_jsx_text_element")] +#[case(Node::MdxTextExpression(MdxTextExpression{value: "".into(), position: None}), "mdx_text_expression")] +#[case(Node::MdxJsEsm(MdxJsEsm{value: "".into(), position: None}), "mdx_js_esm")] +#[case(Node::Text(Text{value: "".to_string(), position: None}), "text")] +fn test_name(#[case] node: Node, #[case] expected: &str) { + assert_eq!(node.name(), expected); +} + +#[rstest] +#[case(Node::Text(Text{value: "test".to_string(), position: None}), "test")] +#[case(Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: vec![Node::Text(Text{value: "test".to_string(), position: None})], position: None}), "test")] +#[case(Node::Blockquote(Blockquote{values: vec![Node::Text(Text{value: "test".to_string(), position: None})], position: None}), "test")] +#[case(Node::Delete(Delete{values: vec![Node::Text(Text{value: "test".to_string(), position: None})], position: None}), "test")] +#[case(Node::Heading(Heading{depth: 1, values: vec![Node::Text(Text{value: "test".to_string(), position: None})], position: None}), "test")] +#[case(Node::Emphasis(Emphasis{values: vec![Node::Text(Text{value: "test".to_string(), position: None})], position: None}), "test")] +#[case(Node::Footnote(Footnote{ident: "test".to_string(), values: vec![Node::Text(Text{value: "test".to_string(), position: None})], position: None}), "test")] +#[case(Node::FootnoteRef(FootnoteRef{ident: "test".to_string(), label: None, position: None}), "test")] +#[case(Node::Html(Html{value: "test".to_string(), position: None}), "test")] +#[case(Node::Yaml(Yaml{value: "test".to_string(), position: None}), "test")] +#[case(Node::Toml(Toml{value: "test".to_string(), position: None}), "test")] +#[case(Node::Image(Image{alt: attr_keys::ALT.to_string(), url: "test".to_string(), title: None, position: None}), "test")] +#[case(Node::ImageRef(ImageRef{alt: attr_keys::ALT.to_string(), ident: "test".to_string(), label: None, position: None}), "test")] +#[case(Node::CodeInline(CodeInline{value: "test".into(), position: None}), "test")] +#[case(Node::MathInline(MathInline{value: "test".into(), position: None}), "test")] +#[case(Node::Link(Link{url: Url::new("test".to_string()), title: None, values: Vec::new(), position: None}), "test")] +#[case(Node::LinkRef(LinkRef{ident: "test".to_string(), values: Vec::new(), label: None, position: None}), "test")] +#[case(Node::Math(Math{value: "test".to_string(), position: None}), "test")] +#[case(Node::Code(Code{value: "test".to_string(), lang: None, fence: true, meta: None, position: None}), "test")] +#[case(Node::Strong(Strong{values: vec![Node::Text(Text{value: "test".to_string(), position: None})], position: None}), "test")] +#[case(Node::TableCell(TableCell{column: 0, row: 0, values: vec![Node::Text(Text{value: "test".to_string(), position: None})], position: None}), "test")] +#[case(Node::TableRow(TableRow{values: vec![Node::TableCell(TableCell{column: 0, row: 0, values: vec![Node::Text(Text{value: "test".to_string(), position: None})], position: None})], position: None}), "test")] +#[case(Node::Break(Break{position: None}), "")] +#[case(Node::HorizontalRule(HorizontalRule{ marker: None,position: None}), "")] +#[case(Node::TableAlign(TableAlign{align: Vec::new(), position: None}), "")] +#[case(Node::MdxFlowExpression(MdxFlowExpression{value: "test".into(), position: None}), "test")] +#[case(Node::MdxTextExpression(MdxTextExpression{value: "test".into(), position: None}), "test")] +#[case(Node::MdxJsEsm(MdxJsEsm{value: "test".into(), position: None}), "test")] +#[case(Node::MdxJsxFlowElement(MdxJsxFlowElement{name: Some(attr_keys::NAME.to_string()), attributes: Vec::new(), children: vec![Node::Text(Text{value: "test".to_string(), position: None})], position: None}), "test")] +#[case(Node::Definition(Definition{ident: "test".to_string(), url: Url::new(attr_keys::URL.to_string()), title: None, label: None, position: None}), attr_keys::URL)] +#[case(Node::Fragment(Fragment {values: vec![Node::Text(Text{value: "test".to_string(), position: None})]}), "test")] +fn test_value(#[case] node: Node, #[case] expected: &str) { + assert_eq!(node.value(), expected); +} + +#[rstest] +#[case(Node::Text(Text{value: "test".to_string(), position: None}), None)] +#[case(Node::Text(Text{value: "test".to_string(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] +#[case(Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: Vec::new(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] +#[case(Node::Blockquote(Blockquote{values: Vec::new(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] +#[case(Node::Delete(Delete{values: Vec::new(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] +#[case(Node::Heading(Heading{depth: 1, values: Vec::new(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] +#[case(Node::Emphasis(Emphasis{values: Vec::new(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] +#[case(Node::Footnote(Footnote{ident: "".to_string(), values: Vec::new(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] +#[case(Node::FootnoteRef(FootnoteRef{ident: "".to_string(), label: None, position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] +#[case(Node::Html(Html{value: "".to_string(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] +#[case(Node::Yaml(Yaml{value: "".to_string(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] +#[case(Node::Toml(Toml{value: "".to_string(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] +#[case(Node::Image(Image{alt: "".to_string(), url: "".to_string(), title: None, position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] +#[case(Node::ImageRef(ImageRef{alt: "".to_string(), ident: "".to_string(), label: None, position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] +#[case(Node::CodeInline(CodeInline{value: "".into(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] +#[case(Node::MathInline(MathInline{value: "".into(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] +#[case(Node::Link(Link{url: Url("".to_string()), title: None, values: Vec::new(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] +#[case(Node::LinkRef(LinkRef{ident: "".to_string(), values: Vec::new(), label: None, position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] +#[case(Node::Math(Math{value: "".to_string(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] +#[case(Node::Code(Code{value: "".to_string(), lang: None, fence: true, meta: None, position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] +#[case(Node::Strong(Strong{values: Vec::new(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] +#[case(Node::TableCell(TableCell{column: 0, row: 0, values: Vec::new(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] +#[case(Node::TableRow(TableRow{values: Vec::new(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] +#[case(Node::TableAlign(TableAlign{align: Vec::new(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] +#[case(Node::Break(Break{position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] +#[case(Node::HorizontalRule(HorizontalRule{ marker: None,position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] +#[case(Node::MdxFlowExpression(MdxFlowExpression{value: "test".into(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] +#[case(Node::MdxTextExpression(MdxTextExpression{value: "test".into(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] +#[case(Node::MdxJsEsm(MdxJsEsm{value: "test".into(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] +#[case(Node::MdxJsxFlowElement(MdxJsxFlowElement{name: Some("div".to_string()), attributes: Vec::new(), children: Vec::new(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] +#[case(Node::MdxJsxTextElement(MdxJsxTextElement{name: Some("span".into()), attributes: Vec::new(), children: Vec::new(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] +#[case(Node::Definition(Definition{ident: "".to_string(), url: Url("".to_string()), title: None, label: None, position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] +#[case(Node::Fragment(Fragment{values: Vec::new()}), None)] +#[case(Node::Fragment(Fragment{values: vec![ + Node::Text(Text{value: "test1".to_string(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), + Node::Text(Text{value: "test2".to_string(), position: Some(Position{start: Point{line: 1, column: 6}, end: Point{line: 1, column: 10}})}) +]}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 10}}))] +#[case(Node::Fragment(Fragment{values: vec![ + Node::Text(Text{value: "test".to_string(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), + Node::Text(Text{value: "test2".to_string(), position: None}) +]}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] +#[case(Node::Fragment(Fragment{values: vec![ + Node::Text(Text{value: "test".to_string(), position: None}), + Node::Text(Text{value: "test2".to_string(), position: Some(Position{start: Point{line: 1, column: 6}, end: Point{line: 1, column: 10}})}) +]}), Some(Position{start: Point{line: 1, column: 6}, end: Point{line: 1, column: 10}}))] +#[case(Node::Fragment(Fragment{values: vec![ + Node::Text(Text{value: "test2".to_string(), position: Some(Position{start: Point{line: 1, column: 6}, end: Point{line: 1, column: 10}})}), + Node::Text(Text{value: "test".to_string(), position: None}) +]}), Some(Position{start: Point{line: 1, column: 6}, end: Point{line: 1, column: 10}}))] +#[case(Node::Fragment(Fragment{values: vec![ + Node::Text(Text{value: "test".to_string(), position: None}), + Node::Text(Text{value: "test2".to_string(), position: None}) +]}), None)] +#[case(Node::Empty, None)] +fn test_position(#[case] node: Node, #[case] expected: Option<Position>) { + assert_eq!(node.position(), expected); +} + +#[rstest] +#[case(Node::Blockquote(Blockquote{values: vec![ + Node::Text(Text{value: "first".to_string(), position: None}), + Node::Text(Text{value: "second".to_string(), position: None}) +], position: None}), 0, Some(Node::Text(Text{value: "first".to_string(), position: None})))] +#[case(Node::Blockquote(Blockquote{values: vec![ + Node::Text(Text{value: "first".to_string(), position: None}), + Node::Text(Text{value: "second".to_string(), position: None}) +], position: None}), 1, Some(Node::Text(Text{value: "second".to_string(), position: None})))] +#[case(Node::Blockquote(Blockquote{values: vec![ + Node::Text(Text{value: "first".to_string(), position: None}) +], position: None}), 1, None)] +#[case(Node::Delete(Delete{values: vec![ + Node::Text(Text{value: "first".to_string(), position: None}), + Node::Text(Text{value: "second".to_string(), position: None}) +], position: None}), 0, Some(Node::Text(Text{value: "first".to_string(), position: None})))] +#[case(Node::Emphasis(Emphasis{values: vec![ + Node::Text(Text{value: "first".to_string(), position: None}), + Node::Text(Text{value: "second".to_string(), position: None}) +], position: None}), 1, Some(Node::Text(Text{value: "second".to_string(), position: None})))] +#[case(Node::Strong(Strong{values: vec![ + Node::Text(Text{value: "first".to_string(), position: None}) +], position: None}), 0, Some(Node::Text(Text{value: "first".to_string(), position: None})))] +#[case(Node::Heading(Heading{depth: 1, values: vec![ + Node::Text(Text{value: "first".to_string(), position: None}), + Node::Text(Text{value: "second".to_string(), position: None}) +], position: None}), 0, Some(Node::Text(Text{value: "first".to_string(), position: None})))] +#[case(Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: vec![ + Node::Text(Text{value: "first".to_string(), position: None}), + Node::Text(Text{value: "second".to_string(), position: None}) +], position: None}), 1, Some(Node::Text(Text{value: "second".to_string(), position: None})))] +#[case(Node::TableCell(TableCell{column: 0, row: 0, values: vec![ + Node::Text(Text{value: "cell content".to_string(), position: None}) +], position: None}), 0, Some(Node::Text(Text{value: "cell content".to_string(), position: None})))] +#[case(Node::TableRow(TableRow{values: vec![ + Node::TableCell(TableCell{column: 0, row: 0, values: Vec::new(), position: None}), + Node::TableCell(TableCell{column: 1, row: 0, values: Vec::new(), position: None}) +], position: None}), 1, Some(Node::TableCell(TableCell{column: 1, row: 0, values: Vec::new(), position: None})))] +#[case(Node::Text(Text{value: "plain text".to_string(), position: None}), 0, None)] +#[case(Node::Code(Code{value: "code".to_string(), lang: None, fence: true, meta: None, position: None}), 0, None)] +#[case(Node::Html(Html{value: "<div>".to_string(), position: None}), 0, None)] +fn test_find_at_index(#[case] node: Node, #[case] index: usize, #[case] expected: Option<Node>) { + assert_eq!(node.find_at_index(index), expected); +} + +#[rstest] +#[case(Node::Blockquote(Blockquote{values: vec!["test".to_string().into()], position: None}), + Node::Fragment(Fragment{values: vec!["test".to_string().into()]}))] +#[case(Node::Delete(Delete{values: vec!["test".to_string().into()], position: None}), + Node::Fragment(Fragment{values: vec!["test".to_string().into()]}))] +#[case(Node::Heading(Heading{depth: 1, values: vec!["test".to_string().into()], position: None}), + Node::Fragment(Fragment{values: vec!["test".to_string().into()]}))] +#[case(Node::Emphasis(Emphasis{values: vec!["test".to_string().into()], position: None}), + Node::Fragment(Fragment{values: vec!["test".to_string().into()]}))] +#[case(Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: vec!["test".to_string().into()], position: None}), + Node::Fragment(Fragment{values: vec!["test".to_string().into()]}))] +#[case(Node::Strong(Strong{values: vec!["test".to_string().into()], position: None}), + Node::Fragment(Fragment{values: vec!["test".to_string().into()]}))] +#[case(Node::Link(Link{url: Url(attr_keys::URL.to_string()), title: None, values: vec!["test".to_string().into()], position: None}), + Node::Fragment(Fragment{values: vec!["test".to_string().into()]}))] +#[case(Node::LinkRef(LinkRef{ident: "id".to_string(), values: vec!["test".to_string().into()], label: None, position: None}), + Node::Fragment(Fragment{values: vec!["test".to_string().into()]}))] +#[case(Node::Footnote(Footnote{ident: "id".to_string(), values: vec!["test".to_string().into()], position: None}), + Node::Fragment(Fragment{values: vec!["test".to_string().into()]}))] +#[case(Node::TableCell(TableCell{column: 0, row: 0, values: vec!["test".to_string().into()], position: None}), + Node::Fragment(Fragment{values: vec!["test".to_string().into()]}))] +#[case(Node::TableRow(TableRow{values: vec!["test".to_string().into()], position: None}), + Node::Fragment(Fragment{values: vec!["test".to_string().into()]}))] +#[case(Node::Fragment(Fragment{values: vec!["test".to_string().into()]}), + Node::Fragment(Fragment{values: vec!["test".to_string().into()]}))] +#[case(Node::Text(Text{value: "test".to_string(), position: None}), + Node::Empty)] +#[case(Node::Code(Code{value: "test".to_string(), lang: None, fence: true, meta: None, position: None}), + Node::Empty)] +#[case(Node::Image(Image{alt: attr_keys::ALT.to_string(), url: attr_keys::URL.to_string(), title: None, position: None}), + Node::Empty)] +#[case(Node::Empty, Node::Empty)] +fn test_to_fragment(#[case] node: Node, #[case] expected: Node) { + assert_eq!(node.clone().to_fragment(), expected); + assert_eq!(node.into_fragment(), expected); +} + +// Regression coverage for the 0.6.2 blank-line bug: `eval_markdown_node` +// replaces a non-matching `select()` result with `child_node.to_fragment()`, +// not `Node::Empty` directly. For container nodes this yields a `Fragment` +// wrapping the (now `Empty`) children, which must still render to "" with +// no stray blank lines, while a partially-matching container must render +// only the surviving content. +#[rstest] +#[case::blockquote_all_empty(Node::Blockquote(Blockquote{values: vec![Node::Empty, Node::Empty], position: None}), "")] +#[case::blockquote_mixed(Node::Blockquote(Blockquote{values: vec![Node::Empty, Node::Text(Text{value: "kept".to_string(), position: None})], position: None}), "kept")] +#[case::delete_all_empty(Node::Delete(Delete{values: vec![Node::Empty, Node::Empty], position: None}), "")] +#[case::delete_mixed(Node::Delete(Delete{values: vec![Node::Text(Text{value: "kept".to_string(), position: None}), Node::Empty], position: None}), "kept")] +#[case::heading_all_empty(Node::Heading(Heading{depth: 1, values: vec![Node::Empty, Node::Empty], position: None}), "")] +#[case::heading_mixed(Node::Heading(Heading{depth: 1, values: vec![Node::Empty, Node::Text(Text{value: "kept".to_string(), position: None})], position: None}), "kept")] +#[case::emphasis_all_empty(Node::Emphasis(Emphasis{values: vec![Node::Empty, Node::Empty], position: None}), "")] +#[case::emphasis_mixed(Node::Emphasis(Emphasis{values: vec![Node::Empty, Node::Text(Text{value: "kept".to_string(), position: None})], position: None}), "kept")] +#[case::list_all_empty(Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: vec![Node::Empty, Node::Empty], position: None}), "")] +#[case::list_mixed(Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: vec![Node::Empty, Node::Text(Text{value: "kept".to_string(), position: None})], position: None}), "kept")] +#[case::strong_all_empty(Node::Strong(Strong{values: vec![Node::Empty, Node::Empty], position: None}), "")] +#[case::strong_mixed(Node::Strong(Strong{values: vec![Node::Text(Text{value: "kept".to_string(), position: None}), Node::Empty], position: None}), "kept")] +#[case::link_all_empty(Node::Link(Link{url: Url(attr_keys::URL.to_string()), title: None, values: vec![Node::Empty, Node::Empty], position: None}), "")] +#[case::link_mixed(Node::Link(Link{url: Url(attr_keys::URL.to_string()), title: None, values: vec![Node::Empty, Node::Text(Text{value: "kept".to_string(), position: None})], position: None}), "kept")] +#[case::link_ref_all_empty(Node::LinkRef(LinkRef{ident: "id".to_string(), values: vec![Node::Empty, Node::Empty], label: None, position: None}), "")] +#[case::link_ref_mixed(Node::LinkRef(LinkRef{ident: "id".to_string(), values: vec![Node::Empty, Node::Text(Text{value: "kept".to_string(), position: None})], label: None, position: None}), "kept")] +#[case::footnote_all_empty(Node::Footnote(Footnote{ident: "id".to_string(), values: vec![Node::Empty, Node::Empty], position: None}), "")] +#[case::footnote_mixed(Node::Footnote(Footnote{ident: "id".to_string(), values: vec![Node::Empty, Node::Text(Text{value: "kept".to_string(), position: None})], position: None}), "kept")] +#[case::table_cell_all_empty(Node::TableCell(TableCell{column: 0, row: 0, values: vec![Node::Empty, Node::Empty], position: None}), "")] +#[case::table_cell_mixed(Node::TableCell(TableCell{column: 0, row: 0, values: vec![Node::Empty, Node::Text(Text{value: "kept".to_string(), position: None})], position: None}), "kept")] +fn test_to_fragment_then_render_skips_non_matching(#[case] node: Node, #[case] expected: &str) { + assert_eq!(node.to_fragment().to_string_with(&RenderOptions::default()), expected); +} + +#[rstest] +#[case( + &mut Node::Blockquote(Blockquote{values: vec![ + Node::Text(Text{value: "old".to_string(), position: None}) + ], position: None}), + Node::Fragment(Fragment{values: vec![ + Node::Text(Text{value: "new".to_string(), position: None}) + ]}), + Node::Blockquote(Blockquote{values: vec![ + Node::Text(Text{value: "new".to_string(), position: None}) + ], position: None}) +)] +#[case( + &mut Node::Delete(Delete{values: vec![ + Node::Text(Text{value: "old".to_string(), position: None}) + ], position: None}), + Node::Fragment(Fragment{values: vec![ + Node::Text(Text{value: "new".to_string(), position: None}) + ]}), + Node::Delete(Delete{values: vec![ + Node::Text(Text{value: "new".to_string(), position: None}) + ], position: None}) +)] +#[case( + &mut Node::Emphasis(Emphasis{values: vec![ + Node::Text(Text{value: "old".to_string(), position: None}) + ], position: None}), + Node::Fragment(Fragment{values: vec![ + Node::Text(Text{value: "new".to_string(), position: None}) + ]}), + Node::Emphasis(Emphasis{values: vec![ + Node::Text(Text{value: "new".to_string(), position: None}) + ], position: None}) +)] +#[case( + &mut Node::Strong(Strong{values: vec![ + Node::Text(Text{value: "old".to_string(), position: None}) + ], position: None}), + Node::Fragment(Fragment{values: vec![ + Node::Text(Text{value: "new".to_string(), position: None}) + ]}), + Node::Strong(Strong{values: vec![ + Node::Text(Text{value: "new".to_string(), position: None}) + ], position: None}) +)] +#[case( + &mut Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: vec![ + Node::Text(Text{value: "old".to_string(), position: None}) + ], position: None}), + Node::Fragment(Fragment{values: vec![ + Node::Text(Text{value: "new".to_string(), position: None}) + ]}), + Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: vec![ + Node::Text(Text{value: "new".to_string(), position: None}) + ], position: None}) +)] +#[case( + &mut Node::Heading(Heading{depth: 1, values: vec![ + Node::Text(Text{value: "old".to_string(), position: None}) + ], position: None}), + Node::Fragment(Fragment{values: vec![ + Node::Text(Text{value: "new".to_string(), position: None}) + ]}), + Node::Heading(Heading{depth: 1, values: vec![ + Node::Text(Text{value: "new".to_string(), position: None}) + ], position: None}) +)] +#[case( + &mut Node::Link(Link{url: Url(attr_keys::URL.to_string()), title: None, values: vec![ + Node::Text(Text{value: "old".to_string(), position: None}) + ], position: None}), + Node::Fragment(Fragment{values: vec![ + Node::Text(Text{value: "new".to_string(), position: None}) + ]}), + Node::Link(Link{url: Url(attr_keys::URL.to_string()), title: None, values: vec![ + Node::Text(Text{value: "new".to_string(), position: None}) + ], position: None}) +)] +#[case( + &mut Node::LinkRef(LinkRef{ident: "id".to_string(), values: vec![ + Node::Text(Text{value: "old".to_string(), position: None}) + ], label: None, position: None}), + Node::Fragment(Fragment{values: vec![ + Node::Text(Text{value: "new".to_string(), position: None}) + ]}), + Node::LinkRef(LinkRef{ident: "id".to_string(), values: vec![ + Node::Text(Text{value: "new".to_string(), position: None}) + ], label: None, position: None}) +)] +#[case( + &mut Node::Footnote(Footnote{ident: "id".to_string(), values: vec![ + Node::Text(Text{value: "old".to_string(), position: None}) + ], position: None}), + Node::Fragment(Fragment{values: vec![ + Node::Text(Text{value: "new".to_string(), position: None}) + ]}), + Node::Footnote(Footnote{ident: "id".to_string(), values: vec![ + Node::Text(Text{value: "new".to_string(), position: None}) + ], position: None}) +)] +#[case( + &mut Node::TableCell(TableCell{column: 0, row: 0, values: vec![ + Node::Text(Text{value: "old".to_string(), position: None}) + ], position: None}), + Node::Fragment(Fragment{values: vec![ + Node::Text(Text{value: "new".to_string(), position: None}) + ]}), + Node::TableCell(TableCell{column: 0, row: 0, values: vec![ + Node::Text(Text{value: "new".to_string(), position: None}) + ], position: None}) +)] +#[case( + &mut Node::TableRow(TableRow{values: vec![ + Node::TableCell(TableCell{column: 0, row: 0, values: vec![ + Node::Text(Text{value: "old".to_string(), position: None}) + ], position: None}) + ], position: None}), + Node::Fragment(Fragment{values: vec![ + Node::TableCell(TableCell{column: 0, row: 0, values: vec![ + Node::Text(Text{value: "new".to_string(), position: None}) + ], position: None}) + ]}), + Node::TableRow(TableRow{values: vec![ + Node::TableCell(TableCell{column: 0, row: 0, values: vec![ + Node::Text(Text{value: "new".to_string(), position: None}) + ], position: None}) + ], position: None}) +)] +#[case( + &mut Node::Text(Text{value: "old".to_string(), position: None}), + Node::Fragment(Fragment{values: vec![ + Node::Text(Text{value: "new".to_string(), position: None}) + ]}), + Node::Text(Text{value: "old".to_string(), position: None}) +)] +#[case( + &mut Node::Blockquote(Blockquote{values: vec![ + Node::Text(Text{value: "text1".to_string(), position: None}), + Node::Text(Text{value: "text2".to_string(), position: None}) + ], position: None}), + Node::Fragment(Fragment{values: vec![ + Node::Text(Text{value: "new1".to_string(), position: None}), + Node::Text(Text{value: "new2".to_string(), position: None}) + ]}), + Node::Blockquote(Blockquote{values: vec![ + Node::Text(Text{value: "new1".to_string(), position: None}), + Node::Text(Text{value: "new2".to_string(), position: None}) + ], position: None}) +)] +#[case( + &mut Node::Strong(Strong{values: vec![ + Node::Text(Text{value: "text1".to_string(), position: None}), + Node::Text(Text{value: "text2".to_string(), position: None}) + ], position: None}), + Node::Fragment(Fragment{values: vec![ + Node::Empty, + Node::Text(Text{value: "new2".to_string(), position: None}) + ]}), + Node::Strong(Strong{values: vec![ + Node::Text(Text{value: "text1".to_string(), position: None}), + Node::Text(Text{value: "new2".to_string(), position: None}) + ], position: None}) +)] +#[case( + &mut Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: vec![ + Node::Text(Text{value: "text1".to_string(), position: None}), + Node::Text(Text{value: "text2".to_string(), position: None}) + ], position: None}), + Node::Fragment(Fragment{values: vec![ + Node::Text(Text{value: "new1".to_string(), position: None}), + Node::Fragment(Fragment{values: Vec::new()}) + ]}), + Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: vec![ + Node::Text(Text{value: "new1".to_string(), position: None}), + Node::Text(Text{value: "text2".to_string(), position: None}) + ], position: None}) +)] +fn test_apply_fragment(#[case] node: &mut Node, #[case] fragment: Node, #[case] expected: Node) { + node.apply_fragment(fragment); + assert_eq!(*node, expected); +} + +#[rstest] +#[case(Node::Text(Text{value: "test".to_string(), position: None}), + Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}, + Node::Text(Text{value: "test".to_string(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}))] +#[case(Node::Code(Code{value: "code".to_string(), lang: None, fence: true, meta: None, position: None}), + Position{start: Point{line: 1, column: 1}, end: Point{line: 3, column: 3}}, + Node::Code(Code{value: "code".to_string(), lang: None, fence: true, meta: None, position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 3, column: 3}})}))] +#[case(Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 1, checked: None, ordered: false, values: vec![], position: None}), + Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}, + Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 1, checked: None, ordered: false, values: vec![], position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}))] +#[case(Node::Definition(Definition{ident: "id".to_string(), url: Url::new(attr_keys::URL.to_string()), title: None, label: None, position: None}), + Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 10}}, + Node::Definition(Definition{ident: "id".to_string(), url: Url::new(attr_keys::URL.to_string()), title: None, label: None, position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 10}})}))] +#[case(Node::Delete(Delete{values: vec![Node::Text(Text{value: "test".to_string(), position: None})], position: None}), + Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}, + Node::Delete(Delete{values: vec![Node::Text(Text{value: "test".to_string(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})})], position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}))] +#[case(Node::Emphasis(Emphasis{values: vec![Node::Text(Text{value: "test".to_string(), position: None})], position: None}), + Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}, + Node::Emphasis(Emphasis{values: vec![Node::Text(Text{value: "test".to_string(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})})], position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}))] +#[case(Node::Footnote(Footnote{ident: "id".to_string(), values: vec![Node::Text(Text{value: "test".to_string(), position: None})], position: None}), + Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}, + Node::Footnote(Footnote{ident: "id".to_string(), values: vec![Node::Text(Text{value: "test".to_string(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})})], position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}))] +#[case(Node::FootnoteRef(FootnoteRef{ident: "id".to_string(), label: Some(attr_keys::LABEL.to_string()), position: None}), + Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}, + Node::FootnoteRef(FootnoteRef{ident: "id".to_string(), label: Some(attr_keys::LABEL.to_string()), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}))] +#[case(Node::Html(Html{value: "<div>test</div>".to_string(), position: None}), + Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 15}}, + Node::Html(Html{value: "<div>test</div>".to_string(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 15}})}))] +#[case(Node::Yaml(Yaml{value: "key: value".to_string(), position: None}), + Position{start: Point{line: 1, column: 1}, end: Point{line: 3, column: 4}}, + Node::Yaml(Yaml{value: "key: value".to_string(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 3, column: 4}})}))] +#[case(Node::Toml(Toml{value: "key = \"value\"".to_string(), position: None}), + Position{start: Point{line: 1, column: 1}, end: Point{line: 3, column: 4}}, + Node::Toml(Toml{value: "key = \"value\"".to_string(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 3, column: 4}})}))] +#[case(Node::Image(Image{alt: attr_keys::ALT.to_string(), url: attr_keys::URL.to_string(), title: None, position: None}), + Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 12}}, + Node::Image(Image{alt: attr_keys::ALT.to_string(), url: attr_keys::URL.to_string(), title: None, position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 12}})}))] +#[case(Node::ImageRef(ImageRef{alt: attr_keys::ALT.to_string(), ident: "id".to_string(), label: None, position: None}), + Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 10}}, + Node::ImageRef(ImageRef{alt: attr_keys::ALT.to_string(), ident: "id".to_string(), label: None, position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 10}})}))] +#[case(Node::CodeInline(CodeInline{value: "code".into(), position: None}), + Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 7}}, + Node::CodeInline(CodeInline{value: "code".into(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 7}})}))] +#[case(Node::MathInline(MathInline{value: "x^2".into(), position: None}), + Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}, + Node::MathInline(MathInline{value: "x^2".into(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}))] +#[case(Node::Link(Link{url: Url::new(attr_keys::URL.to_string()), title: None, values: vec![Node::Text(Text{value: "text".to_string(), position: None})], position: None}), + Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 10}}, + Node::Link(Link{url: Url::new(attr_keys::URL.to_string()), title: None, values: vec![Node::Text(Text{value: "text".to_string(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 10}})})], position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 10}})}))] +#[case(Node::LinkRef(LinkRef{ident: "id".to_string(), values: vec![Node::Text(Text{value: "text".to_string(), position: None})], label: None, position: None}), + Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 10}}, + Node::LinkRef(LinkRef{ident: "id".to_string(), values: vec![Node::Text(Text{value: "text".to_string(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 10}})})], label: None, position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 10}})}))] +#[case(Node::Math(Math{value: "x^2".to_string(), position: None}), + Position{start: Point{line: 1, column: 1}, end: Point{line: 3, column: 3}}, + Node::Math(Math{value: "x^2".to_string(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 3, column: 3}})}))] +#[case(Node::TableCell(TableCell{column: 0, row: 0, values: vec![Node::Text(Text{value: "cell".to_string(), position: None})], position: None}), + Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 6}}, + Node::TableCell(TableCell{column: 0, row: 0, values: vec![Node::Text(Text{value: "cell".to_string(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 6}})})], position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 6}})}))] +#[case(Node::TableAlign(TableAlign{align: vec![TableAlignKind::Left, TableAlignKind::Right], position: None}), + Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 15}}, + Node::TableAlign(TableAlign{align: vec![TableAlignKind::Left, TableAlignKind::Right], position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 15}})}))] +#[case(Node::MdxFlowExpression(MdxFlowExpression{value: "test".into(), position: None}), + Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 7}}, + Node::MdxFlowExpression(MdxFlowExpression{value: "test".into(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 7}})}))] +#[case(Node::MdxTextExpression(MdxTextExpression{value: "test".into(), position: None}), + Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 7}}, + Node::MdxTextExpression(MdxTextExpression{value: "test".into(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 7}})}))] +#[case(Node::MdxJsEsm(MdxJsEsm{value: "import React from 'react'".into(), position: None}), + Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 25}}, + Node::MdxJsEsm(MdxJsEsm{value: "import React from 'react'".into(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 25}})}))] +#[case(Node::MdxJsxTextElement(MdxJsxTextElement{name: Some("span".into()), attributes: Vec::new(), children: vec![Node::Text(Text{value: "text".to_string(), position: None})], position: None}), + Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 20}}, + Node::MdxJsxTextElement(MdxJsxTextElement{name: Some("span".into()), attributes: Vec::new(), children: vec![Node::Text(Text{value: "text".to_string(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 20}})})], position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 20}})}))] +#[case(Node::Break(Break{position: None}), + Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 2}}, + Node::Break(Break{position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 2}})}))] +#[case(Node::Empty, + Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}, + Node::Empty)] +#[case(Node::Fragment(Fragment{values: vec![ + Node::Text(Text{value: "test1".to_string(), position: None}), + Node::Text(Text{value: "test2".to_string(), position: None}) + ]}), + Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 10}}, + Node::Fragment(Fragment{values: vec![ + Node::Text(Text{value: "test1".to_string(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 10}})}), + Node::Text(Text{value: "test2".to_string(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 10}})}) + ]}))] +#[case(Node::Blockquote(Blockquote{values: vec![ + Node::Text(Text{value: "test".to_string(), position: None})], position: None}), + Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}, + Node::Blockquote(Blockquote{values: vec![ + Node::Text(Text{value: "test".to_string(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}) + ], position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}))] +#[case(Node::Heading(Heading{depth: 1, values: vec![ + Node::Text(Text{value: "test".to_string(), position: None})], position: None}), + Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}, + Node::Heading(Heading{depth: 1, values: vec![ + Node::Text(Text{value: "test".to_string(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}) + ], position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}))] +#[case(Node::Strong(Strong{values: vec![ + Node::Text(Text{value: "test".to_string(), position: None})], position: None}), + Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}, + Node::Strong(Strong{values: vec![ + Node::Text(Text{value: "test".to_string(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}) + ], position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}))] +#[case(Node::TableRow(TableRow{values: vec![ + Node::TableCell(TableCell{column: 0, row: 0, values: vec![ + Node::Text(Text{value: "cell".to_string(), position: None}) + ], position: None}) + ], position: None}), + Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 10}}, + Node::TableRow(TableRow{values: vec![ + Node::TableCell(TableCell{column: 0, row: 0, values: vec![ + Node::Text(Text{value: "cell".to_string(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 10}})}) + ], position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 10}})}) + ], position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 10}})}))] +#[case(Node::MdxJsxFlowElement(MdxJsxFlowElement{ + name: Some("div".to_string()), + attributes: Vec::new(), + children: vec![Node::Text(Text{value: "content".to_string(), position: None})], + position: None + }), + Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 20}}, + Node::MdxJsxFlowElement(MdxJsxFlowElement{ + name: Some("div".to_string()), + attributes: Vec::new(), + children: vec![Node::Text(Text{value: "content".to_string(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 20}})})], + position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 20}}) + }))] +fn test_set_position(#[case] mut node: Node, #[case] position: Position, #[case] expected: Node) { + node.set_position(Some(position)); + assert_eq!(node, expected); +} + +fn some_position() -> Option<Position> { + Some(Position { + start: Point { line: 1, column: 1 }, + end: Point { line: 1, column: 5 }, + }) +} + +#[test] +fn test_strip_positions_top_level() { + let mut node = Node::Text(Text { + value: "test".to_string(), + position: some_position(), + }); + node.strip_positions(); + assert_eq!(node.position(), None); +} + +#[test] +fn test_strip_positions_recurses_into_children() { + let mut node = Node::Blockquote(Blockquote { + values: vec![Node::Strong(Strong { + values: vec![Node::Text(Text { + value: "test".to_string(), + position: some_position(), + })], + position: some_position(), + })], + position: some_position(), + }); + + node.strip_positions(); + + assert_eq!(node.position(), None); + let strong = &node.children()[0]; + assert_eq!(strong.position(), None); + assert_eq!(strong.children()[0].position(), None); +} + +#[test] +fn test_clear_text_position_at_leaf_only() { + let mut node = Node::Heading(Heading { + depth: 1, + values: vec![Node::Text(Text { + value: "title".to_string(), + position: some_position(), + })], + position: some_position(), + }); + + node.clear_text_position_at(0); + + assert_eq!(node.position(), some_position(), "outer node keeps its position"); + assert_eq!(node.children()[0].position(), None, "escaped leaf loses its position"); +} + +#[test] +fn test_clear_text_position_at_selected_index() { + let mut node = Node::List(List { + marker: None, + start: None, + spread: false, + index: 0, + level: 0, + checked: None, + ordered: false, + values: vec![ + Node::Text(Text { + value: "a".to_string(), + position: some_position(), + }), + Node::Text(Text { + value: "b".to_string(), + position: some_position(), + }), + ], + position: some_position(), + }); + + node.clear_text_position_at(1); + + assert_eq!( + node.children()[0].position(), + some_position(), + "untouched sibling keeps its position" + ); + assert_eq!(node.children()[1].position(), None, "selected leaf loses its position"); +} + +#[rstest] +#[case(Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: vec!["test".to_string().into()], position: None}), true)] +#[case(Node::List(List{ marker: None,start: None, spread: false, index: 1, level: 2, checked: Some(true), ordered: false, values: vec!["test".to_string().into()], position: None}), true)] +#[case(Node::Text(Text{value: "test".to_string(), position: None}), false)] +fn test_is_list(#[case] node: Node, #[case] expected: bool) { + assert_eq!(node.is_list(), expected); +} + +#[rstest] +#[case(Url::new("https://example.com".to_string()), RenderOptions{link_url_style: UrlSurroundStyle::None, ..Default::default()}, "https://example.com")] +#[case(Url::new("https://example.com".to_string()), RenderOptions{link_url_style: UrlSurroundStyle::Angle, ..Default::default()}, "<https://example.com>")] +#[case(Url::new("".to_string()), RenderOptions::default(), "<>")] +fn test_url_to_string_with(#[case] url: Url, #[case] options: RenderOptions, #[case] expected: &str) { + assert_eq!(url.to_string_with(&options), expected); +} + +#[rstest] +#[case(Title::new(attr_keys::TITLE.to_string()), RenderOptions::default(), "\"title\"")] +#[case(Title::new(r#"title with "quotes""#.to_string()), RenderOptions::default(), r#""title with \"quotes\"""#)] +#[case(Title::new("title with spaces".to_string()), RenderOptions::default(), "\"title with spaces\"")] +#[case(Title::new("".to_string()), RenderOptions::default(), "\"\"")] +#[case(Title::new(attr_keys::TITLE.to_string()), RenderOptions{link_title_style: TitleSurroundStyle::Single, ..Default::default()}, "'title'")] +#[case(Title::new("title with 'quotes'".to_string()), RenderOptions{link_title_style: TitleSurroundStyle::Double, ..Default::default()}, "\"title with 'quotes'\"")] +#[case(Title::new(attr_keys::TITLE.to_string()), RenderOptions{link_title_style: TitleSurroundStyle::Paren, ..Default::default()}, "(title)")] +fn test_title_to_string_with(#[case] title: Title, #[case] options: RenderOptions, #[case] expected: &str) { + assert_eq!(title.to_string_with(&options), expected); +} + +#[rstest] +#[case(Node::Fragment(Fragment{values: vec![]}), true)] +#[case(Node::Fragment(Fragment{values: vec![ + Node::Text(Text{value: "not_empty".to_string(), position: None}) +]}), false)] +#[case(Node::Fragment(Fragment{values: vec![ + Node::Fragment(Fragment{values: vec![]}), + Node::Fragment(Fragment{values: vec![]}) +]}), true)] +#[case(Node::Fragment(Fragment{values: vec![ + Node::Fragment(Fragment{values: vec![]}), + Node::Text(Text{value: "not_empty".to_string(), position: None}) +]}), false)] +#[case(Node::Text(Text{value: "not_fragment".to_string(), position: None}), false)] +fn test_is_empty_fragment(#[case] node: Node, #[case] expected: bool) { + assert_eq!(node.is_empty_fragment(), expected); +} + +#[rstest] +#[case::footnote(Node::Footnote(Footnote{ident: "id".to_string(), values: Vec::new(), position: None}), attr_keys::IDENT, Some(AttrValue::String("id".to_string())))] +#[case::footnote(Node::Footnote(Footnote{ident: "id".to_string(), values: Vec::new(), position: None}), "unknown", None)] +#[case::html(Node::Html(Html{value: "<div>test</div>".to_string(), position: None}), attr_keys::VALUE, Some(AttrValue::String("<div>test</div>".to_string())))] +#[case::html(Node::Html(Html{value: "<div>test</div>".to_string(), position: None}), "unknown", None)] +#[case::text(Node::Text(Text{value: "text".to_string(), position: None}), attr_keys::VALUE, Some(AttrValue::String("text".to_string())))] +#[case::text(Node::Text(Text{value: "text".to_string(), position: None}), "unknown", None)] +#[case::code(Node::Code(Code{value: "code".to_string(), lang: Some("rust".to_string()), meta: Some("meta".to_string()), fence: true, position: None}), attr_keys::VALUE, Some(AttrValue::String("code".to_string())))] +#[case::code(Node::Code(Code{value: "code".to_string(), lang: Some("rust".to_string()), meta: Some("meta".to_string()), fence: true, position: None}), attr_keys::LANG, Some(AttrValue::String("rust".to_string())))] +#[case::code(Node::Code(Code{value: "code".to_string(), lang: Some("rust".to_string()), meta: Some("meta".to_string()), fence: true, position: None}), "meta", Some(AttrValue::String("meta".to_string())))] +#[case::code(Node::Code(Code{value: "code".to_string(), lang: Some("rust".to_string()), meta: Some("meta".to_string()), fence: true, position: None}), attr_keys::FENCE, Some(AttrValue::Boolean(true)))] +#[case::code(Node::Code(Code{value: "code".to_string(), lang: None, meta: None, fence: false, position: None}), attr_keys::FENCE, Some(AttrValue::Boolean(false)))] +#[case::code_inline(Node::CodeInline(CodeInline{value: "inline".into(), position: None}), attr_keys::VALUE, Some(AttrValue::String("inline".to_string())))] +#[case::math_inline(Node::MathInline(MathInline{value: "math".into(), position: None}), attr_keys::VALUE, Some(AttrValue::String("math".to_string())))] +#[case::math(Node::Math(Math{value: "math".to_string(), position: None}), attr_keys::VALUE, Some(AttrValue::String("math".to_string())))] +#[case::yaml(Node::Yaml(Yaml{value: "yaml".to_string(), position: None}), attr_keys::VALUE, Some(AttrValue::String("yaml".to_string())))] +#[case::toml(Node::Toml(Toml{value: "toml".to_string(), position: None}), attr_keys::VALUE, Some(AttrValue::String("toml".to_string())))] +#[case::image(Node::Image(Image{alt: attr_keys::ALT.to_string(), url: attr_keys::URL.to_string(), title: Some(attr_keys::TITLE.to_string()), position: None}), attr_keys::ALT, Some(AttrValue::String(attr_keys::ALT.to_string())))] +#[case::image(Node::Image(Image{alt: attr_keys::ALT.to_string(), url: attr_keys::URL.to_string(), title: Some(attr_keys::TITLE.to_string()), position: None}), attr_keys::URL, Some(AttrValue::String(attr_keys::URL.to_string())))] +#[case::image(Node::Image(Image{alt: attr_keys::ALT.to_string(), url: attr_keys::URL.to_string(), title: Some(attr_keys::TITLE.to_string()), position: None}), attr_keys::TITLE, Some(AttrValue::String(attr_keys::TITLE.to_string())))] +#[case::image_ref(Node::ImageRef(ImageRef{alt: attr_keys::ALT.to_string(), ident: "id".to_string(), label: Some(attr_keys::LABEL.to_string()), position: None}), attr_keys::ALT, Some(AttrValue::String(attr_keys::ALT.to_string())))] +#[case::image_ref(Node::ImageRef(ImageRef{alt: attr_keys::ALT.to_string(), ident: "id".to_string(), label: Some(attr_keys::LABEL.to_string()), position: None}), attr_keys::IDENT, Some(AttrValue::String("id".to_string())))] +#[case::image_ref(Node::ImageRef(ImageRef{alt: attr_keys::ALT.to_string(), ident: "id".to_string(), label: Some(attr_keys::LABEL.to_string()), position: None}), attr_keys::LABEL, Some(AttrValue::String(attr_keys::LABEL.to_string())))] +#[case::link(Node::Link(Link{url: Url::new(attr_keys::URL.to_string()), title: Some(Title::new(attr_keys::TITLE.to_string())), values: Vec::new(), position: None}), attr_keys::URL, Some(AttrValue::String(attr_keys::URL.to_string())))] +#[case::link(Node::Link(Link{url: Url::new(attr_keys::URL.to_string()), title: Some(Title::new(attr_keys::TITLE.to_string())), values: Vec::new(), position: None}), attr_keys::TITLE, Some(AttrValue::String(attr_keys::TITLE.to_string())))] +#[case::link_ref(Node::LinkRef(LinkRef{ident: "id".to_string(), values: Vec::new(), label: Some(attr_keys::LABEL.to_string()), position: None}), attr_keys::IDENT, Some(AttrValue::String("id".to_string())))] +#[case::link_ref(Node::LinkRef(LinkRef{ident: "id".to_string(), values: Vec::new(), label: Some(attr_keys::LABEL.to_string()), position: None}), attr_keys::LABEL, Some(AttrValue::String(attr_keys::LABEL.to_string())))] +#[case::footnote_ref(Node::FootnoteRef(FootnoteRef{ident: "id".to_string(), label: Some(attr_keys::LABEL.to_string()), position: None}), attr_keys::IDENT, Some(AttrValue::String("id".to_string())))] +#[case::footnote_ref(Node::FootnoteRef(FootnoteRef{ident: "id".to_string(), label: Some(attr_keys::LABEL.to_string()), position: None}), attr_keys::LABEL, Some(AttrValue::String(attr_keys::LABEL.to_string())))] +#[case::definition(Node::Definition(Definition{ident: "id".to_string(), url: Url::new(attr_keys::URL.to_string()), title: Some(Title::new(attr_keys::TITLE.to_string())), label: Some(attr_keys::LABEL.to_string()), position: None}), attr_keys::IDENT, Some(AttrValue::String("id".to_string())))] +#[case::definition(Node::Definition(Definition{ident: "id".to_string(), url: Url::new(attr_keys::URL.to_string()), title: Some(Title::new(attr_keys::TITLE.to_string())), label: Some(attr_keys::LABEL.to_string()), position: None}), attr_keys::URL, Some(AttrValue::String(attr_keys::URL.to_string())))] +#[case::definition(Node::Definition(Definition{ident: "id".to_string(), url: Url::new(attr_keys::URL.to_string()), title: Some(Title::new(attr_keys::TITLE.to_string())), label: Some(attr_keys::LABEL.to_string()), position: None}), attr_keys::TITLE, Some(AttrValue::String(attr_keys::TITLE.to_string())))] +#[case::definition(Node::Definition(Definition{ident: "id".to_string(), url: Url::new(attr_keys::URL.to_string()), title: Some(Title::new(attr_keys::TITLE.to_string())), label: Some(attr_keys::LABEL.to_string()), position: None}), attr_keys::LABEL, Some(AttrValue::String(attr_keys::LABEL.to_string())))] +#[case::heading(Node::Heading(Heading{depth: 3, values: Vec::new(), position: None}), "depth", Some(AttrValue::Integer(3)))] +#[case::list(Node::List(List{ marker: None,start: None, spread: false, index: 2, level: 1, checked: Some(true), ordered: true, values: Vec::new(), position: None}), "index", Some(AttrValue::Integer(2)))] +#[case::list(Node::List(List{ marker: None,start: None, spread: false, index: 2, level: 1, checked: Some(true), ordered: true, values: Vec::new(), position: None}), "level", Some(AttrValue::Integer(1)))] +#[case::list(Node::List(List{ marker: None,start: None, spread: false, index: 2, level: 1, checked: Some(true), ordered: true, values: Vec::new(), position: None}), "ordered", Some(AttrValue::Boolean(true)))] +#[case::list(Node::List(List{ marker: None,start: None, spread: false, index: 2, level: 1, checked: Some(true), ordered: true, values: Vec::new(), position: None}), attr_keys::CHECKED, Some(AttrValue::Boolean(true)))] +#[case::table_cell(Node::TableCell(TableCell{column: 1, row: 2, values: Vec::new(), position: None}), "column", Some(AttrValue::Integer(1)))] +#[case::table_cell(Node::TableCell(TableCell{column: 1, row: 2, values: Vec::new(), position: None}), "row", Some(AttrValue::Integer(2)))] +#[case::table_align(Node::TableAlign(TableAlign{align: vec![TableAlignKind::Left, TableAlignKind::Right], position: None}), "align", Some(AttrValue::String(":---,---:".to_string())))] +#[case::mdx_flow_expression(Node::MdxFlowExpression(MdxFlowExpression{value: "expr".into(), position: None}), attr_keys::VALUE, Some(AttrValue::String("expr".to_string())))] +#[case::mdx_flow_expression(Node::MdxTextExpression(MdxTextExpression{value: "expr".into(), position: None}), attr_keys::VALUE, Some(AttrValue::String("expr".to_string())))] +#[case::mdx_js_esm(Node::MdxJsEsm(MdxJsEsm{value: "esm".into(), position: None}), attr_keys::VALUE, Some(AttrValue::String("esm".to_string())))] +#[case::mdx_jsx_flow_element(Node::MdxJsxFlowElement(MdxJsxFlowElement{name: Some("div".to_string()), attributes: Vec::new(), children: Vec::new(), position: None}), attr_keys::NAME, Some(AttrValue::String("div".to_string())))] +#[case::mdx_jsx_flow_element(Node::MdxJsxTextElement(MdxJsxTextElement{name: Some("span".into()), attributes: Vec::new(), children: Vec::new(), position: None}), attr_keys::NAME, Some(AttrValue::String("span".to_string())))] +#[case::break_(Node::Break(Break{position: None}), attr_keys::VALUE, None)] +#[case::horizontal_rule(Node::HorizontalRule(HorizontalRule{ marker: None,position: None}), attr_keys::VALUE, None)] +#[case::fragment(Node::Fragment(Fragment{values: Vec::new()}), attr_keys::VALUE, Some(AttrValue::String("".to_string())))] +#[case::heading(Node::Heading(Heading{depth: 1, values: vec![Node::Text(Text{value: "heading text".to_string(), position: None})], position: None}), attr_keys::VALUE, Some(AttrValue::String("heading text".to_string())))] +#[case::heading(Node::Heading(Heading{depth: 2, values: vec![], position: None}), attr_keys::VALUE, Some(AttrValue::String("".to_string())))] +#[case::heading(Node::Heading(Heading{depth: 3, values: vec![ + Node::Text(Text{value: "first".to_string(), position: None}), + Node::Text(Text{value: "second".to_string(), position: None}), +], position: None}), attr_keys::VALUE, Some(AttrValue::String("firstsecond".to_string())))] +#[case( + Node::List(List { marker: None, + index: 0, + level: 1, + checked: None, + ordered: false, + start: None, spread: false, + values: vec![ + Node::Text(Text { value: "item1".to_string(), position: None }), + Node::Text(Text { value: "item2".to_string(), position: None }), + ], + position: None, + }), + attr_keys::VALUE, + Some(AttrValue::String("item1item2".to_string())) +)] +#[case( + Node::TableCell(TableCell { + column: 1, + row: 2, + values: vec![Node::Text(Text { + value: "cell_value".to_string(), + position: None, + })], + position: None, + }), + attr_keys::VALUE, + Some(AttrValue::String("cell_value".to_string())) +)] +#[case::footnote( + Node::Footnote(Footnote { + ident: "id".to_string(), + values: vec![Node::Text(Text { + value: "footnote value".to_string(), + position: None, + })], + position: None, + }), + attr_keys::VALUE, + Some(AttrValue::String("footnote value".to_string())) +)] +#[case::link( + Node::Link(Link { + url: Url::new("https://example.com".to_string()), + title: Some(Title::new("Example".to_string())), + values: vec![Node::Text(Text { + value: "link text".to_string(), + position: None, + })], + position: None, + }), + attr_keys::VALUE, + Some(AttrValue::String("link text".to_string())) +)] +#[case::empty(Node::Empty, attr_keys::VALUE, None)] +#[case::heading( + Node::Heading(Heading { + depth: 1, + values: vec![ + Node::Text(Text { + value: "child1".to_string(), + position: None, + }), + Node::Text(Text { + value: "child2".to_string(), + position: None, + }), + ], + position: None, + }), + attr_keys::CHILDREN, + Some(AttrValue::Array(vec![ + Node::Text(Text { + value: "child1".to_string(), + position: None, + }), + Node::Text(Text { + value: "child2".to_string(), + position: None, + }), + ])) + )] +#[case::list( + Node::List(List { marker: None, + index: 0, + level: 1, + checked: None, + ordered: false, + start: None, spread: false, + values: vec![ + Node::Text(Text { + value: "item1".to_string(), + position: None, + }), + ], + position: None, + }), + attr_keys::CHILDREN, + Some(AttrValue::Array(vec![ + Node::Text(Text { + value: "item1".to_string(), + position: None, + }), + ])) + )] +#[case::blockquote( + Node::Blockquote(Blockquote { + values: vec![ + Node::Text(Text { + value: "quote".to_string(), + position: None, + }), + ], + position: None, + }), + attr_keys::VALUES, + Some(AttrValue::Array(vec![ + Node::Text(Text { + value: "quote".to_string(), + position: None, + }), + ])) + )] +#[case::link( + Node::Link(Link { + url: Url::new(attr_keys::URL.to_string()), + title: None, + values: vec![ + Node::Text(Text { + value: "link".to_string(), + position: None, + }), + ], + position: None, + }), + attr_keys::VALUES, + Some(AttrValue::Array(vec![ + Node::Text(Text { + value: "link".to_string(), + position: None, + }), + ])) + )] +#[case::table_cell( + Node::TableCell(TableCell { + column: 0, + row: 0, + values: vec![ + Node::Text(Text { + value: "cell".to_string(), + position: None, + }), + ], + position: None, + }), + attr_keys::CHILDREN, + Some(AttrValue::Array(vec![ + Node::Text(Text { + value: "cell".to_string(), + position: None, + }), + ])) + )] +#[case::strong( + Node::Strong(Strong { + values: vec![ + Node::Text(Text { + value: "bold".to_string(), + position: None, + }), + ], + position: None, + }), + attr_keys::CHILDREN, + Some(AttrValue::Array(vec![ + Node::Text(Text { + value: "bold".to_string(), + position: None, + }), + ])) + )] +#[case::em( + Node::Emphasis(Emphasis { + values: vec![], + position: None, + }), + attr_keys::CHILDREN, + Some(AttrValue::Array(vec![])) + )] +fn test_attr(#[case] node: Node, #[case] attr: &str, #[case] expected: Option<AttrValue>) { + assert_eq!(node.attr(attr), expected); +} + +#[rstest] +#[case::heading_with_position( + Node::Heading(Heading { + depth: 1, + values: vec![], + position: Some(Position { + start: Point { line: 3, column: 1 }, + end: Point { line: 5, column: 4 }, + }), + }), + attr_keys::LINE, + Some(AttrValue::Integer(3)) +)] +#[case::code_end_line( + Node::Code(Code { + value: "x".to_string(), + lang: None, + meta: None, + fence: true, + position: Some(Position { + start: Point { line: 3, column: 1 }, + end: Point { line: 5, column: 4 }, + }), + }), + attr_keys::END_LINE, + Some(AttrValue::Integer(5)) +)] +#[case::synthetic_node_has_no_line( + Node::Heading(Heading { + depth: 1, + values: vec![], + position: None, + }), + attr_keys::LINE, + None +)] +#[case::synthetic_node_has_no_end_line( + Node::Heading(Heading { + depth: 1, + values: vec![], + position: None, + }), + attr_keys::END_LINE, + None +)] +fn test_line_attr(#[case] node: Node, #[case] attr: &str, #[case] expected: Option<AttrValue>) { + assert_eq!(node.attr(attr), expected); +} + +#[rstest] +#[case::heading( + Node::Heading(Heading{depth: 1, values: vec![], position: None}), + vec![Node::Text(Text{value: "child".to_string(), position: None})], + Node::Heading(Heading{depth: 1, values: vec![Node::Text(Text{value: "child".to_string(), position: None})], position: None}) +)] +#[case::list( + Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: vec![], position: None}), + vec![Node::Text(Text{value: "item".to_string(), position: None})], + Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: vec![Node::Text(Text{value: "item".to_string(), position: None})], position: None}) +)] +#[case::blockquote( + Node::Blockquote(Blockquote{values: vec![], position: None}), + vec![Node::Text(Text{value: "quote".to_string(), position: None})], + Node::Blockquote(Blockquote{values: vec![Node::Text(Text{value: "quote".to_string(), position: None})], position: None}) +)] +#[case::link( + Node::Link(Link{url: Url::new(attr_keys::URL.to_string()), title: None, values: vec![], position: None}), + vec![Node::Text(Text{value: "link".to_string(), position: None})], + Node::Link(Link{url: Url::new(attr_keys::URL.to_string()), title: None, values: vec![Node::Text(Text{value: "link".to_string(), position: None})], position: None}) +)] +#[case::footnote( + Node::Footnote(Footnote{ident: "1".to_string(), values: vec![], position: None}), + vec![Node::Text(Text{value: "note".to_string(), position: None})], + Node::Footnote(Footnote{ident: "1".to_string(), values: vec![Node::Text(Text{value: "note".to_string(), position: None})], position: None}) +)] +#[case::table_cell( + Node::TableCell(TableCell{column: 0, row: 0, values: vec![], position: None}), + vec![Node::Text(Text{value: "cell".to_string(), position: None})], + Node::TableCell(TableCell{column: 0, row: 0, values: vec![Node::Text(Text{value: "cell".to_string(), position: None})], position: None}) +)] +#[case::table_row( + Node::TableRow(TableRow{values: vec![], position: None}), + vec![Node::Text(Text{value: "row".to_string(), position: None})], + Node::TableRow(TableRow{values: vec![Node::Text(Text{value: "row".to_string(), position: None})], position: None}) +)] +#[case::strong( + Node::Strong(Strong{values: vec![], position: None}), + vec![Node::Text(Text{value: "bold".to_string(), position: None})], + Node::Strong(Strong{values: vec![Node::Text(Text{value: "bold".to_string(), position: None})], position: None}) +)] +#[case::delete( + Node::Delete(Delete{values: vec![], position: None}), + vec![Node::Text(Text{value: "del".to_string(), position: None})], + Node::Delete(Delete{values: vec![Node::Text(Text{value: "del".to_string(), position: None})], position: None}) +)] +#[case::emphasis( + Node::Emphasis(Emphasis{values: vec![], position: None}), + vec![Node::Text(Text{value: "em".to_string(), position: None})], + Node::Emphasis(Emphasis{values: vec![Node::Text(Text{value: "em".to_string(), position: None})], position: None}) +)] +#[case::fragment( + Node::Fragment(Fragment{values: vec![]}), + vec![Node::Text(Text{value: "frag".to_string(), position: None})], + Node::Fragment(Fragment{values: vec![Node::Text(Text{value: "frag".to_string(), position: None})]}) +)] +#[case::mdx_jsx_flow_element( + Node::MdxJsxFlowElement(MdxJsxFlowElement{name: Some("div".to_string()), attributes: Vec::new(), children: vec![], position: None}), + vec![Node::Text(Text{value: "mdx".to_string(), position: None})], + Node::MdxJsxFlowElement(MdxJsxFlowElement{name: Some("div".to_string()), attributes: Vec::new(), children: vec![Node::Text(Text{value: "mdx".to_string(), position: None})], position: None}) +)] +#[case::mdx_jsx_text_element( + Node::MdxJsxTextElement(MdxJsxTextElement{name: Some("span".into()), attributes: Vec::new(), children: vec![], position: None}), + vec![Node::Text(Text{value: "mdx".to_string(), position: None})], + Node::MdxJsxTextElement(MdxJsxTextElement{name: Some("span".into()), attributes: Vec::new(), children: vec![Node::Text(Text{value: "mdx".to_string(), position: None})], position: None}) +)] +#[case::leaf_node_is_noop( + Node::Text(Text{value: "leaf".to_string(), position: None}), + vec![Node::Text(Text{value: "ignored".to_string(), position: None})], + Node::Text(Text{value: "leaf".to_string(), position: None}) +)] +fn test_set_children(#[case] mut node: Node, #[case] children: Vec<Node>, #[case] expected: Node) { + node.set_children(children.clone()); + assert_eq!(node, expected); + assert_eq!( + node.children(), + if matches!(expected, Node::Text(_)) { + Vec::new() + } else { + children + } + ); +} + +#[rstest] +#[case( + Node::Text(Text{value: "old".to_string(), position: None}), + attr_keys::VALUE, + "new", + Node::Text(Text{value: "new".to_string(), position: None}) +)] +#[case( + Node::Code(Code{value: "old".to_string(), lang: Some("rust".to_string()), fence: true, meta: None, position: None}), + attr_keys::VALUE, + "new_code", + Node::Code(Code{value: "new_code".to_string(), lang: Some("rust".to_string()), fence: true, meta: None, position: None}) +)] +#[case( + Node::Code(Code{value: "code".to_string(), lang: Some("rust".to_string()), fence: true, meta: None, position: None}), + attr_keys::LANG, + "python", + Node::Code(Code{value: "code".to_string(), lang: Some("python".to_string()), fence: true, meta: None, position: None}) +)] +#[case( + Node::Code(Code{value: "code".to_string(), lang: None, fence: false, meta: None, position: None}), + attr_keys::FENCE, + "true", + Node::Code(Code{value: "code".to_string(), lang: None, fence: true, meta: None, position: None}) +)] +#[case( + Node::Image(Image{alt: attr_keys::ALT.to_string(), url: attr_keys::URL.to_string(), title: None, position: None}), + attr_keys::ALT, + "new_alt", + Node::Image(Image{alt: "new_alt".to_string(), url: attr_keys::URL.to_string(), title: None, position: None}) +)] +#[case( + Node::Image(Image{alt: attr_keys::ALT.to_string(), url: attr_keys::URL.to_string(), title: None, position: None}), + attr_keys::URL, + "new_url", + Node::Image(Image{alt: attr_keys::ALT.to_string(), url: "new_url".to_string(), title: None, position: None}) +)] +#[case( + Node::Image(Image{alt: attr_keys::ALT.to_string(), url: attr_keys::URL.to_string(), title: Some(attr_keys::TITLE.to_string()), position: None}), + attr_keys::TITLE, + "new_title", + Node::Image(Image{alt: attr_keys::ALT.to_string(), url: attr_keys::URL.to_string(), title: Some("new_title".to_string()), position: None}) +)] +#[case( + Node::Heading(Heading{depth: 2, values: vec![], position: None}), + "depth", + "3", + Node::Heading(Heading{depth: 3, values: vec![], position: None}) +)] +#[case( + Node::List(List{ marker: None,start: None, spread: false, index: 1, level: 2, checked: Some(true), ordered: false, values: vec![], position: None}), + attr_keys::CHECKED, + "false", + Node::List(List{ marker: None,start: None, spread: false, index: 1, level: 2, checked: Some(false), ordered: false, values: vec![], position: None}) +)] +#[case( + Node::List(List{ marker: None,start: None, spread: false, index: 1, level: 2, checked: Some(true), ordered: false, values: vec![], position: None}), + "ordered", + "true", + Node::List(List{ marker: None,start: None, spread: false, index: 1, level: 2, checked: Some(true), ordered: true, values: vec![], position: None}) +)] +#[case( + Node::TableCell(TableCell{column: 1, row: 2, values: vec![], position: None}), + "column", + "3", + Node::TableCell(TableCell{column: 3, row: 2, values: vec![], position: None}) +)] +#[case( + Node::TableCell(TableCell{column: 1, row: 2, values: vec![], position: None}), + "row", + "5", + Node::TableCell(TableCell{column: 1, row: 5, values: vec![], position: None}) +)] +#[case( + Node::Definition(Definition{ident: "id".to_string(), url: Url::new(attr_keys::URL.to_string()), title: None, label: None, position: None}), + attr_keys::IDENT, + "new_id", + Node::Definition(Definition{ident: "new_id".to_string(), url: Url::new(attr_keys::URL.to_string()), title: None, label: None, position: None}) +)] +#[case( + Node::Definition(Definition{ident: "id".to_string(), url: Url::new(attr_keys::URL.to_string()), title: None, label: None, position: None}), + attr_keys::URL, + "new_url", + Node::Definition(Definition{ident: "id".to_string(), url: Url::new("new_url".to_string()), title: None, label: None, position: None}) +)] +#[case( + Node::Definition(Definition{ident: "id".to_string(), url: Url::new(attr_keys::URL.to_string()), title: None, label: None, position: None}), + attr_keys::LABEL, + "new_label", + Node::Definition(Definition{ident: "id".to_string(), url: Url::new(attr_keys::URL.to_string()), title: None, label: Some("new_label".to_string()), position: None}) +)] +#[case( + Node::Definition(Definition{ident: "id".to_string(), url: Url::new(attr_keys::URL.to_string()), title: None, label: None, position: None}), + attr_keys::TITLE, + "new_title", + Node::Definition(Definition{ident: "id".to_string(), url: Url::new(attr_keys::URL.to_string()), title: Some(Title::new("new_title".to_string())), label: None, position: None}) +)] +#[case( + Node::ImageRef(ImageRef{alt: attr_keys::ALT.to_string(), ident: "id".to_string(), label: Some(attr_keys::LABEL.to_string()), position: None}), + attr_keys::ALT, + "new_alt", + Node::ImageRef(ImageRef{alt: "new_alt".to_string(), ident: "id".to_string(), label: Some(attr_keys::LABEL.to_string()), position: None}) +)] +#[case( + Node::ImageRef(ImageRef{alt: attr_keys::ALT.to_string(), ident: "id".to_string(), label: Some(attr_keys::LABEL.to_string()), position: None}), + attr_keys::IDENT, + "new_id", + Node::ImageRef(ImageRef{alt: attr_keys::ALT.to_string(), ident: "new_id".to_string(), label: Some(attr_keys::LABEL.to_string()), position: None}) +)] +#[case( + Node::ImageRef(ImageRef{alt: attr_keys::ALT.to_string(), ident: "id".to_string(), label: Some(attr_keys::LABEL.to_string()), position: None}), + attr_keys::LABEL, + "new_label", + Node::ImageRef(ImageRef{alt: attr_keys::ALT.to_string(), ident: "id".to_string(), label: Some("new_label".to_string()), position: None}) +)] +#[case( + Node::ImageRef(ImageRef{alt: attr_keys::ALT.to_string(), ident: "id".to_string(), label: None, position: None}), + attr_keys::LABEL, + "new_label", + Node::ImageRef(ImageRef{alt: attr_keys::ALT.to_string(), ident: "id".to_string(), label: Some("new_label".to_string()), position: None}) +)] +#[case( + Node::LinkRef(LinkRef{ident: "id".to_string(), values: vec![], label: Some(attr_keys::LABEL.to_string()), position: None}), + attr_keys::IDENT, + "new_id", + Node::LinkRef(LinkRef{ident: "new_id".to_string(), values: vec![], label: Some(attr_keys::LABEL.to_string()), position: None}) +)] +#[case( + Node::LinkRef(LinkRef{ident: "id".to_string(), values: vec![], label: Some(attr_keys::LABEL.to_string()), position: None}), + attr_keys::LABEL, + "new_label", + Node::LinkRef(LinkRef{ident: "id".to_string(), values: vec![], label: Some("new_label".to_string()), position: None}) +)] +#[case( + Node::LinkRef(LinkRef{ident: "id".to_string(), values: vec![], label: None, position: None}), + attr_keys::LABEL, + "new_label", + Node::LinkRef(LinkRef{ident: "id".to_string(), values: vec![], label: Some("new_label".to_string()), position: None}) +)] +#[case( + Node::LinkRef(LinkRef{ident: "id".to_string(), values: vec![], label: Some(attr_keys::LABEL.to_string()), position: None}), + "unknown", + "ignored", + Node::LinkRef(LinkRef{ident: "id".to_string(), values: vec![], label: Some(attr_keys::LABEL.to_string()), position: None}) +)] +#[case( + Node::FootnoteRef(FootnoteRef{ident: "id".to_string(), label: Some(attr_keys::LABEL.to_string()), position: None}), + attr_keys::IDENT, + "new_id", + Node::FootnoteRef(FootnoteRef{ident: "new_id".to_string(), label: Some(attr_keys::LABEL.to_string()), position: None}) +)] +#[case( + Node::FootnoteRef(FootnoteRef{ident: "id".to_string(), label: Some(attr_keys::LABEL.to_string()), position: None}), + attr_keys::LABEL, + "new_label", + Node::FootnoteRef(FootnoteRef{ident: "id".to_string(), label: Some("new_label".to_string()), position: None}) +)] +#[case( + Node::FootnoteRef(FootnoteRef{ident: "id".to_string(), label: None, position: None}), + attr_keys::LABEL, + "new_label", + Node::FootnoteRef(FootnoteRef{ident: "id".to_string(), label: Some("new_label".to_string()), position: None}) +)] +#[case( + Node::FootnoteRef(FootnoteRef{ident: "id".to_string(), label: Some(attr_keys::LABEL.to_string()), position: None}), + "unknown", + "ignored", + Node::FootnoteRef(FootnoteRef{ident: "id".to_string(), label: Some(attr_keys::LABEL.to_string()), position: None}) +)] +#[case(Node::Empty, attr_keys::VALUE, "ignored", Node::Empty)] +#[case( + Node::TableAlign(TableAlign{align: vec![TableAlignKind::Left, TableAlignKind::Right], position: None}), + "align", + "---,:---:", + Node::TableAlign(TableAlign{align: vec![TableAlignKind::None, TableAlignKind::Center], position: None}) +)] +#[case( + Node::TableAlign(TableAlign{align: vec![], position: None}), + "align", + ":---,---:", + Node::TableAlign(TableAlign{align: vec![TableAlignKind::Left, TableAlignKind::Right], position: None}) +)] +#[case( + Node::TableAlign(TableAlign{align: vec![TableAlignKind::Left], position: None}), + "unknown", + "ignored", + Node::TableAlign(TableAlign{align: vec![TableAlignKind::Left], position: None}) +)] +#[case( + Node::MdxFlowExpression(MdxFlowExpression{value: "old".into(), position: None}), + attr_keys::VALUE, + "new_expr", + Node::MdxFlowExpression(MdxFlowExpression{value: "new_expr".into(), position: None}) +)] +#[case( + Node::MdxFlowExpression(MdxFlowExpression{value: "expr".into(), position: None}), + "unknown", + "ignored", + Node::MdxFlowExpression(MdxFlowExpression{value: "expr".into(), position: None}) +)] +#[case( + Node::MdxTextExpression(MdxTextExpression{value: "old".into(), position: None}), + attr_keys::VALUE, + "new_expr", + Node::MdxTextExpression(MdxTextExpression{value: "new_expr".into(), position: None}) +)] +#[case( + Node::MdxTextExpression(MdxTextExpression{value: "expr".into(), position: None}), + "unknown", + "ignored", + Node::MdxTextExpression(MdxTextExpression{value: "expr".into(), position: None}) +)] +#[case( + Node::MdxJsEsm(MdxJsEsm{value: "import x".into(), position: None}), + attr_keys::VALUE, + "import y", + Node::MdxJsEsm(MdxJsEsm{value: "import y".into(), position: None}) +)] +#[case( + Node::MdxJsEsm(MdxJsEsm{value: "import x".into(), position: None}), + "unknown", + "ignored", + Node::MdxJsEsm(MdxJsEsm{value: "import x".into(), position: None}) +)] +#[case( + Node::MdxJsxFlowElement(MdxJsxFlowElement{name: Some("div".to_string()), attributes: Vec::new(), children: Vec::new(), position: None}), + attr_keys::NAME, + "section", + Node::MdxJsxFlowElement(MdxJsxFlowElement{name: Some("section".to_string()), attributes: Vec::new(), children: Vec::new(), position: None}) +)] +#[case( + Node::MdxJsxFlowElement(MdxJsxFlowElement{name: None, attributes: Vec::new(), children: Vec::new(), position: None}), + attr_keys::NAME, + "main", + Node::MdxJsxFlowElement(MdxJsxFlowElement{name: Some("main".to_string()), attributes: Vec::new(), children: Vec::new(), position: None}) +)] +#[case( + Node::MdxJsxFlowElement(MdxJsxFlowElement{name: Some("div".to_string()), attributes: Vec::new(), children: Vec::new(), position: None}), + "unknown", + "ignored", + Node::MdxJsxFlowElement(MdxJsxFlowElement{name: Some("div".to_string()), attributes: Vec::new(), children: Vec::new(), position: None}) +)] +#[case( + Node::MdxJsxTextElement(MdxJsxTextElement{name: Some("span".into()), attributes: Vec::new(), children: Vec::new(), position: None}), + attr_keys::NAME, + "b", + Node::MdxJsxTextElement(MdxJsxTextElement{name: Some("b".into()), attributes: Vec::new(), children: Vec::new(), position: None}) +)] +#[case( + Node::MdxJsxTextElement(MdxJsxTextElement{name: None, attributes: Vec::new(), children: Vec::new(), position: None}), + attr_keys::NAME, + "i", + Node::MdxJsxTextElement(MdxJsxTextElement{name: Some("i".into()), attributes: Vec::new(), children: Vec::new(), position: None}) +)] +#[case( + Node::MdxJsxTextElement(MdxJsxTextElement{name: Some("span".into()), attributes: Vec::new(), children: Vec::new(), position: None}), + "unknown", + "ignored", + Node::MdxJsxTextElement(MdxJsxTextElement{name: Some("span".into()), attributes: Vec::new(), children: Vec::new(), position: None}) +)] +fn test_set_attr(#[case] mut node: Node, #[case] attr: &str, #[case] value: &str, #[case] expected: Node) { + node.set_attr(attr, value); + assert_eq!(node, expected); +} + +#[rstest] +#[case(AttrValue::String("test".to_string()), AttrValue::String("test".to_string()), true)] +#[case(AttrValue::String("test".to_string()), AttrValue::String("other".to_string()), false)] +#[case(AttrValue::Integer(42), AttrValue::Integer(42), true)] +#[case(AttrValue::Integer(42), AttrValue::Integer(0), false)] +#[case(AttrValue::Boolean(true), AttrValue::Boolean(true), true)] +#[case(AttrValue::Boolean(true), AttrValue::Boolean(false), false)] +#[case(AttrValue::String("42".to_string()), AttrValue::Integer(42), false)] +#[case(AttrValue::Boolean(false), AttrValue::Integer(0), false)] +fn test_attr_value_eq(#[case] a: AttrValue, #[case] b: AttrValue, #[case] expected: bool) { + assert_eq!(a == b, expected); +} + +#[rstest] +#[case(AttrValue::String("test".to_string()), "test")] +#[case(AttrValue::Integer(42), "42")] +#[case(AttrValue::Boolean(true), "true")] +fn test_attr_value_as_str(#[case] value: AttrValue, #[case] expected: &str) { + assert_eq!(&value.as_string(), expected); +} + +#[rstest] +#[case(AttrValue::Integer(42), Some(42))] +#[case(AttrValue::String("42".to_string()), Some(42))] +#[case(AttrValue::Boolean(false), Some(0))] +fn test_attr_value_as_i64(#[case] value: AttrValue, #[case] expected: Option<i64>) { + assert_eq!(value.as_i64(), expected); +} + +// --- Callout tests --- + +#[cfg(feature = "callout")] +#[rstest] +// basic render +#[case( + Node::Callout(Callout { fold: None, kind: "NOTE".to_string(), title: None, + values: vec![Node::Text(Text { value: "content".to_string(), position: None })], + position: None }), + "> [!NOTE]\n> content" +)] +// with title +#[case( + Node::Callout(Callout { fold: None, kind: "WARNING".to_string(), title: Some("Heads up".to_string()), + values: vec![Node::Text(Text { value: "watch out".to_string(), position: None })], + position: None }), + "> [!WARNING] Heads up\n> watch out" +)] +// empty body +#[case( + Node::Callout(Callout { fold: None, kind: "TIP".to_string(), title: None, values: vec![], position: None }), + "> [!TIP]" +)] +// multiline body: single Text with embedded '\n' — each line prefixed with "> " +#[case( + Node::Callout(Callout { fold: None, kind: "INFO".to_string(), title: None, + values: vec![Node::Text(Text { value: "line one\nline two".to_string(), position: None })], + position: None }), + "> [!INFO]\n> line one\n> line two" +)] +// two separate position-less Text values are concatenated inline (no implicit newline) +#[case( + Node::Callout(Callout { fold: None, kind: "INFO".to_string(), title: None, + values: vec![ + Node::Text(Text { value: "part a".to_string(), position: None }), + Node::Text(Text { value: "part b".to_string(), position: None }), + ], + position: None }), + "> [!INFO]\n> part apart b" +)] +fn test_callout_render(#[case] node: Node, #[case] expected: &str) { + assert_eq!(node.to_string_with(&RenderOptions::default()), expected); +} + +#[cfg(feature = "callout")] +#[rstest] +#[case(Node::Callout(Callout { fold: None, kind: "NOTE".to_string(), title: None, values: vec![], position: None }), "kind", Some(AttrValue::String("NOTE".to_string())))] +#[case(Node::Callout(Callout { fold: None, kind: "WARNING".to_string(), title: Some("Title".to_string()), values: vec![], position: None }), "title", Some(AttrValue::String("Title".to_string())))] +#[case(Node::Callout(Callout { fold: None, kind: "TIP".to_string(), title: None, values: vec![], position: None }), "title", None)] +#[case(Node::Callout(Callout { fold: None, kind: "NOTE".to_string(), title: None, values: vec![Node::Text(Text { value: "body".to_string(), position: None })], position: None }), "value", Some(AttrValue::String("body".to_string())))] +#[case(Node::Callout(Callout { fold: None, kind: "NOTE".to_string(), title: None, values: vec![Node::Text(Text { value: "body".to_string(), position: None })], position: None }), "children", Some(AttrValue::Array(vec![Node::Text(Text { value: "body".to_string(), position: None })])))] +fn test_callout_attr(#[case] node: Node, #[case] attr: &str, #[case] expected: Option<AttrValue>) { + assert_eq!(node.attr(attr), expected); +} + +#[cfg(feature = "callout")] +#[rstest] +#[case( + Node::Callout(Callout { fold: None, kind: "NOTE".to_string(), title: None, values: vec![], position: None }), + "kind", "WARNING", + Node::Callout(Callout { fold: None, kind: "WARNING".to_string(), title: None, values: vec![], position: None }) +)] +// set_attr stores kind as-is without case conversion +#[case( + Node::Callout(Callout { fold: None, kind: "NOTE".to_string(), title: None, values: vec![], position: None }), + "kind", "tip", + Node::Callout(Callout { fold: None, kind: "tip".to_string(), title: None, values: vec![], position: None }) +)] +#[case( + Node::Callout(Callout { fold: None, kind: "NOTE".to_string(), title: None, values: vec![], position: None }), + "title", "My title", + Node::Callout(Callout { fold: None, kind: "NOTE".to_string(), title: Some("My title".to_string()), values: vec![], position: None }) +)] +// empty string clears title +#[case( + Node::Callout(Callout { fold: None, kind: "NOTE".to_string(), title: Some("old".to_string()), values: vec![], position: None }), + "title", "", + Node::Callout(Callout { fold: None, kind: "NOTE".to_string(), title: None, values: vec![], position: None }) +)] +fn test_callout_set_attr(#[case] mut node: Node, #[case] attr: &str, #[case] value: &str, #[case] expected: Node) { + node.set_attr(attr, value); + assert_eq!(node, expected); +} + +#[cfg(feature = "callout")] +#[rstest] +// with_value changes first body node +#[case( + Node::Callout(Callout { fold: None, kind: "NOTE".to_string(), title: None, + values: vec![Node::Text(Text { value: "old".to_string(), position: None })], + position: None }), + "new", + Node::Callout(Callout { fold: None, kind: "NOTE".to_string(), title: None, + values: vec![Node::Text(Text { value: "new".to_string(), position: None })], + position: None }) +)] +fn test_callout_with_value(#[case] node: Node, #[case] value: &str, #[case] expected: Node) { + assert_eq!(node.clone().with_value(value), expected); + assert_eq!(node.into_with_value(value), expected); +} + +#[cfg(feature = "callout")] +#[rstest] +#[case(Node::Callout(Callout { fold: None, kind: "NOTE".to_string(), title: None, values: vec![], position: None }), "callout")] +fn test_callout_name(#[case] node: Node, #[case] expected: &str) { + assert_eq!(node.name(), expected); +} + +#[cfg(feature = "callout")] +#[rstest] +#[case( + Node::Callout(Callout { fold: None, kind: "NOTE".to_string(), title: None, + values: vec![Node::Text(Text { value: "body".to_string(), position: None })], + position: None }), + "body" +)] +fn test_callout_value(#[case] node: Node, #[case] expected: &str) { + assert_eq!(node.value(), expected); +} + +#[cfg(feature = "callout")] +#[rstest] +#[case(Node::Callout(Callout { fold: None, kind: "NOTE".to_string(), title: None, values: vec![], position: None }), true)] +#[case(Node::Blockquote(Blockquote { values: vec![], position: None }), false)] +#[case(Node::Text(Text { value: "test".to_string(), position: None }), false)] +fn test_is_callout(#[case] node: Node, #[case] expected: bool) { + assert_eq!(node.is_callout(), expected); +} + +#[cfg(feature = "callout")] +#[test] +fn test_callout_end_to_end() { + use crate::Markdown; + // plain blockquote stays as Blockquote + let plain = Markdown::from_markdown_str("> just a quote").unwrap(); + assert!(plain.nodes[0].is_blockquote()); + + // [!NOTE] is parsed as Callout + let note = Markdown::from_markdown_str("> [!NOTE]\n> body").unwrap(); + assert!(note.nodes[0].is_callout()); + assert_eq!(note.nodes[0].attr("kind"), Some(AttrValue::String("NOTE".to_string()))); + + // [!WARNING] with title + let warn = Markdown::from_markdown_str("> [!WARNING] Watch out\n> content").unwrap(); + assert!(warn.nodes[0].is_callout()); + assert_eq!( + warn.nodes[0].attr("title"), + Some(AttrValue::String("Watch out".to_string())) + ); +} + +// --- Embed tests --- + +#[cfg(feature = "embed")] +#[rstest] +// plain embed +#[case("![[target]]", vec![Node::Embed(Embed { target: "target".to_string(), display: None, position: None })])] +// embed with display hint (image size) +#[case("![[image.png|400]]", vec![Node::Embed(Embed { target: "image.png".to_string(), display: Some("400".to_string()), position: None })])] +// embed with section reference in target +#[case("![[note#Heading]]", vec![Node::Embed(Embed { target: "note#Heading".to_string(), display: None, position: None })])] +// embed at start of text +#[case("![[note]] rest", vec![ + Node::Embed(Embed { target: "note".to_string(), display: None, position: None }), + Node::Text(Text { value: " rest".to_string(), position: None }), +])] +// embed in middle +#[case("before ![[note]] after", vec![ + Node::Text(Text { value: "before ".to_string(), position: None }), + Node::Embed(Embed { target: "note".to_string(), display: None, position: None }), + Node::Text(Text { value: " after".to_string(), position: None }), +])] +// multiple embeds +#[case("![[a]] and ![[b]]", vec![ + Node::Embed(Embed { target: "a".to_string(), display: None, position: None }), + Node::Text(Text { value: " and ".to_string(), position: None }), + Node::Embed(Embed { target: "b".to_string(), display: None, position: None }), +])] +// multibyte target +#[case("![[ノート]]", vec![Node::Embed(Embed { target: "ノート".to_string(), display: None, position: None })])] +// no embed — plain text unchanged +#[case("plain text", vec![Node::Text(Text { value: "plain text".to_string(), position: None })])] +// unclosed embed — treated as plain text +#[case("![[unclosed", vec![Node::Text(Text { value: "![[unclosed".to_string(), position: None })])] +// lone ! before non-embed — treated as plain text +#[case("! not an embed", vec![Node::Text(Text { value: "! not an embed".to_string(), position: None })])] +fn test_parse_embeds_in_text(#[case] input: &str, #[case] expected: Vec<Node>) { + let mut result = Vec::new(); + Node::parse_embeds_into(input, None, &mut result); + assert_eq!(result, expected); +} + +#[cfg(feature = "embed")] +#[rstest] +// plain embed renders without display +#[case( + Node::Embed(Embed { target: "note.md".to_string(), display: None, position: None }), + "![[note.md]]" +)] +// embed with display hint +#[case( + Node::Embed(Embed { target: "image.png".to_string(), display: Some("400".to_string()), position: None }), + "![[image.png|400]]" +)] +// embed with section reference +#[case( + Node::Embed(Embed { target: "note#Intro".to_string(), display: None, position: None }), + "![[note#Intro]]" +)] +fn test_embed_render(#[case] node: Node, #[case] expected: &str) { + assert_eq!(node.to_string_with(&RenderOptions::default()), expected); +} + +#[cfg(feature = "embed")] +#[rstest] +#[case(Node::Embed(Embed { target: "note".to_string(), display: None, position: None }), "url", Some(AttrValue::String("note".to_string())))] +// value returns target when no display +#[case(Node::Embed(Embed { target: "note".to_string(), display: None, position: None }), "value", Some(AttrValue::String("note".to_string())))] +// url always returns target regardless of display +#[case(Node::Embed(Embed { target: "note".to_string(), display: Some("400".to_string()), position: None }), "url", Some(AttrValue::String("note".to_string())))] +// value returns display when present +#[case(Node::Embed(Embed { target: "note".to_string(), display: Some("400".to_string()), position: None }), "value", Some(AttrValue::String("400".to_string())))] +// unknown attr returns None +#[case(Node::Embed(Embed { target: "note".to_string(), display: None, position: None }), "unknown", None)] +fn test_embed_attr(#[case] node: Node, #[case] attr: &str, #[case] expected: Option<AttrValue>) { + assert_eq!(node.attr(attr), expected); +} + +#[cfg(feature = "embed")] +#[rstest] +// set url changes target +#[case( + Node::Embed(Embed { target: "old".to_string(), display: None, position: None }), + "url", "new", + Node::Embed(Embed { target: "new".to_string(), display: None, position: None }) +)] +// set value changes display +#[case( + Node::Embed(Embed { target: "note".to_string(), display: None, position: None }), + "value", "800", + Node::Embed(Embed { target: "note".to_string(), display: Some("800".to_string()), position: None }) +)] +// empty value clears display +#[case( + Node::Embed(Embed { target: "note".to_string(), display: Some("400".to_string()), position: None }), + "value", "", + Node::Embed(Embed { target: "note".to_string(), display: None, position: None }) +)] +fn test_embed_set_attr(#[case] mut node: Node, #[case] attr: &str, #[case] value: &str, #[case] expected: Node) { + node.set_attr(attr, value); + assert_eq!(node, expected); +} + +#[cfg(feature = "embed")] +#[rstest] +// no display: with_value sets target +#[case( + Node::Embed(Embed { target: "old".to_string(), display: None, position: None }), + "new", + Node::Embed(Embed { target: "new".to_string(), display: None, position: None }) +)] +// with display: with_value sets display +#[case( + Node::Embed(Embed { target: "note".to_string(), display: Some("400".to_string()), position: None }), + "800", + Node::Embed(Embed { target: "note".to_string(), display: Some("800".to_string()), position: None }) +)] +fn test_embed_with_value(#[case] node: Node, #[case] value: &str, #[case] expected: Node) { + assert_eq!(node.clone().with_value(value), expected); + assert_eq!(node.into_with_value(value), expected); +} + +#[cfg(feature = "embed")] +#[rstest] +#[case(Node::Embed(Embed { target: "note".to_string(), display: None, position: None }), "embed")] +fn test_embed_name(#[case] node: Node, #[case] expected: &str) { + assert_eq!(node.name(), expected); +} + +#[cfg(feature = "embed")] +#[rstest] +// value returns display when present +#[case(Node::Embed(Embed { target: "note".to_string(), display: Some("400".to_string()), position: None }), "400")] +// value returns target when no display +#[case(Node::Embed(Embed { target: "note".to_string(), display: None, position: None }), "note")] +fn test_embed_value(#[case] node: Node, #[case] expected: &str) { + assert_eq!(node.value(), expected); +} + +#[cfg(feature = "embed")] +#[rstest] +#[case(Node::Embed(Embed { target: "note".to_string(), display: None, position: None }), true)] +#[case(Node::Text(Text { value: "test".to_string(), position: None }), false)] +#[case(Node::Image(Image { alt: "".to_string(), url: "img.png".to_string(), title: None, position: None }), false)] +fn test_is_embed(#[case] node: Node, #[case] expected: bool) { + assert_eq!(node.is_embed(), expected); +} + +#[cfg(feature = "embed")] +#[test] +fn test_embed_end_to_end() { + use crate::Markdown; + let md = Markdown::from_markdown_str("See ![[note.md]] for details.").unwrap(); + let embed = md.nodes.iter().find(|n| n.is_embed()); + assert!(embed.is_some()); + assert_eq!( + embed.unwrap().attr("url"), + Some(AttrValue::String("note.md".to_string())) + ); +} + +#[cfg(all(feature = "callout", feature = "wikilink"))] +#[rstest] +// callout body containing a wikilink +#[case("> [!NOTE]\n> See [[note]]", 1, "> [!NOTE]\n> See [[note]]\n")] +fn test_callout_with_wikilink(#[case] input: &str, #[case] expected_nodes: usize, #[case] expected_output: &str) { + use crate::Markdown; + let md = Markdown::from_markdown_str(input).unwrap(); + assert_eq!(md.nodes.len(), expected_nodes); + assert_eq!(md.to_string(), expected_output); +} + +#[cfg(feature = "callout")] +#[rstest] +// callout body with bold +#[case("> [!NOTE]\n> **important**", 1, "> [!NOTE]\n> **important**\n")] +// callout body with inline code +#[case("> [!NOTE]\n> Use `code`", 1, "> [!NOTE]\n> Use `code`\n")] +// multiple callouts in one document +#[case( + "> [!NOTE]\n> first\n\n> [!WARNING]\n> second", + 2, + "> [!NOTE]\n> first\n\n> [!WARNING]\n> second\n" +)] +// callout after heading +#[case("# Title\n\n> [!NOTE]\n> body", 2, "# Title\n\n> [!NOTE]\n> body\n")] +// callout body with a multi-item list — second item must not be over-indented +#[case("> [!NOTE]\n> - item 1\n> - item 2", 1, "> [!NOTE]\n> - item 1\n> - item 2\n")] +fn test_callout_combined_with_other_nodes( + #[case] input: &str, + #[case] expected_nodes: usize, + #[case] expected_output: &str, +) { + use crate::Markdown; + let md = Markdown::from_markdown_str(input).unwrap(); + assert_eq!(md.nodes.len(), expected_nodes); + assert_eq!(md.to_string(), expected_output); +} + +#[cfg(feature = "embed")] +#[rstest] +// embed inside list item +#[case("- ![[note.md]]", 1, "- ![[note.md]]\n")] +// embed inside heading +#[case("# See ![[note]]", 1, "# See ![[note]]\n")] +// embed inside bold +#[case("**![[note]]**", 1, "**![[note]]**\n")] +fn test_embed_combined_with_other_nodes( + #[case] input: &str, + #[case] expected_nodes: usize, + #[case] expected_output: &str, +) { + use crate::Markdown; + let md = Markdown::from_markdown_str(input).unwrap(); + assert_eq!(md.nodes.len(), expected_nodes); + assert_eq!(md.to_string(), expected_output); +} + +#[cfg(all(feature = "callout", feature = "embed"))] +#[rstest] +// wikilink inside callout body + embed as separate node +#[case("> [!NOTE]\n> [[link]]\n\n![[embed]]", 2, "> [!NOTE]\n> [[link]]\n\n![[embed]]\n")] +fn test_callout_and_embed_together(#[case] input: &str, #[case] expected_nodes: usize, #[case] expected_output: &str) { + use crate::Markdown; + let md = Markdown::from_markdown_str(input).unwrap(); + assert_eq!(md.nodes.len(), expected_nodes); + assert_eq!(md.to_string(), expected_output); +} + +#[cfg(all(feature = "embed", feature = "wikilink"))] +#[test] +fn test_embed_does_not_conflict_with_wikilink() { + use crate::Markdown; + // ![[embed]] must become Embed, not WikiLink with preceding "!" + let md = Markdown::from_markdown_str("![[embed]] and [[link]]").unwrap(); + let embed = md.nodes.iter().find(|n| n.is_embed()); + let wikilink = md.nodes.iter().find(|n| n.is_wikilink()); + assert!(embed.is_some(), "expected Embed node"); + assert!(wikilink.is_some(), "expected WikiLink node"); +} + +#[rstest] +#[case::escaped_when_parsed_from_source( + Node::Text(Text { + value: "*a*".to_string(), + position: Some(Position { + start: Point { line: 1, column: 1 }, + end: Point { line: 1, column: 4 }, + }), + }), + "\\*a\\*" +)] +#[case::untouched_when_built_programmatically( + Node::Text(Text { value: "*a*".to_string(), position: None }), + "*a*" +)] +fn test_text_escaping_gated_by_position(#[case] node: Node, #[case] expected: &str) { + assert_eq!(node.to_string_with(&RenderOptions::default()), expected); +} + +#[rstest] +#[case::bare_safe("https://example.com/x", UrlSurroundStyle::None, "https://example.com/x")] +#[case::space_forces_angle("my uri", UrlSurroundStyle::None, "<my uri>")] +#[case::empty_forces_angle("", UrlSurroundStyle::None, "<>")] +#[case::unbalanced_paren_escaped("b)c", UrlSurroundStyle::None, "b\\)c")] +#[case::both_parens_escaped("foo(and(bar)", UrlSurroundStyle::None, "foo\\(and\\(bar\\)")] +#[case::forced_angle_style("plain", UrlSurroundStyle::Angle, "<plain>")] +#[case::angle_escapes_inner_angle_and_backslash("foo\\>", UrlSurroundStyle::None, "<foo\\\\\\>>")] +fn test_render_link_destination(#[case] url: &str, #[case] style: UrlSurroundStyle, #[case] expected: &str) { + assert_eq!(render_link_destination(url, &style), expected); +} + +#[rstest] +#[case::double_quote_escaped(r#"a "b" c"#, TitleSurroundStyle::Double, r#""a \"b\" c""#)] +#[case::single_quote_escaped("a 'b' c", TitleSurroundStyle::Single, "'a \\'b\\' c'")] +#[case::paren_escapes_both("a (b) c", TitleSurroundStyle::Paren, "(a \\(b\\) c)")] +#[case::backslash_escaped("a\\b", TitleSurroundStyle::Double, "\"a\\\\b\"")] +fn test_render_link_title(#[case] title: &str, #[case] style: TitleSurroundStyle, #[case] expected: &str) { + assert_eq!(render_link_title(title, &style), expected); +} + +#[rstest] +#[case::shortcut_kept_when_ident_matches_label_present("foo", "foo", Some("foo".to_string()), "[foo]")] +#[case::shortcut_kept_when_ident_matches_label_absent("foo", "foo", None, "[foo]")] +#[case::full_form_when_ident_differs("foo", "bar", Some("bar".to_string()), "[foo][bar]")] +#[case::full_form_when_plain_needs_broad_escaping("foo*bar", "foo*bar", Some("foo*bar".to_string()), "[foo*bar][foo*bar]")] +fn test_link_ref_shortcut_vs_full( + #[case] text: &str, + #[case] ident: &str, + #[case] label: Option<String>, + #[case] expected: &str, +) { + let node = Node::LinkRef(LinkRef { + ident: ident.to_string(), + values: vec![text.to_string().into()], + label, + position: None, + }); + assert_eq!(node.to_string_with(&RenderOptions::default()), expected); +} + +#[test] +fn test_link_ref_display_escaping_does_not_leak_into_matching_label() { + // The display text (first bracket) is escaped for safety; the matching key + // (second bracket) must stay the definition-compatible, narrowly-escaped label. + let node = Node::LinkRef(LinkRef { + ident: "foo*bar".to_string(), + values: vec![Node::Text(Text { + value: "foo*bar".to_string(), + position: Some(Position { + start: Point { line: 1, column: 1 }, + end: Point { line: 1, column: 8 }, + }), + })], + label: Some("foo*bar".to_string()), + position: None, + }); + assert_eq!(node.to_string_with(&RenderOptions::default()), "[foo\\*bar][foo*bar]"); +} + +#[rstest] +#[case::shortcut_kept_when_ident_matches("foo bar", "foo bar", "![foo bar]")] +#[case::full_form_when_ident_differs("foo bar", "foo*bar*", "![foo bar][foo*bar*]")] +fn test_image_ref_shortcut_vs_full(#[case] alt: &str, #[case] ident: &str, #[case] expected: &str) { + let node = Node::ImageRef(ImageRef { + alt: alt.to_string(), + ident: ident.to_string(), + label: None, + position: None, + }); + assert_eq!(node.to_string_with(&RenderOptions::default()), expected); +} + +#[rstest] +#[case::no_newline_still_shifted("abc", 3, " abc")] +#[case::zero_delta_unchanged("a\n b", 0, "a\n b")] +#[case::first_line_also_shifted("a\nb", 2, " a\n b")] +#[case::negative_delta_clamped_at_zero(" a\n b", -5, "a\nb")] +fn test_reindent_all_lines(#[case] content: &str, #[case] delta: isize, #[case] expected: &str) { + assert_eq!(reindent_all_lines(content, delta), expected); +} + +#[rstest] +#[case::unordered_bullet(false, 0, None, 2)] +#[case::ordered_single_digit(true, 0, None, 3)] +#[case::ordered_double_digit(true, 9, None, 4)] +#[case::ordered_with_start(true, 0, Some(10), 4)] +fn test_list_own_prefix_width( + #[case] ordered: bool, + #[case] index: usize, + #[case] start: Option<u32>, + #[case] expected: usize, +) { + assert_eq!(list_own_prefix_width(ordered, index, start), expected); +} From cde0de709af3494259919fc9d45aeecd9356dcbf Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Thu, 8 Oct 2026 13:09:42 +0900 Subject: [PATCH 47/88] =?UTF-8?q?=F0=9F=90=9B=20fix(markdown):=20keep=20he?= =?UTF-8?q?ading=20depths=20from=201=20to=206=20with=20a=20HeadingDepth=20?= =?UTF-8?q?type?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Heading.depth was a u8, so nothing stopped a depth of 0 or 7 and more: to_h("x", 9) wrote "######### x", which reads back as a paragraph, and set_attr("depth", 257) wrapped to 1. shift_right on a heading added to the u8 before checking the result, which overflows for large shifts. Heading.depth is now a HeadingDepth that only holds 1 to 6. It has H1 to H6, new and TryFrom<u8> for checked construction, saturating for clamping, and checked_add and saturating_sub for shifting. It serializes as the same number in JSON and prints as one in Debug. - to_h clamps the depth to 1 to 6. - set_attr("depth", ...) leaves the heading as it is for a value out of range. - shift_left and shift_right shift through HeadingDepth. - name() and NodeKind no longer need arms for depths that cannot occur. --- crates/mq-help/src/docs/table.rs | 2 +- crates/mq-lang/src/runtime/builtin.rs | 30 ++-- .../mq-lang/src/runtime/builtin/compress.rs | 2 +- crates/mq-lang/src/runtime/builtin/convert.rs | 77 +++++---- crates/mq-lang/src/tarn/tests.rs | 6 +- crates/mq-lang/tests/integration_tests.rs | 85 +++++----- crates/mq-markdown/src/lib.rs | 10 +- crates/mq-markdown/src/node.rs | 99 ++++++++++- crates/mq-markdown/src/node/attr.rs | 12 +- crates/mq-markdown/src/node/render.rs | 8 +- crates/mq-markdown/src/node/tests.rs | 158 ++++++++++++------ crates/mq-markdown/src/node_kind.rs | 4 +- crates/mq-markdown/src/parser/block.rs | 19 ++- crates/mq-markdown/src/parser/tree.rs | 4 +- crates/mq-run/src/output/table.rs | 2 +- 15 files changed, 350 insertions(+), 168 deletions(-) diff --git a/crates/mq-help/src/docs/table.rs b/crates/mq-help/src/docs/table.rs index 662f96716..91cd21c13 100644 --- a/crates/mq-help/src/docs/table.rs +++ b/crates/mq-help/src/docs/table.rs @@ -3640,7 +3640,7 @@ x = 1 BuiltinDoc { name: "to_h", aliases: &[], - description: "Creates a markdown heading node with the given value and depth.", + description: "Creates a markdown heading node with the given value and depth. A depth outside of 1 to 6 becomes the nearest of them.", params: &["value", "depth"], param_types: &["dynamic", "number"], returns: "markdown", diff --git a/crates/mq-lang/src/runtime/builtin.rs b/crates/mq-lang/src/runtime/builtin.rs index 01445af2c..e4c9c98fc 100644 --- a/crates/mq-lang/src/runtime/builtin.rs +++ b/crates/mq-lang/src/runtime/builtin.rs @@ -3182,14 +3182,14 @@ fn to_h_impl(_: &Ident, _: &RuntimeValue, args: Args, _: &SharedEnv) -> Result<R match args.as_slice() { [RuntimeValue::Markdown(node, _), RuntimeValue::Number(depth)] => { Ok(mq_markdown::Node::Heading(mq_markdown::Heading { - depth: (*depth).value() as u8, + depth: mq_markdown::HeadingDepth::saturating((*depth).value() as i64), values: node.node_values(), position: None, }) .into()) } [a, RuntimeValue::Number(depth)] if !a.is_none() => Ok(mq_markdown::Node::Heading(mq_markdown::Heading { - depth: (*depth).value() as u8, + depth: mq_markdown::HeadingDepth::saturating((*depth).value() as i64), values: vec![a.to_string().into()], position: None, }) @@ -3284,7 +3284,7 @@ fn to_md_name_impl(_: &Ident, _: &RuntimeValue, args: Args, _: &SharedEnv) -> Re fn md_heading_level_impl(_: &Ident, _: &RuntimeValue, args: Args, _: &SharedEnv) -> Result<RuntimeValue, Error> { let level = match args.as_slice() { [RuntimeValue::Markdown(node, _)] => match &**node { - mq_markdown::Node::Heading(mq_markdown::Heading { depth, .. }) if (1..=6).contains(depth) => *depth, + mq_markdown::Node::Heading(mq_markdown::Heading { depth, .. }) => depth.get(), _ => 0, }, _ => 0, @@ -4540,7 +4540,7 @@ fn shift_left_impl(ident: &Ident, _: &RuntimeValue, mut args: Args, _: &SharedEn if let mq_markdown::Node::Heading(heading) = runtime_value::markdown_mut(node) { let shift_amount = n.to_int().max(0).min(u8::MAX as i64) as u8; - heading.depth = heading.depth.saturating_sub(shift_amount).max(1); + heading.depth = heading.depth.saturating_sub(shift_amount); Ok(mq_markdown::Node::Heading(std::mem::take(heading)).into()) } else { Ok(RuntimeValue::Markdown(std::mem::take(node), selector.take())) @@ -4582,8 +4582,8 @@ fn shift_right_impl(ident: &Ident, _: &RuntimeValue, mut args: Args, _: &SharedE if let mq_markdown::Node::Heading(heading) = runtime_value::markdown_mut(node) { let shift_amount = n.to_int().max(0).min(u8::MAX as i64) as u8; - if heading.depth + shift_amount <= 6 { - heading.depth += shift_amount; + if let Some(depth) = heading.depth.checked_add(shift_amount) { + heading.depth = depth; } Ok(mq_markdown::Node::Heading(std::mem::take(heading)).into()) } else { @@ -6239,7 +6239,7 @@ pub fn eval_selector_with_args(node: &mq_markdown::Node, selector: &Selector, ar } if let mq_markdown::Node::Heading(mq_markdown::Heading { depth, .. }) = node { - depths.contains(depth) + depths.contains(&depth.get()) } else { false } @@ -7672,12 +7672,12 @@ mod tests { true )] #[case::heading_matching_depth( - Node::Heading(mq_markdown::Heading { depth: 2, values: vec!["test".to_string().into()], position: None }), + Node::Heading(mq_markdown::Heading { depth: mq_markdown::HeadingDepth::H2, values: vec!["test".to_string().into()], position: None }), Selector::Heading(Some(2)), true )] #[case::heading_wrong_depth( - Node::Heading(mq_markdown::Heading { depth: 2, values: vec!["test".to_string().into()], position: None }), + Node::Heading(mq_markdown::Heading { depth: mq_markdown::HeadingDepth::H2, values: vec!["test".to_string().into()], position: None }), Selector::Heading(Some(3)), false )] @@ -7881,7 +7881,7 @@ mod tests { }), ], position: None, - depth: 1, + depth: mq_markdown::HeadingDepth::H1, }); let result = eval_selector(&node, &Selector::Recursive); assert_eq!( @@ -7926,7 +7926,7 @@ mod tests { let heading = Node::Heading(mq_markdown::Heading { values: vec![inner_text.clone()], position: None, - depth: 2, + depth: mq_markdown::HeadingDepth::H2, }); let node = Node::Blockquote(mq_markdown::Blockquote { values: vec![heading.clone()], @@ -9766,25 +9766,25 @@ mod tests { #[rstest] #[case::heading_depth_match( - Node::Heading(mq_markdown::Heading { depth: 1, values: vec![], position: None }), + Node::Heading(mq_markdown::Heading { depth: mq_markdown::HeadingDepth::H1, values: vec![], position: None }), Selector::Heading(None), vec![RuntimeValue::Number(1.into())], true )] #[case::heading_depth_no_match( - Node::Heading(mq_markdown::Heading { depth: 2, values: vec![], position: None }), + Node::Heading(mq_markdown::Heading { depth: mq_markdown::HeadingDepth::H2, values: vec![], position: None }), Selector::Heading(None), vec![RuntimeValue::Number(1.into())], false )] #[case::heading_multi_depth_match( - Node::Heading(mq_markdown::Heading { depth: 2, values: vec![], position: None }), + Node::Heading(mq_markdown::Heading { depth: mq_markdown::HeadingDepth::H2, values: vec![], position: None }), Selector::Heading(None), vec![RuntimeValue::Number(1.into()), RuntimeValue::Number(2.into())], true )] #[case::heading_no_args_fallback( - Node::Heading(mq_markdown::Heading { depth: 1, values: vec![], position: None }), + Node::Heading(mq_markdown::Heading { depth: mq_markdown::HeadingDepth::H1, values: vec![], position: None }), Selector::Heading(None), vec![], true diff --git a/crates/mq-lang/src/runtime/builtin/compress.rs b/crates/mq-lang/src/runtime/builtin/compress.rs index 8d3b77f2f..6e1258faf 100644 --- a/crates/mq-lang/src/runtime/builtin/compress.rs +++ b/crates/mq-lang/src/runtime/builtin/compress.rs @@ -373,7 +373,7 @@ mod tests { fn heading(depth: u8, value: &str) -> Node { Node::Heading(Heading { - depth, + depth: mq_markdown::HeadingDepth::new(depth).expect("a heading depth"), values: vec![text(value)], position: None, }) diff --git a/crates/mq-lang/src/runtime/builtin/convert.rs b/crates/mq-lang/src/runtime/builtin/convert.rs index 5dff169c1..eb7a78166 100644 --- a/crates/mq-lang/src/runtime/builtin/convert.rs +++ b/crates/mq-lang/src/runtime/builtin/convert.rs @@ -12,7 +12,7 @@ use url::Url; #[derive(Debug, Clone, PartialEq, Eq)] pub(super) enum ConvertKind { Blockquote, - Heading(u8), + Heading(mq_markdown::HeadingDepth), HorizontalRule, Link(String), ListItem, @@ -57,12 +57,12 @@ impl TryFrom<&RuntimeValue> for Convert { fn try_from(value: &RuntimeValue) -> Result<Self, Self::Error> { match value { RuntimeValue::Symbol(symbol) => match symbol.to_string().as_str() { - "h1" => Ok(Convert::Markdown(ConvertKind::Heading(1))), - "h2" => Ok(Convert::Markdown(ConvertKind::Heading(2))), - "h3" => Ok(Convert::Markdown(ConvertKind::Heading(3))), - "h4" => Ok(Convert::Markdown(ConvertKind::Heading(4))), - "h5" => Ok(Convert::Markdown(ConvertKind::Heading(5))), - "h6" => Ok(Convert::Markdown(ConvertKind::Heading(6))), + "h1" => Ok(Convert::Markdown(ConvertKind::Heading(mq_markdown::HeadingDepth::H1))), + "h2" => Ok(Convert::Markdown(ConvertKind::Heading(mq_markdown::HeadingDepth::H2))), + "h3" => Ok(Convert::Markdown(ConvertKind::Heading(mq_markdown::HeadingDepth::H3))), + "h4" => Ok(Convert::Markdown(ConvertKind::Heading(mq_markdown::HeadingDepth::H4))), + "h5" => Ok(Convert::Markdown(ConvertKind::Heading(mq_markdown::HeadingDepth::H5))), + "h6" => Ok(Convert::Markdown(ConvertKind::Heading(mq_markdown::HeadingDepth::H6))), "html" => Ok(Convert::Html), "md5" => Ok(Convert::Md5), @@ -75,12 +75,12 @@ impl TryFrom<&RuntimeValue> for Convert { _ => Err(Error::InvalidConvert(symbol.to_string())), }, RuntimeValue::String(s) => match s.as_str() { - "#" => Ok(Convert::Markdown(ConvertKind::Heading(1))), - "##" => Ok(Convert::Markdown(ConvertKind::Heading(2))), - "###" => Ok(Convert::Markdown(ConvertKind::Heading(3))), - "####" => Ok(Convert::Markdown(ConvertKind::Heading(4))), - "#####" => Ok(Convert::Markdown(ConvertKind::Heading(5))), - "######" => Ok(Convert::Markdown(ConvertKind::Heading(6))), + "#" => Ok(Convert::Markdown(ConvertKind::Heading(mq_markdown::HeadingDepth::H1))), + "##" => Ok(Convert::Markdown(ConvertKind::Heading(mq_markdown::HeadingDepth::H2))), + "###" => Ok(Convert::Markdown(ConvertKind::Heading(mq_markdown::HeadingDepth::H3))), + "####" => Ok(Convert::Markdown(ConvertKind::Heading(mq_markdown::HeadingDepth::H4))), + "#####" => Ok(Convert::Markdown(ConvertKind::Heading(mq_markdown::HeadingDepth::H5))), + "######" => Ok(Convert::Markdown(ConvertKind::Heading(mq_markdown::HeadingDepth::H6))), ">" => Ok(Convert::Markdown(ConvertKind::Blockquote)), "-" => Ok(Convert::Markdown(ConvertKind::ListItem)), "~~" => Ok(Convert::Markdown(ConvertKind::Strikethrough)), @@ -962,23 +962,41 @@ mod tests { // Test convert::try_from #[rstest] - #[case::h1_symbol(RuntimeValue::Symbol(Ident::new("h1")), Convert::Markdown(ConvertKind::Heading(1)))] - #[case::h2_symbol(RuntimeValue::Symbol(Ident::new("h2")), Convert::Markdown(ConvertKind::Heading(2)))] - #[case::h3_symbol(RuntimeValue::Symbol(Ident::new("h3")), Convert::Markdown(ConvertKind::Heading(3)))] - #[case::h4_symbol(RuntimeValue::Symbol(Ident::new("h4")), Convert::Markdown(ConvertKind::Heading(4)))] - #[case::h5_symbol(RuntimeValue::Symbol(Ident::new("h5")), Convert::Markdown(ConvertKind::Heading(5)))] - #[case::h6_symbol(RuntimeValue::Symbol(Ident::new("h6")), Convert::Markdown(ConvertKind::Heading(6)))] + #[case::h1_symbol( + RuntimeValue::Symbol(Ident::new("h1")), + Convert::Markdown(ConvertKind::Heading(mq_markdown::HeadingDepth::H1)) + )] + #[case::h2_symbol( + RuntimeValue::Symbol(Ident::new("h2")), + Convert::Markdown(ConvertKind::Heading(mq_markdown::HeadingDepth::H2)) + )] + #[case::h3_symbol( + RuntimeValue::Symbol(Ident::new("h3")), + Convert::Markdown(ConvertKind::Heading(mq_markdown::HeadingDepth::H3)) + )] + #[case::h4_symbol( + RuntimeValue::Symbol(Ident::new("h4")), + Convert::Markdown(ConvertKind::Heading(mq_markdown::HeadingDepth::H4)) + )] + #[case::h5_symbol( + RuntimeValue::Symbol(Ident::new("h5")), + Convert::Markdown(ConvertKind::Heading(mq_markdown::HeadingDepth::H5)) + )] + #[case::h6_symbol( + RuntimeValue::Symbol(Ident::new("h6")), + Convert::Markdown(ConvertKind::Heading(mq_markdown::HeadingDepth::H6)) + )] #[case::html_symbol(RuntimeValue::Symbol(Ident::new("html")), Convert::Html)] #[case::md5_symbol(RuntimeValue::Symbol(Ident::new("md5")), Convert::Md5)] #[case::sha256_symbol(RuntimeValue::Symbol(Ident::new("sha256")), Convert::Sha256)] #[case::text_symbol(RuntimeValue::Symbol(Ident::new("text")), Convert::Text)] #[case::sh_symbol(RuntimeValue::Symbol(Ident::new("sh")), Convert::Shell)] - #[case::h1_string(RuntimeValue::String(Shared::new("#".to_string())), Convert::Markdown(ConvertKind::Heading(1)))] - #[case::h2_string(RuntimeValue::String(Shared::new("##".to_string())), Convert::Markdown(ConvertKind::Heading(2)))] - #[case::h3_string(RuntimeValue::String(Shared::new("###".to_string())), Convert::Markdown(ConvertKind::Heading(3)))] - #[case::h4_string(RuntimeValue::String(Shared::new("####".to_string())), Convert::Markdown(ConvertKind::Heading(4)))] - #[case::h5_string(RuntimeValue::String(Shared::new("#####".to_string())), Convert::Markdown(ConvertKind::Heading(5)))] - #[case::h6_string(RuntimeValue::String(Shared::new("######".to_string())), Convert::Markdown(ConvertKind::Heading(6)))] + #[case::h1_string(RuntimeValue::String(Shared::new("#".to_string())), Convert::Markdown(ConvertKind::Heading(mq_markdown::HeadingDepth::H1)))] + #[case::h2_string(RuntimeValue::String(Shared::new("##".to_string())), Convert::Markdown(ConvertKind::Heading(mq_markdown::HeadingDepth::H2)))] + #[case::h3_string(RuntimeValue::String(Shared::new("###".to_string())), Convert::Markdown(ConvertKind::Heading(mq_markdown::HeadingDepth::H3)))] + #[case::h4_string(RuntimeValue::String(Shared::new("####".to_string())), Convert::Markdown(ConvertKind::Heading(mq_markdown::HeadingDepth::H4)))] + #[case::h5_string(RuntimeValue::String(Shared::new("#####".to_string())), Convert::Markdown(ConvertKind::Heading(mq_markdown::HeadingDepth::H5)))] + #[case::h6_string(RuntimeValue::String(Shared::new("######".to_string())), Convert::Markdown(ConvertKind::Heading(mq_markdown::HeadingDepth::H6)))] #[case::blockquote_string(RuntimeValue::String(Shared::new(">".to_string())), Convert::Markdown(ConvertKind::Blockquote))] #[case::list_item_string(RuntimeValue::String(Shared::new("-".to_string())), Convert::Markdown(ConvertKind::ListItem))] #[case::strikethrough_string(RuntimeValue::String(Shared::new("~~".to_string())), Convert::Markdown(ConvertKind::Strikethrough))] @@ -1641,6 +1659,7 @@ mod tests { #[case(5)] #[case(6)] fn test_convert_all_heading_levels(#[case] depth: u8) { + let depth = mq_markdown::HeadingDepth::new(depth).expect("a heading depth"); let convert = Convert::Markdown(ConvertKind::Heading(depth)); let input = RuntimeValue::String(Shared::new("Test".to_string())); let result = convert.convert(&input); @@ -1748,7 +1767,7 @@ mod tests { position: None, }); let input = RuntimeValue::new_markdown(markdown_node); - let convert = Convert::Markdown(ConvertKind::Heading(2)); + let convert = Convert::Markdown(ConvertKind::Heading(mq_markdown::HeadingDepth::H2)); let result = convert.convert(&input); @@ -1758,13 +1777,13 @@ mod tests { let mq_markdown::Node::Heading(heading) = Shared::unwrap_or_clone(node) else { panic!("Expected Heading node") }; - assert_eq!(heading.depth, 2); + assert_eq!(heading.depth, mq_markdown::HeadingDepth::H2); } // Test convert with number input #[test] fn test_convert_number_to_heading() { - let convert = Convert::Markdown(ConvertKind::Heading(1)); + let convert = Convert::Markdown(ConvertKind::Heading(mq_markdown::HeadingDepth::H1)); let input = RuntimeValue::Number(42.into()); let result = convert.convert(&input); @@ -1774,7 +1793,7 @@ mod tests { let mq_markdown::Node::Heading(heading) = Shared::unwrap_or_clone(node) else { panic!("Expected Heading node") }; - assert_eq!(heading.depth, 1); + assert_eq!(heading.depth, mq_markdown::HeadingDepth::H1); assert_eq!(heading.values.len(), 1); } diff --git a/crates/mq-lang/src/tarn/tests.rs b/crates/mq-lang/src/tarn/tests.rs index c618f6b90..6581993c2 100644 --- a/crates/mq-lang/src/tarn/tests.rs +++ b/crates/mq-lang/src/tarn/tests.rs @@ -2011,7 +2011,7 @@ fn heading(depth: u8) -> RuntimeValue { RuntimeValue::new_markdown(mq_markdown::Node::Heading(mq_markdown::Heading { values: vec![], position: None, - depth, + depth: mq_markdown::HeadingDepth::new(depth).expect("a heading depth"), })) } @@ -2350,12 +2350,12 @@ fn markdown_selector_recurses_into_a_non_matching_container_to_find_matches_belo let matching_child = mq_markdown::Node::Heading(mq_markdown::Heading { values: vec![], position: None, - depth: 1, + depth: mq_markdown::HeadingDepth::H1, }); let outer = mq_markdown::Node::Heading(mq_markdown::Heading { values: vec![matching_child.clone(), text_node("no match anywhere")], position: None, - depth: 2, + depth: mq_markdown::HeadingDepth::H2, }); let code = ".h1"; let token_arena = Shared::new(SharedCell::new(Arena::new(100))); diff --git a/crates/mq-lang/tests/integration_tests.rs b/crates/mq-lang/tests/integration_tests.rs index 7df52ef79..25bc620b6 100644 --- a/crates/mq-lang/tests/integration_tests.rs +++ b/crates/mq-lang/tests/integration_tests.rs @@ -223,7 +223,7 @@ fn to_code_round_trip_preserves_compound_assign_result(mut engine: DefaultEngine vec![RuntimeValue::new_markdown(mq_markdown::Node::Heading(mq_markdown::Heading { values: vec![], position: None, - depth: 1, + depth: mq_markdown::HeadingDepth::H1, }))], Ok(vec![RuntimeValue::new_markdown(mq_markdown::Node::Text(mq_markdown::Text { value: "h1".to_string(), @@ -240,7 +240,7 @@ fn to_code_round_trip_preserves_compound_assign_result(mut engine: DefaultEngine vec![RuntimeValue::new_markdown(mq_markdown::Node::Heading(mq_markdown::Heading { values: vec![], position: None, - depth: 3, + depth: mq_markdown::HeadingDepth::H3, }))], Ok(vec![RuntimeValue::new_markdown(mq_markdown::Node::Text(mq_markdown::Text { value: "deep".to_string(), @@ -337,7 +337,7 @@ fn to_code_round_trip_preserves_compound_assign_result(mut engine: DefaultEngine vec![RuntimeValue::new_markdown(mq_markdown::Node::Heading(mq_markdown::Heading { values: vec![], position: None, - depth: 2, + depth: mq_markdown::HeadingDepth::H2, }))], Ok(vec![RuntimeValue::new_markdown(mq_markdown::Node::Text(mq_markdown::Text { value: "top-level".to_string(), @@ -400,7 +400,7 @@ fn to_code_round_trip_preserves_compound_assign_result(mut engine: DefaultEngine vec![RuntimeValue::new_markdown(mq_markdown::Node::Heading(mq_markdown::Heading { values: vec![], position: None, - depth: 1, + depth: mq_markdown::HeadingDepth::H1, }))], Ok(vec![RuntimeValue::new_markdown(mq_markdown::Node::Text(mq_markdown::Text { value: "other".to_string(), @@ -614,15 +614,15 @@ fn to_code_round_trip_preserves_compound_assign_result(mut engine: DefaultEngine vec![RuntimeValue::new_markdown(mq_markdown::Node::Heading(mq_markdown::Heading{ values: vec![ mq_markdown::Node::Link(mq_markdown::Link { url: mq_markdown::Url::new("url".to_string()), title: None, values: Vec::new(), position: None }), mq_markdown::Node::Image(mq_markdown::Image{ alt: "".to_string(), url: "url".to_string(), title: None, position: None }) - ], position: None, depth: 1 }))], + ], position: None, depth: mq_markdown::HeadingDepth::H1 }))], Ok(vec![RuntimeValue::new_markdown(mq_markdown::Node::Link(mq_markdown::Link { url: mq_markdown::Url::new("test".to_string()), title: None, values: Vec::new(), position: None }))].into()))] #[case::selector("nodes | .h", vec![ - RuntimeValue::new_markdown(mq_markdown::Node::Heading(mq_markdown::Heading{ values: vec![mq_markdown::Node::Text(mq_markdown::Text { value: "text".to_string(), position: None }),], position: None, depth: 1 })), + RuntimeValue::new_markdown(mq_markdown::Node::Heading(mq_markdown::Heading{ values: vec![mq_markdown::Node::Text(mq_markdown::Text { value: "text".to_string(), position: None }),], position: None, depth: mq_markdown::HeadingDepth::H1 })), RuntimeValue::String(Shared::new("test".to_string())), ], Ok(vec![ - RuntimeValue::new_markdown(mq_markdown::Node::Heading(mq_markdown::Heading{ values: vec![mq_markdown::Node::Text(mq_markdown::Text { value: "text".to_string(), position: None }),], position: None, depth: 1 })), + RuntimeValue::new_markdown(mq_markdown::Node::Heading(mq_markdown::Heading{ values: vec![mq_markdown::Node::Text(mq_markdown::Text { value: "text".to_string(), position: None }),], position: None, depth: mq_markdown::HeadingDepth::H1 })), RuntimeValue::NONE ].into()))] #[case::selector("nodes | .h", @@ -1173,7 +1173,7 @@ fn to_code_round_trip_preserves_compound_assign_result(mut engine: DefaultEngine vec![RuntimeValue::new_markdown(mq_markdown::Node::Heading(mq_markdown::Heading { values: vec![], position: None, - depth: 1, + depth: mq_markdown::HeadingDepth::H1, }))], Ok(vec![RuntimeValue::new_markdown(mq_markdown::Node::Text(mq_markdown::Text { value: "true".to_string(), @@ -1192,7 +1192,7 @@ fn to_code_round_trip_preserves_compound_assign_result(mut engine: DefaultEngine vec![RuntimeValue::new_markdown(mq_markdown::Node::Heading(mq_markdown::Heading { values: vec![], position: None, - depth: 1, + depth: mq_markdown::HeadingDepth::H1, }))], Ok(vec![RuntimeValue::new_markdown(mq_markdown::Node::Text(mq_markdown::Text { value: "true".to_string(), @@ -1202,7 +1202,7 @@ fn to_code_round_trip_preserves_compound_assign_result(mut engine: DefaultEngine vec![RuntimeValue::new_markdown(mq_markdown::Node::Heading(mq_markdown::Heading { values: vec![], position: None, - depth: 2, + depth: mq_markdown::HeadingDepth::H2, }))], Ok(vec![RuntimeValue::new_markdown(mq_markdown::Node::Text(mq_markdown::Text { value: "false".to_string(), @@ -1221,7 +1221,7 @@ fn to_code_round_trip_preserves_compound_assign_result(mut engine: DefaultEngine vec![RuntimeValue::new_markdown(mq_markdown::Node::Heading(mq_markdown::Heading { values: vec![], position: None, - depth: 2, + depth: mq_markdown::HeadingDepth::H2, }))], Ok(vec![RuntimeValue::new_markdown(mq_markdown::Node::Text(mq_markdown::Text { value: "true".to_string(), @@ -1231,7 +1231,7 @@ fn to_code_round_trip_preserves_compound_assign_result(mut engine: DefaultEngine vec![RuntimeValue::new_markdown(mq_markdown::Node::Heading(mq_markdown::Heading { values: vec![], position: None, - depth: 3, + depth: mq_markdown::HeadingDepth::H3, }))], Ok(vec![RuntimeValue::new_markdown(mq_markdown::Node::Text(mq_markdown::Text { value: "false".to_string(), @@ -1250,7 +1250,7 @@ fn to_code_round_trip_preserves_compound_assign_result(mut engine: DefaultEngine vec![RuntimeValue::new_markdown(mq_markdown::Node::Heading(mq_markdown::Heading { values: vec![], position: None, - depth: 3, + depth: mq_markdown::HeadingDepth::H3, }))], Ok(vec![RuntimeValue::new_markdown(mq_markdown::Node::Text(mq_markdown::Text { value: "true".to_string(), @@ -1260,7 +1260,7 @@ fn to_code_round_trip_preserves_compound_assign_result(mut engine: DefaultEngine vec![RuntimeValue::new_markdown(mq_markdown::Node::Heading(mq_markdown::Heading { values: vec![], position: None, - depth: 4, + depth: mq_markdown::HeadingDepth::H4, }))], Ok(vec![RuntimeValue::new_markdown(mq_markdown::Node::Text(mq_markdown::Text { value: "false".to_string(), @@ -1279,7 +1279,7 @@ fn to_code_round_trip_preserves_compound_assign_result(mut engine: DefaultEngine vec![RuntimeValue::new_markdown(mq_markdown::Node::Heading(mq_markdown::Heading { values: vec![], position: None, - depth: 4, + depth: mq_markdown::HeadingDepth::H4, }))], Ok(vec![RuntimeValue::new_markdown(mq_markdown::Node::Text(mq_markdown::Text { value: "true".to_string(), @@ -1289,7 +1289,7 @@ fn to_code_round_trip_preserves_compound_assign_result(mut engine: DefaultEngine vec![RuntimeValue::new_markdown(mq_markdown::Node::Heading(mq_markdown::Heading { values: vec![], position: None, - depth: 5, + depth: mq_markdown::HeadingDepth::H5, }))], Ok(vec![RuntimeValue::new_markdown(mq_markdown::Node::Text(mq_markdown::Text { value: "false".to_string(), @@ -1308,7 +1308,7 @@ fn to_code_round_trip_preserves_compound_assign_result(mut engine: DefaultEngine vec![RuntimeValue::new_markdown(mq_markdown::Node::Heading(mq_markdown::Heading { values: vec![], position: None, - depth: 5, + depth: mq_markdown::HeadingDepth::H5, }))], Ok(vec![RuntimeValue::new_markdown(mq_markdown::Node::Text(mq_markdown::Text { value: "true".to_string(), @@ -1318,7 +1318,7 @@ fn to_code_round_trip_preserves_compound_assign_result(mut engine: DefaultEngine vec![RuntimeValue::new_markdown(mq_markdown::Node::Heading(mq_markdown::Heading { values: vec![], position: None, - depth: 4, + depth: mq_markdown::HeadingDepth::H4, }))], Ok(vec![RuntimeValue::new_markdown(mq_markdown::Node::Text(mq_markdown::Text { value: "false".to_string(), @@ -1337,7 +1337,7 @@ fn to_code_round_trip_preserves_compound_assign_result(mut engine: DefaultEngine vec![RuntimeValue::new_markdown(mq_markdown::Node::Heading(mq_markdown::Heading { values: vec![], position: None, - depth: 6, + depth: mq_markdown::HeadingDepth::H6, }))], Ok(vec![RuntimeValue::new_markdown(mq_markdown::Node::Text(mq_markdown::Text { value: "true".to_string(), @@ -1347,7 +1347,7 @@ fn to_code_round_trip_preserves_compound_assign_result(mut engine: DefaultEngine vec![RuntimeValue::new_markdown(mq_markdown::Node::Heading(mq_markdown::Heading { values: vec![], position: None, - depth: 5, + depth: mq_markdown::HeadingDepth::H5, }))], Ok(vec![RuntimeValue::new_markdown(mq_markdown::Node::Text(mq_markdown::Text { value: "false".to_string(), @@ -1468,7 +1468,7 @@ fn to_code_round_trip_preserves_compound_assign_result(mut engine: DefaultEngine vec![RuntimeValue::new_markdown(mq_markdown::Node::Heading(mq_markdown::Heading { values: vec![], position: None, - depth: 1, + depth: mq_markdown::HeadingDepth::H1, }))], Ok(vec![RuntimeValue::new_markdown(mq_markdown::Node::Text(mq_markdown::Text { value: "false".to_string(), @@ -1788,7 +1788,7 @@ fn to_code_round_trip_preserves_compound_assign_result(mut engine: DefaultEngine mq_markdown::Node::Link(mq_markdown::Link { url: mq_markdown::Url::new("url".to_string()), title: None, values: Vec::new(), position: None }), ], position: None, - depth: 1, + depth: mq_markdown::HeadingDepth::H1, })), ], Ok(vec![ @@ -1809,7 +1809,7 @@ fn to_code_round_trip_preserves_compound_assign_result(mut engine: DefaultEngine mq_markdown::Node::Text(mq_markdown::Text { value: "nested".to_string(), position: None }), ], position: None, - depth: 2, + depth: mq_markdown::HeadingDepth::H2, }), ], position: None, @@ -1822,7 +1822,7 @@ fn to_code_round_trip_preserves_compound_assign_result(mut engine: DefaultEngine mq_markdown::Node::Text(mq_markdown::Text { value: "nested".to_string(), position: None }), ], position: None, - depth: 2, + depth: mq_markdown::HeadingDepth::H2, })), ].into()))] #[case::recursive_selector_pipe_filter("nodes | .. | filter(fn(x): select(x, .text);)", @@ -1833,7 +1833,7 @@ fn to_code_round_trip_preserves_compound_assign_result(mut engine: DefaultEngine mq_markdown::Node::Link(mq_markdown::Link { url: mq_markdown::Url::new("url".to_string()), title: None, values: Vec::new(), position: None }), ], position: None, - depth: 1, + depth: mq_markdown::HeadingDepth::H1, })), ], Ok(vec![ @@ -2269,42 +2269,42 @@ fn to_code_round_trip_preserves_compound_assign_result(mut engine: DefaultEngine #[case::shift_right_header_level_h1("to_markdown(\"# Heading 1\") | first() | shift_right(1)", vec![RuntimeValue::None], Ok(vec![RuntimeValue::new_markdown(mq_markdown::Node::Heading(mq_markdown::Heading { - depth: 2, + depth: mq_markdown::HeadingDepth::H2, values: vec!["Heading 1".to_string().into()], position: None }))].into()))] #[case::shift_right_header_level_h1_operator("let md = do to_markdown(\"# Heading 1\") | first(); | md >> 1", vec![RuntimeValue::None], Ok(vec![RuntimeValue::new_markdown(mq_markdown::Node::Heading(mq_markdown::Heading { - depth: 2, + depth: mq_markdown::HeadingDepth::H2, values: vec!["Heading 1".to_string().into()], position: None }))].into()))] #[case::shift_right_header_level_h6("to_markdown(\"###### Heading 6\") | first() | shift_right(1)", vec![RuntimeValue::None], Ok(vec![RuntimeValue::new_markdown(mq_markdown::Node::Heading(mq_markdown::Heading { - depth: 6, + depth: mq_markdown::HeadingDepth::H6, values: vec!["Heading 6".to_string().into()], position: None }))].into()))] #[case::shift_left_header_level_h2("to_markdown(\"## Heading 2\") | first() | shift_left(1)", vec![RuntimeValue::None], Ok(vec![RuntimeValue::new_markdown(mq_markdown::Node::Heading(mq_markdown::Heading { - depth: 1, + depth: mq_markdown::HeadingDepth::H1, values: vec!["Heading 2".to_string().into()], position: None }))].into()))] #[case::shift_left_header_level_h2_via_binding("let md = do to_markdown(\"## Heading 2\") | first(); | md << 1", vec![RuntimeValue::None], Ok(vec![RuntimeValue::new_markdown(mq_markdown::Node::Heading(mq_markdown::Heading { - depth: 1, + depth: mq_markdown::HeadingDepth::H1, values: vec!["Heading 2".to_string().into()], position: None }))].into()))] #[case::shift_left_header_level_h1("to_markdown(\"# Heading 1\") | first() | shift_left(1)", vec![RuntimeValue::None], Ok(vec![RuntimeValue::new_markdown(mq_markdown::Node::Heading(mq_markdown::Heading { - depth: 1, + depth: mq_markdown::HeadingDepth::H1, values: vec!["Heading 1".to_string().into()], position: None }))].into()))] @@ -2323,35 +2323,35 @@ fn to_code_round_trip_preserves_compound_assign_result(mut engine: DefaultEngine #[case::convert_string_to_h1_function("convert(\"Hello\", :h1)", vec![RuntimeValue::None], Ok(vec![RuntimeValue::new_markdown(mq_markdown::Node::Heading(mq_markdown::Heading { - depth: 1, + depth: mq_markdown::HeadingDepth::H1, values: vec!["Hello".to_string().into()], position: None, }))].into()))] #[case::convert_string_to_h1_operator("\"Hello\" @ :h1", vec![RuntimeValue::None], Ok(vec![RuntimeValue::new_markdown(mq_markdown::Node::Heading(mq_markdown::Heading { - depth: 1, + depth: mq_markdown::HeadingDepth::H1, values: vec!["Hello".to_string().into()], position: None, }))].into()))] #[case::convert_string_to_h2_operator("\"Hello\" @ :h2", vec![RuntimeValue::None], Ok(vec![RuntimeValue::new_markdown(mq_markdown::Node::Heading(mq_markdown::Heading { - depth: 2, + depth: mq_markdown::HeadingDepth::H2, values: vec!["Hello".to_string().into()], position: None, }))].into()))] #[case::convert_string_to_h1_via_string_operator("\"Hello\" @ \"#\"", vec![RuntimeValue::None], Ok(vec![RuntimeValue::new_markdown(mq_markdown::Node::Heading(mq_markdown::Heading { - depth: 1, + depth: mq_markdown::HeadingDepth::H1, values: vec!["Hello".to_string().into()], position: None, }))].into()))] #[case::convert_string_to_h2_via_string_operator("\"Hello\" @ \"##\"", vec![RuntimeValue::None], Ok(vec![RuntimeValue::new_markdown(mq_markdown::Node::Heading(mq_markdown::Heading { - depth: 2, + depth: mq_markdown::HeadingDepth::H2, values: vec!["Hello".to_string().into()], position: None, }))].into()))] @@ -2539,7 +2539,7 @@ fn to_code_round_trip_preserves_compound_assign_result(mut engine: DefaultEngine // type pattern `:markdown` matches any markdown node value #[case::match_type_markdown( r#"match (.) do | :markdown: "is markdown" | _: "other" end"#, - vec![RuntimeValue::new_markdown(mq_markdown::Node::Heading(mq_markdown::Heading { depth: 1, values: vec!["Title".to_string().into()], position: None }))], + vec![RuntimeValue::new_markdown(mq_markdown::Node::Heading(mq_markdown::Heading { depth: mq_markdown::HeadingDepth::H1, values: vec!["Title".to_string().into()], position: None }))], Ok(vec![RuntimeValue::new_markdown(mq_markdown::Node::Text(mq_markdown::Text { value: "is markdown".to_string(), position: None }))].into()))] // type pattern `:markdown` does not match a plain string #[case::match_type_markdown_no_match_on_string( @@ -2574,7 +2574,7 @@ fn to_code_round_trip_preserves_compound_assign_result(mut engine: DefaultEngine r#""<h1>Hello</h1>" | from_html() | first()"#, vec![RuntimeValue::None], Ok(vec![RuntimeValue::new_markdown(mq_markdown::Node::Heading(mq_markdown::Heading { - depth: 1, + depth: mq_markdown::HeadingDepth::H1, values: vec!["Hello".to_string().into()], position: None, }))].into()))] @@ -2648,7 +2648,7 @@ fn to_code_round_trip_preserves_compound_assign_result(mut engine: DefaultEngine #[case::token_count_no_model_simple(r#"token_count("Hello, world!")"#, vec![RuntimeValue::None], Ok(vec![RuntimeValue::Number(4.into())].into()))] #[case::token_count_no_model_markdown(r#"to_md_text("Hello, world!") | token_count()"#, vec![RuntimeValue::None], Ok(vec![RuntimeValue::Number(4.into())].into()))] #[case::token_count_no_model_none(r#"token_count(None)"#, vec![RuntimeValue::None], Ok(vec![RuntimeValue::Number(0.into())].into()))] -#[case::token_compress_under_budget(r##"to_markdown("# Title") | token_compress(1000)"##, vec![RuntimeValue::None], Ok(vec![RuntimeValue::Array(Shared::new(vec![RuntimeValue::new_markdown(mq_markdown::Node::Heading(mq_markdown::Heading{depth: 1, values: vec!["Title".to_string().into()], position: None}))]))].into()))] +#[case::token_compress_under_budget(r##"to_markdown("# Title") | token_compress(1000)"##, vec![RuntimeValue::None], Ok(vec![RuntimeValue::Array(Shared::new(vec![RuntimeValue::new_markdown(mq_markdown::Node::Heading(mq_markdown::Heading{depth: mq_markdown::HeadingDepth::H1, values: vec!["Title".to_string().into()], position: None}))]))].into()))] #[case::token_compress_none(r#"token_compress(None, 100)"#, vec![RuntimeValue::None], Ok(vec![RuntimeValue::Array(Shared::new(vec![]))].into()))] #[case::explode_simple(r##"explode("abc")"##, vec![RuntimeValue::None], Ok(vec![RuntimeValue::Array(Shared::new(vec![RuntimeValue::Number(97.into()), RuntimeValue::Number(98.into()), RuntimeValue::Number(99.into())]))].into()))] #[case::implode_simple(r##"implode([97, 98, 99])"##, vec![RuntimeValue::None], Ok(vec![RuntimeValue::String(Shared::new("abc".to_string()))].into()))] @@ -2656,8 +2656,13 @@ fn to_code_round_trip_preserves_compound_assign_result(mut engine: DefaultEngine #[case::nan_builtin("nan() | is_nan()", vec![RuntimeValue::None], Ok(vec![RuntimeValue::Boolean(true)].into()))] #[case::infinite_builtin("infinite() > 0", vec![RuntimeValue::None], Ok(vec![RuntimeValue::Boolean(true)].into()))] #[case::to_md_text_simple(r##"to_md_text("hello")"##, vec![RuntimeValue::None], Ok(vec![RuntimeValue::new_markdown(mq_markdown::Node::Text(mq_markdown::Text{value: "hello".to_string(), position: None}))].into()))] -#[case::to_h_simple(r##"to_h("title", 1)"##, vec![RuntimeValue::None], Ok(vec![RuntimeValue::new_markdown(mq_markdown::Node::Heading(mq_markdown::Heading{depth: 1, values: vec!["title".to_string().into()], position: None}))].into()))] +#[case::to_h_simple(r##"to_h("title", 1)"##, vec![RuntimeValue::None], Ok(vec![RuntimeValue::new_markdown(mq_markdown::Node::Heading(mq_markdown::Heading{depth: mq_markdown::HeadingDepth::H1, values: vec!["title".to_string().into()], position: None}))].into()))] #[case::to_h_none("to_h(None, 1)", vec![RuntimeValue::None], Ok(vec![RuntimeValue::None].into()))] +#[case::to_h_depth_above_six(r##"to_h("title", 9)"##, vec![RuntimeValue::None], Ok(vec![RuntimeValue::new_markdown(mq_markdown::Node::Heading(mq_markdown::Heading{depth: mq_markdown::HeadingDepth::H6, values: vec!["title".to_string().into()], position: None}))].into()))] +#[case::to_h_depth_below_one(r##"to_h("title", 0)"##, vec![RuntimeValue::None], Ok(vec![RuntimeValue::new_markdown(mq_markdown::Node::Heading(mq_markdown::Heading{depth: mq_markdown::HeadingDepth::H1, values: vec!["title".to_string().into()], position: None}))].into()))] +#[case::shift_right_heading_past_six(r##"to_h("title", 6) | shift_right(255)"##, vec![RuntimeValue::None], Ok(vec![RuntimeValue::new_markdown(mq_markdown::Node::Heading(mq_markdown::Heading{depth: mq_markdown::HeadingDepth::H6, values: vec!["title".to_string().into()], position: None}))].into()))] +#[case::shift_right_heading(r##"to_h("title", 2) | shift_right(3)"##, vec![RuntimeValue::None], Ok(vec![RuntimeValue::new_markdown(mq_markdown::Node::Heading(mq_markdown::Heading{depth: mq_markdown::HeadingDepth::H5, values: vec!["title".to_string().into()], position: None}))].into()))] +#[case::shift_left_heading_past_one(r##"to_h("title", 2) | shift_left(5)"##, vec![RuntimeValue::None], Ok(vec![RuntimeValue::new_markdown(mq_markdown::Node::Heading(mq_markdown::Heading{depth: mq_markdown::HeadingDepth::H1, values: vec!["title".to_string().into()], position: None}))].into()))] #[case::to_code_none(r#"to_code(None, "rust")"#, vec![RuntimeValue::None], Ok(vec![RuntimeValue::None].into()))] #[case::to_math_none("to_math(None)", vec![RuntimeValue::None], Ok(vec![RuntimeValue::None].into()))] #[case::to_math_inline_none("to_math_inline(None)", vec![RuntimeValue::None], Ok(vec![RuntimeValue::None].into()))] diff --git a/crates/mq-markdown/src/lib.rs b/crates/mq-markdown/src/lib.rs index f6ff14efd..9e42c3664 100644 --- a/crates/mq-markdown/src/lib.rs +++ b/crates/mq-markdown/src/lib.rs @@ -91,11 +91,11 @@ mod parser; pub use markdown::{Markdown, ParseOptions, to_html}; pub use node::{ Blockquote, Break, Code, CodeInline, ColorTheme, Definition, Delete, Emphasis, Footnote, FootnoteRef, Fragment, - Heading, HorizontalRule, HorizontalRuleMarker, Html, Image, ImageRef, Link, LinkRef, List, ListMarker, ListStyle, - Math, MathInline, MdxAttributeContent, MdxAttributeValue, MdxFlowExpression, MdxJsEsm, MdxJsxAttribute, - MdxJsxFlowElement, MdxJsxTextElement, MdxTextExpression, Node, Point, Position, RenderOptions, Strong, TableAlign, - TableAlignKind, TableCell, TableRow, Text, Title, TitleSurroundStyle, Toml, Url, UrlSurroundStyle, Yaml, - attr_value::AttrValue, + Heading, HeadingDepth, HorizontalRule, HorizontalRuleMarker, Html, Image, ImageRef, Link, LinkRef, List, + ListMarker, ListStyle, Math, MathInline, MdxAttributeContent, MdxAttributeValue, MdxFlowExpression, MdxJsEsm, + MdxJsxAttribute, MdxJsxFlowElement, MdxJsxTextElement, MdxTextExpression, Node, Point, Position, RenderOptions, + Strong, TableAlign, TableAlignKind, TableCell, TableRow, Text, Title, TitleSurroundStyle, Toml, Url, + UrlSurroundStyle, Yaml, attr_value::AttrValue, }; pub use node_kind::{AttrSpec, AttrType, NodeKind}; pub use parser::{MdxDiagnostic, MdxError, MdxErrorKind, MdxFound, MdxPlace}; diff --git a/crates/mq-markdown/src/node.rs b/crates/mq-markdown/src/node.rs index 3fb5634bf..fab572755 100644 --- a/crates/mq-markdown/src/node.rs +++ b/crates/mq-markdown/src/node.rs @@ -617,6 +617,96 @@ pub struct LinkRef { pub position: Option<Position>, } +/// The depth of a heading, from 1 to 6. +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[cfg_attr( + feature = "json", + derive(serde::Serialize, serde::Deserialize), + serde(try_from = "u8", into = "u8") +)] +pub struct HeadingDepth(u8); + +impl fmt::Debug for HeadingDepth { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Debug::fmt(&self.0, f) + } +} + +impl HeadingDepth { + pub const H1: Self = Self(1); + pub const H2: Self = Self(2); + pub const H3: Self = Self(3); + pub const H4: Self = Self(4); + pub const H5: Self = Self(5); + pub const H6: Self = Self(6); + + /// The depth `depth`, if it is from 1 to 6. + pub const fn new(depth: u8) -> Option<Self> { + if depth >= 1 && depth <= 6 { + Some(Self(depth)) + } else { + None + } + } + + /// The depth nearest to `depth` from 1 to 6. + pub fn saturating(depth: i64) -> Self { + Self(depth.clamp(1, 6) as u8) + } + + pub const fn get(self) -> u8 { + self.0 + } + + /// The depth `levels` deeper, if that is still at most 6. + pub fn checked_add(self, levels: u8) -> Option<Self> { + self.0.checked_add(levels).and_then(Self::new) + } + + /// The depth `levels` shallower, or 1 when that is less. + pub fn saturating_sub(self, levels: u8) -> Self { + Self(self.0.saturating_sub(levels).max(1)) + } +} + +impl Default for HeadingDepth { + fn default() -> Self { + Self::H1 + } +} + +impl TryFrom<u8> for HeadingDepth { + type Error = InvalidHeadingDepth; + + fn try_from(depth: u8) -> Result<Self, Self::Error> { + Self::new(depth).ok_or(InvalidHeadingDepth(depth)) + } +} + +impl From<HeadingDepth> for u8 { + fn from(depth: HeadingDepth) -> Self { + depth.0 + } +} + +impl Display for HeadingDepth { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.0.fmt(f) + } +} + +/// A heading depth outside of 1 to 6. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct InvalidHeadingDepth(pub u8); + +impl Display for InvalidHeadingDepth { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "heading depth {} is not from 1 to 6", self.0) + } +} + +impl std::error::Error for InvalidHeadingDepth {} + #[derive(Debug, Clone, PartialEq, Default)] #[cfg_attr( feature = "json", @@ -624,7 +714,7 @@ pub struct LinkRef { serde(rename_all = "camelCase", tag = "type") )] pub struct Heading { - pub depth: u8, + pub depth: HeadingDepth, pub values: Vec<Node>, #[cfg_attr(feature = "json", serde(skip_serializing_if = "Option::is_none"))] pub position: Option<Position>, @@ -1289,14 +1379,13 @@ impl Node { Self::Break { .. } => "break".into(), Self::Definition(_) => "definition".into(), Self::Delete(_) => "delete".into(), - Self::Heading(Heading { depth, .. }) => match depth { + Self::Heading(Heading { depth, .. }) => match depth.get() { 1 => "h1".into(), 2 => "h2".into(), 3 => "h3".into(), 4 => "h4".into(), 5 => "h5".into(), - 6 => "h6".into(), - _ => "h".into(), + _ => "h6".into(), }, Self::Emphasis(_) => "emphasis".into(), Self::Footnote(_) => "footnote".into(), @@ -1690,7 +1779,7 @@ impl Node { depth: heading_depth, .. }) = &self { - depth.is_none() || *heading_depth == depth.unwrap() + depth.is_none_or(|depth| heading_depth.get() == depth) } else { false } diff --git a/crates/mq-markdown/src/node/attr.rs b/crates/mq-markdown/src/node/attr.rs index ad5429b87..69fdc21b6 100644 --- a/crates/mq-markdown/src/node/attr.rs +++ b/crates/mq-markdown/src/node/attr.rs @@ -132,7 +132,7 @@ impl Node { _ => None, }, Node::Heading(Heading { depth, values, .. }) => match attr { - attr_keys::DEPTH | attr_keys::LEVEL => Some(AttrValue::Integer(*depth as i64)), + attr_keys::DEPTH | attr_keys::LEVEL => Some(AttrValue::Integer(i64::from(depth.get()))), attr_keys::VALUE => Some(AttrValue::String(values_to_string(values, &RenderOptions::default()))), attr_keys::VALUES | attr_keys::CHILDREN => Some(AttrValue::Array(values.clone())), _ => None, @@ -344,10 +344,14 @@ impl Node { }, Node::Heading(h) => match attr { attr_keys::DEPTH | attr_keys::LEVEL => { - h.depth = match value { - AttrValue::Integer(i) => i as u8, - _ => value_str.parse::<u8>().unwrap_or(h.depth), + // A depth outside of 1 to 6 leaves the heading as it is. + let depth = match value { + AttrValue::Integer(i) => u8::try_from(i).ok(), + _ => value_str.parse::<u8>().ok(), }; + if let Some(depth) = depth.and_then(HeadingDepth::new) { + h.depth = depth; + } } _ => (), }, diff --git a/crates/mq-markdown/src/node/render.rs b/crates/mq-markdown/src/node/render.rs index a22807944..3691a73ed 100644 --- a/crates/mq-markdown/src/node/render.rs +++ b/crates/mq-markdown/src/node/render.rs @@ -191,8 +191,8 @@ impl Node { } }; // Multi-line content must stay setext for depths 1-2; ATX has no setext form. - if text.contains('\n') && matches!(depth, 1 | 2) { - let underline = if *depth == 1 { "===" } else { "---" }; + if text.contains('\n') && *depth <= HeadingDepth::H2 { + let underline = if *depth == HeadingDepth::H1 { "===" } else { "---" }; format!("{}{}\n{}{}", hs, text, underline, he) } else { // A trailing `#` run reads back as an ATX closing sequence and gets @@ -210,7 +210,7 @@ impl Node { None if !text.is_empty() => format!("\\{text}"), _ => text, }; - format!("{}{} {}{}", hs, "#".repeat(*depth as usize), text, he) + format!("{}{} {}{}", hs, "#".repeat(usize::from(depth.get())), text, he) } } Self::Html(Html { value, .. }) => { @@ -1222,7 +1222,7 @@ mod tests { depth in 1u8..=6, ) { let value = format!("{}{}", body, "#".repeat(hashes)); - let node = Node::Heading(Heading { depth, values: vec![value.into()], position: None }); + let node = Node::Heading(Heading { depth: HeadingDepth::saturating(i64::from(depth)), values: vec![value.into()], position: None }); let _ = node.to_string_with(&RenderOptions::default()); } diff --git a/crates/mq-markdown/src/node/tests.rs b/crates/mq-markdown/src/node/tests.rs index 3d6b04440..1aba7116d 100644 --- a/crates/mq-markdown/src/node/tests.rs +++ b/crates/mq-markdown/src/node/tests.rs @@ -184,9 +184,9 @@ fn map_values_into_owned_moves_nodes_through_the_callback() { #[case::strong(Node::Strong(Strong{values: vec!["test".to_string().into()], position: None }), "test".to_string(), Node::Strong(Strong{values: vec!["test".to_string().into()], position: None }))] -#[case::heading(Node::Heading(Heading {depth: 1, values: vec!["test".to_string().into()], position: None }), +#[case::heading(Node::Heading(Heading {depth: HeadingDepth::H1, values: vec!["test".to_string().into()], position: None }), "test".to_string(), - Node::Heading(Heading{depth: 1, values: vec!["test".to_string().into()], position: None }))] + Node::Heading(Heading{depth: HeadingDepth::H1, values: vec!["test".to_string().into()], position: None }))] #[case::link(Node::Link(Link {url: Url::new("test".to_string()), values: Vec::new(), title: None, position: None }), "test".to_string(), Node::Link(Link{url: Url::new("test".to_string()), values: Vec::new(), title: None, position: None }))] @@ -349,13 +349,13 @@ fn test_with_value(#[case] node: Node, #[case] input: String, #[case] expected: Node::Text(Text{value: "new".to_string(), position: None}), Node::Text(Text{value: "second".to_string(), position: None}) ], position: None}))] -#[case(Node::Heading(Heading{depth: 1, values: vec![ +#[case(Node::Heading(Heading{depth: HeadingDepth::H1, values: vec![ Node::Text(Text{value: "first".to_string(), position: None}), Node::Text(Text{value: "second".to_string(), position: None}) ], position: None}), "new", 1, - Node::Heading(Heading{depth: 1, values: vec![ + Node::Heading(Heading{depth: HeadingDepth::H1, values: vec![ Node::Text(Text{value: "first".to_string(), position: None}), Node::Text(Text{value: "new".to_string(), position: None}) ], position: None}))] @@ -579,9 +579,9 @@ fn test_is_code(#[case] node: Node, #[case] expected: bool, #[case] lang: Option } #[rstest] -#[case(Node::Heading(Heading{depth: 1, values: vec!["test".to_string().into()], position: None}), true, Some(1))] -#[case(Node::Heading(Heading{depth: 2, values: vec!["test".to_string().into()], position: None}), false, Some(1))] -#[case(Node::Heading(Heading{depth: 1, values: vec!["test".to_string().into()], position: None}), true, None)] +#[case(Node::Heading(Heading{depth: HeadingDepth::H1, values: vec!["test".to_string().into()], position: None}), true, Some(1))] +#[case(Node::Heading(Heading{depth: HeadingDepth::H2, values: vec!["test".to_string().into()], position: None}), false, Some(1))] +#[case(Node::Heading(Heading{depth: HeadingDepth::H1, values: vec!["test".to_string().into()], position: None}), true, None)] #[case(Node::Text(Text{value: "test".to_string(), position: None}), false, None)] fn test_is_heading(#[case] node: Node, #[case] expected: bool, #[case] depth: Option<u8>) { assert_eq!(node.is_heading(depth), expected); @@ -816,7 +816,7 @@ fn test_is_html(#[case] node: Node, #[case] expected: bool) { &Node::Emphasis(Emphasis{values: vec!["test".to_string().into()], position: None})), vec!["test".to_string().into()])] #[case(Node::node_values( - &Node::Heading(Heading{depth: 1, values: vec!["test".to_string().into()], position: None})), + &Node::Heading(Heading{depth: HeadingDepth::H1, values: vec!["test".to_string().into()], position: None})), vec!["test".to_string().into()])] #[case(Node::node_values( &Node::List(List{ marker: None,values: vec!["test".to_string().into()], ordered: false, level: 1, checked: Some(false), index: 0, start: None, spread: false, position: None})), @@ -946,15 +946,15 @@ fn test_is_msx_js_esm(#[case] node: Node, #[case] expected: bool) { #[case::emphasis_nested_in_emphasis_alternates_delimiter(Node::Emphasis(Emphasis{values: vec![Node::Emphasis(Emphasis{values: vec!["foo".to_string().into()], position: None})], position: None}), RenderOptions::default(), "_*foo*_")] #[case::footnote(Node::Footnote(Footnote{ident: "id".to_string(), values: vec![attr_keys::LABEL.to_string().into()], position: None}), RenderOptions::default(), "[^id]: label")] #[case::footnote_ref(Node::FootnoteRef(FootnoteRef{ident: attr_keys::LABEL.to_string(), label: Some(attr_keys::LABEL.to_string()), position: None}), RenderOptions::default(), "[^label]")] -#[case::heading(Node::Heading(Heading{depth: 1, values: vec!["test".to_string().into()], position: None}), RenderOptions::default(), "# test")] -#[case::heading(Node::Heading(Heading{depth: 3, values: vec!["test".to_string().into()], position: None}), RenderOptions::default(), "### test")] -#[case::heading_multiline_h1_stays_setext(Node::Heading(Heading{depth: 1, values: vec!["Foo\nBar".to_string().into()], position: None}), RenderOptions::default(), "Foo\nBar\n===")] -#[case::heading_multiline_h2_stays_setext(Node::Heading(Heading{depth: 2, values: vec!["Foo\nBar".to_string().into()], position: None}), RenderOptions::default(), "Foo\nBar\n---")] -#[case::heading_multiline_h3_joins_with_space(Node::Heading(Heading{depth: 3, values: vec!["Foo\nBar".to_string().into()], position: None}), RenderOptions::default(), "### Foo Bar")] -#[case::heading_trailing_hash_escaped(Node::Heading(Heading{depth: 1, values: vec!["foo #".to_string().into()], position: None}), RenderOptions::default(), "# foo \\#")] -#[case::heading_trailing_hash_run_escaped(Node::Heading(Heading{depth: 3, values: vec!["foo ###".to_string().into()], position: None}), RenderOptions::default(), "### foo \\###")] -#[case::heading_trailing_hash_after_multibyte_char(Node::Heading(Heading{depth: 1, values: vec!["foo あ#".to_string().into()], position: None}), RenderOptions::default(), "# foo あ\\#")] -#[case::heading_no_trailing_hash_unaffected(Node::Heading(Heading{depth: 1, values: vec!["foo bar".to_string().into()], position: None}), RenderOptions::default(), "# foo bar")] +#[case::heading(Node::Heading(Heading{depth: HeadingDepth::H1, values: vec!["test".to_string().into()], position: None}), RenderOptions::default(), "# test")] +#[case::heading(Node::Heading(Heading{depth: HeadingDepth::H3, values: vec!["test".to_string().into()], position: None}), RenderOptions::default(), "### test")] +#[case::heading_multiline_h1_stays_setext(Node::Heading(Heading{depth: HeadingDepth::H1, values: vec!["Foo\nBar".to_string().into()], position: None}), RenderOptions::default(), "Foo\nBar\n===")] +#[case::heading_multiline_h2_stays_setext(Node::Heading(Heading{depth: HeadingDepth::H2, values: vec!["Foo\nBar".to_string().into()], position: None}), RenderOptions::default(), "Foo\nBar\n---")] +#[case::heading_multiline_h3_joins_with_space(Node::Heading(Heading{depth: HeadingDepth::H3, values: vec!["Foo\nBar".to_string().into()], position: None}), RenderOptions::default(), "### Foo Bar")] +#[case::heading_trailing_hash_escaped(Node::Heading(Heading{depth: HeadingDepth::H1, values: vec!["foo #".to_string().into()], position: None}), RenderOptions::default(), "# foo \\#")] +#[case::heading_trailing_hash_run_escaped(Node::Heading(Heading{depth: HeadingDepth::H3, values: vec!["foo ###".to_string().into()], position: None}), RenderOptions::default(), "### foo \\###")] +#[case::heading_trailing_hash_after_multibyte_char(Node::Heading(Heading{depth: HeadingDepth::H1, values: vec!["foo あ#".to_string().into()], position: None}), RenderOptions::default(), "# foo あ\\#")] +#[case::heading_no_trailing_hash_unaffected(Node::Heading(Heading{depth: HeadingDepth::H1, values: vec!["foo bar".to_string().into()], position: None}), RenderOptions::default(), "# foo bar")] #[case::html(Node::Html(Html{value: "<div>test</div>".to_string(), position: None}), RenderOptions::default(), "<div>test</div>")] #[case::image(Node::Image(Image{alt: attr_keys::ALT.to_string(), url: attr_keys::URL.to_string(), title: None, position: None}), RenderOptions::default(), "![alt](url)")] #[case::image(Node::Image(Image{alt: attr_keys::ALT.to_string(), url: "url with space".to_string(), title: Some(attr_keys::TITLE.to_string()), position: None}), RenderOptions::default(), "![alt](<url with space> \"title\")")] @@ -1154,13 +1154,12 @@ fn test_node_partial_ord() { #[case(Node::Break(Break{position: None}), "break")] #[case(Node::Definition(Definition{ident: "".to_string(), url: Url::new("".to_string()), title: None, label: None, position: None}), "definition")] #[case(Node::Delete(Delete{values: Vec::new(), position: None}), "delete")] -#[case(Node::Heading(Heading{depth: 1, values: Vec::new(), position: None}), "h1")] -#[case(Node::Heading(Heading{depth: 2, values: Vec::new(), position: None}), "h2")] -#[case(Node::Heading(Heading{depth: 3, values: Vec::new(), position: None}), "h3")] -#[case(Node::Heading(Heading{depth: 4, values: Vec::new(), position: None}), "h4")] -#[case(Node::Heading(Heading{depth: 5, values: Vec::new(), position: None}), "h5")] -#[case(Node::Heading(Heading{depth: 6, values: Vec::new(), position: None}), "h6")] -#[case(Node::Heading(Heading{depth: 7, values: Vec::new(), position: None}), "h")] +#[case(Node::Heading(Heading{depth: HeadingDepth::H1, values: Vec::new(), position: None}), "h1")] +#[case(Node::Heading(Heading{depth: HeadingDepth::H2, values: Vec::new(), position: None}), "h2")] +#[case(Node::Heading(Heading{depth: HeadingDepth::H3, values: Vec::new(), position: None}), "h3")] +#[case(Node::Heading(Heading{depth: HeadingDepth::H4, values: Vec::new(), position: None}), "h4")] +#[case(Node::Heading(Heading{depth: HeadingDepth::H5, values: Vec::new(), position: None}), "h5")] +#[case(Node::Heading(Heading{depth: HeadingDepth::H6, values: Vec::new(), position: None}), "h6")] #[case(Node::Emphasis(Emphasis{values: Vec::new(), position: None}), "emphasis")] #[case(Node::Footnote(Footnote{ident: "".to_string(), values: Vec::new(), position: None}), "footnote")] #[case(Node::FootnoteRef(FootnoteRef{ident: "".to_string(), label: None, position: None}), "footnoteref")] @@ -1196,7 +1195,7 @@ fn test_name(#[case] node: Node, #[case] expected: &str) { #[case(Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: vec![Node::Text(Text{value: "test".to_string(), position: None})], position: None}), "test")] #[case(Node::Blockquote(Blockquote{values: vec![Node::Text(Text{value: "test".to_string(), position: None})], position: None}), "test")] #[case(Node::Delete(Delete{values: vec![Node::Text(Text{value: "test".to_string(), position: None})], position: None}), "test")] -#[case(Node::Heading(Heading{depth: 1, values: vec![Node::Text(Text{value: "test".to_string(), position: None})], position: None}), "test")] +#[case(Node::Heading(Heading{depth: HeadingDepth::H1, values: vec![Node::Text(Text{value: "test".to_string(), position: None})], position: None}), "test")] #[case(Node::Emphasis(Emphasis{values: vec![Node::Text(Text{value: "test".to_string(), position: None})], position: None}), "test")] #[case(Node::Footnote(Footnote{ident: "test".to_string(), values: vec![Node::Text(Text{value: "test".to_string(), position: None})], position: None}), "test")] #[case(Node::FootnoteRef(FootnoteRef{ident: "test".to_string(), label: None, position: None}), "test")] @@ -1233,7 +1232,7 @@ fn test_value(#[case] node: Node, #[case] expected: &str) { #[case(Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: Vec::new(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] #[case(Node::Blockquote(Blockquote{values: Vec::new(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] #[case(Node::Delete(Delete{values: Vec::new(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] -#[case(Node::Heading(Heading{depth: 1, values: Vec::new(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] +#[case(Node::Heading(Heading{depth: HeadingDepth::H1, values: Vec::new(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] #[case(Node::Emphasis(Emphasis{values: Vec::new(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] #[case(Node::Footnote(Footnote{ident: "".to_string(), values: Vec::new(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] #[case(Node::FootnoteRef(FootnoteRef{ident: "".to_string(), label: None, position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}), Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}))] @@ -1309,7 +1308,7 @@ fn test_position(#[case] node: Node, #[case] expected: Option<Position>) { #[case(Node::Strong(Strong{values: vec![ Node::Text(Text{value: "first".to_string(), position: None}) ], position: None}), 0, Some(Node::Text(Text{value: "first".to_string(), position: None})))] -#[case(Node::Heading(Heading{depth: 1, values: vec![ +#[case(Node::Heading(Heading{depth: HeadingDepth::H1, values: vec![ Node::Text(Text{value: "first".to_string(), position: None}), Node::Text(Text{value: "second".to_string(), position: None}) ], position: None}), 0, Some(Node::Text(Text{value: "first".to_string(), position: None})))] @@ -1336,7 +1335,7 @@ fn test_find_at_index(#[case] node: Node, #[case] index: usize, #[case] expected Node::Fragment(Fragment{values: vec!["test".to_string().into()]}))] #[case(Node::Delete(Delete{values: vec!["test".to_string().into()], position: None}), Node::Fragment(Fragment{values: vec!["test".to_string().into()]}))] -#[case(Node::Heading(Heading{depth: 1, values: vec!["test".to_string().into()], position: None}), +#[case(Node::Heading(Heading{depth: HeadingDepth::H1, values: vec!["test".to_string().into()], position: None}), Node::Fragment(Fragment{values: vec!["test".to_string().into()]}))] #[case(Node::Emphasis(Emphasis{values: vec!["test".to_string().into()], position: None}), Node::Fragment(Fragment{values: vec!["test".to_string().into()]}))] @@ -1379,8 +1378,8 @@ fn test_to_fragment(#[case] node: Node, #[case] expected: Node) { #[case::blockquote_mixed(Node::Blockquote(Blockquote{values: vec![Node::Empty, Node::Text(Text{value: "kept".to_string(), position: None})], position: None}), "kept")] #[case::delete_all_empty(Node::Delete(Delete{values: vec![Node::Empty, Node::Empty], position: None}), "")] #[case::delete_mixed(Node::Delete(Delete{values: vec![Node::Text(Text{value: "kept".to_string(), position: None}), Node::Empty], position: None}), "kept")] -#[case::heading_all_empty(Node::Heading(Heading{depth: 1, values: vec![Node::Empty, Node::Empty], position: None}), "")] -#[case::heading_mixed(Node::Heading(Heading{depth: 1, values: vec![Node::Empty, Node::Text(Text{value: "kept".to_string(), position: None})], position: None}), "kept")] +#[case::heading_all_empty(Node::Heading(Heading{depth: HeadingDepth::H1, values: vec![Node::Empty, Node::Empty], position: None}), "")] +#[case::heading_mixed(Node::Heading(Heading{depth: HeadingDepth::H1, values: vec![Node::Empty, Node::Text(Text{value: "kept".to_string(), position: None})], position: None}), "kept")] #[case::emphasis_all_empty(Node::Emphasis(Emphasis{values: vec![Node::Empty, Node::Empty], position: None}), "")] #[case::emphasis_mixed(Node::Emphasis(Emphasis{values: vec![Node::Empty, Node::Text(Text{value: "kept".to_string(), position: None})], position: None}), "kept")] #[case::list_all_empty(Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: vec![Node::Empty, Node::Empty], position: None}), "")] @@ -1456,13 +1455,13 @@ fn test_to_fragment_then_render_skips_non_matching(#[case] node: Node, #[case] e ], position: None}) )] #[case( - &mut Node::Heading(Heading{depth: 1, values: vec![ + &mut Node::Heading(Heading{depth: HeadingDepth::H1, values: vec![ Node::Text(Text{value: "old".to_string(), position: None}) ], position: None}), Node::Fragment(Fragment{values: vec![ Node::Text(Text{value: "new".to_string(), position: None}) ]}), - Node::Heading(Heading{depth: 1, values: vec![ + Node::Heading(Heading{depth: HeadingDepth::H1, values: vec![ Node::Text(Text{value: "new".to_string(), position: None}) ], position: None}) )] @@ -1675,10 +1674,10 @@ fn test_apply_fragment(#[case] node: &mut Node, #[case] fragment: Node, #[case] Node::Blockquote(Blockquote{values: vec![ Node::Text(Text{value: "test".to_string(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}) ], position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}))] -#[case(Node::Heading(Heading{depth: 1, values: vec![ +#[case(Node::Heading(Heading{depth: HeadingDepth::H1, values: vec![ Node::Text(Text{value: "test".to_string(), position: None})], position: None}), Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}}, - Node::Heading(Heading{depth: 1, values: vec![ + Node::Heading(Heading{depth: HeadingDepth::H1, values: vec![ Node::Text(Text{value: "test".to_string(), position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}) ], position: Some(Position{start: Point{line: 1, column: 1}, end: Point{line: 1, column: 5}})}))] #[case(Node::Strong(Strong{values: vec![ @@ -1757,7 +1756,7 @@ fn test_strip_positions_recurses_into_children() { #[test] fn test_clear_text_position_at_leaf_only() { let mut node = Node::Heading(Heading { - depth: 1, + depth: HeadingDepth::H1, values: vec![Node::Text(Text { value: "title".to_string(), position: some_position(), @@ -1883,7 +1882,7 @@ fn test_is_empty_fragment(#[case] node: Node, #[case] expected: bool) { #[case::definition(Node::Definition(Definition{ident: "id".to_string(), url: Url::new(attr_keys::URL.to_string()), title: Some(Title::new(attr_keys::TITLE.to_string())), label: Some(attr_keys::LABEL.to_string()), position: None}), attr_keys::URL, Some(AttrValue::String(attr_keys::URL.to_string())))] #[case::definition(Node::Definition(Definition{ident: "id".to_string(), url: Url::new(attr_keys::URL.to_string()), title: Some(Title::new(attr_keys::TITLE.to_string())), label: Some(attr_keys::LABEL.to_string()), position: None}), attr_keys::TITLE, Some(AttrValue::String(attr_keys::TITLE.to_string())))] #[case::definition(Node::Definition(Definition{ident: "id".to_string(), url: Url::new(attr_keys::URL.to_string()), title: Some(Title::new(attr_keys::TITLE.to_string())), label: Some(attr_keys::LABEL.to_string()), position: None}), attr_keys::LABEL, Some(AttrValue::String(attr_keys::LABEL.to_string())))] -#[case::heading(Node::Heading(Heading{depth: 3, values: Vec::new(), position: None}), "depth", Some(AttrValue::Integer(3)))] +#[case::heading(Node::Heading(Heading{depth: HeadingDepth::H3, values: Vec::new(), position: None}), "depth", Some(AttrValue::Integer(3)))] #[case::list(Node::List(List{ marker: None,start: None, spread: false, index: 2, level: 1, checked: Some(true), ordered: true, values: Vec::new(), position: None}), "index", Some(AttrValue::Integer(2)))] #[case::list(Node::List(List{ marker: None,start: None, spread: false, index: 2, level: 1, checked: Some(true), ordered: true, values: Vec::new(), position: None}), "level", Some(AttrValue::Integer(1)))] #[case::list(Node::List(List{ marker: None,start: None, spread: false, index: 2, level: 1, checked: Some(true), ordered: true, values: Vec::new(), position: None}), "ordered", Some(AttrValue::Boolean(true)))] @@ -1899,9 +1898,9 @@ fn test_is_empty_fragment(#[case] node: Node, #[case] expected: bool) { #[case::break_(Node::Break(Break{position: None}), attr_keys::VALUE, None)] #[case::horizontal_rule(Node::HorizontalRule(HorizontalRule{ marker: None,position: None}), attr_keys::VALUE, None)] #[case::fragment(Node::Fragment(Fragment{values: Vec::new()}), attr_keys::VALUE, Some(AttrValue::String("".to_string())))] -#[case::heading(Node::Heading(Heading{depth: 1, values: vec![Node::Text(Text{value: "heading text".to_string(), position: None})], position: None}), attr_keys::VALUE, Some(AttrValue::String("heading text".to_string())))] -#[case::heading(Node::Heading(Heading{depth: 2, values: vec![], position: None}), attr_keys::VALUE, Some(AttrValue::String("".to_string())))] -#[case::heading(Node::Heading(Heading{depth: 3, values: vec![ +#[case::heading(Node::Heading(Heading{depth: HeadingDepth::H1, values: vec![Node::Text(Text{value: "heading text".to_string(), position: None})], position: None}), attr_keys::VALUE, Some(AttrValue::String("heading text".to_string())))] +#[case::heading(Node::Heading(Heading{depth: HeadingDepth::H2, values: vec![], position: None}), attr_keys::VALUE, Some(AttrValue::String("".to_string())))] +#[case::heading(Node::Heading(Heading{depth: HeadingDepth::H3, values: vec![ Node::Text(Text{value: "first".to_string(), position: None}), Node::Text(Text{value: "second".to_string(), position: None}), ], position: None}), attr_keys::VALUE, Some(AttrValue::String("firstsecond".to_string())))] @@ -1962,7 +1961,7 @@ fn test_is_empty_fragment(#[case] node: Node, #[case] expected: bool) { #[case::empty(Node::Empty, attr_keys::VALUE, None)] #[case::heading( Node::Heading(Heading { - depth: 1, + depth: HeadingDepth::H1, values: vec![ Node::Text(Text { value: "child1".to_string(), @@ -2101,7 +2100,7 @@ fn test_attr(#[case] node: Node, #[case] attr: &str, #[case] expected: Option<At #[rstest] #[case::heading_with_position( Node::Heading(Heading { - depth: 1, + depth: HeadingDepth::H1, values: vec![], position: Some(Position { start: Point { line: 3, column: 1 }, @@ -2127,7 +2126,7 @@ fn test_attr(#[case] node: Node, #[case] attr: &str, #[case] expected: Option<At )] #[case::synthetic_node_has_no_line( Node::Heading(Heading { - depth: 1, + depth: HeadingDepth::H1, values: vec![], position: None, }), @@ -2136,7 +2135,7 @@ fn test_attr(#[case] node: Node, #[case] attr: &str, #[case] expected: Option<At )] #[case::synthetic_node_has_no_end_line( Node::Heading(Heading { - depth: 1, + depth: HeadingDepth::H1, values: vec![], position: None, }), @@ -2149,9 +2148,9 @@ fn test_line_attr(#[case] node: Node, #[case] attr: &str, #[case] expected: Opti #[rstest] #[case::heading( - Node::Heading(Heading{depth: 1, values: vec![], position: None}), + Node::Heading(Heading{depth: HeadingDepth::H1, values: vec![], position: None}), vec![Node::Text(Text{value: "child".to_string(), position: None})], - Node::Heading(Heading{depth: 1, values: vec![Node::Text(Text{value: "child".to_string(), position: None})], position: None}) + Node::Heading(Heading{depth: HeadingDepth::H1, values: vec![Node::Text(Text{value: "child".to_string(), position: None})], position: None}) )] #[case::list( Node::List(List{ marker: None,start: None, spread: false, index: 0, level: 0, checked: None, ordered: false, values: vec![], position: None}), @@ -2275,10 +2274,10 @@ fn test_set_children(#[case] mut node: Node, #[case] children: Vec<Node>, #[case Node::Image(Image{alt: attr_keys::ALT.to_string(), url: attr_keys::URL.to_string(), title: Some("new_title".to_string()), position: None}) )] #[case( - Node::Heading(Heading{depth: 2, values: vec![], position: None}), + Node::Heading(Heading{depth: HeadingDepth::H2, values: vec![], position: None}), "depth", "3", - Node::Heading(Heading{depth: 3, values: vec![], position: None}) + Node::Heading(Heading{depth: HeadingDepth::H3, values: vec![], position: None}) )] #[case( Node::List(List{ marker: None,start: None, spread: false, index: 1, level: 2, checked: Some(true), ordered: false, values: vec![], position: None}), @@ -3030,3 +3029,68 @@ fn test_list_own_prefix_width( ) { assert_eq!(list_own_prefix_width(ordered, index, start), expected); } + +#[rstest] +#[case(0, None)] +#[case(1, Some(HeadingDepth::H1))] +#[case(6, Some(HeadingDepth::H6))] +#[case(7, None)] +fn heading_depth_new(#[case] depth: u8, #[case] expected: Option<HeadingDepth>) { + assert_eq!(HeadingDepth::new(depth), expected); + assert_eq!(HeadingDepth::try_from(depth).ok(), expected); +} + +#[rstest] +#[case(i64::MIN, HeadingDepth::H1)] +#[case(0, HeadingDepth::H1)] +#[case(3, HeadingDepth::H3)] +#[case(257, HeadingDepth::H6)] +fn heading_depth_saturating(#[case] depth: i64, #[case] expected: HeadingDepth) { + assert_eq!(HeadingDepth::saturating(depth), expected); +} + +#[rstest] +#[case(HeadingDepth::H2, 3, Some(HeadingDepth::H5))] +#[case(HeadingDepth::H6, 1, None)] +#[case(HeadingDepth::H6, u8::MAX, None)] +fn heading_depth_checked_add(#[case] depth: HeadingDepth, #[case] levels: u8, #[case] expected: Option<HeadingDepth>) { + assert_eq!(depth.checked_add(levels), expected); +} + +#[rstest] +#[case(HeadingDepth::H4, 2, HeadingDepth::H2)] +#[case(HeadingDepth::H2, u8::MAX, HeadingDepth::H1)] +fn heading_depth_saturating_sub(#[case] depth: HeadingDepth, #[case] levels: u8, #[case] expected: HeadingDepth) { + assert_eq!(depth.saturating_sub(levels), expected); +} + +#[rstest] +#[case::integer(AttrValue::Integer(3), HeadingDepth::H3)] +#[case::string(AttrValue::String("4".to_string()), HeadingDepth::H4)] +#[case::too_deep(AttrValue::Integer(7), HeadingDepth::H2)] +#[case::wraps_as_byte(AttrValue::Integer(257), HeadingDepth::H2)] +#[case::negative(AttrValue::Integer(-1), HeadingDepth::H2)] +#[case::zero_string(AttrValue::String("0".to_string()), HeadingDepth::H2)] +fn set_attr_keeps_heading_depth_in_range(#[case] value: AttrValue, #[case] expected: HeadingDepth) { + let mut node = Node::Heading(Heading { + depth: HeadingDepth::H2, + values: vec!["a".to_string().into()], + position: None, + }); + node.set_attr(attr_keys::DEPTH, value); + assert!(matches!(node, Node::Heading(Heading { depth, .. }) if depth == expected)); +} + +#[cfg(feature = "json")] +#[test] +fn heading_depth_is_a_number_in_json() { + let heading = Heading { + depth: HeadingDepth::H3, + values: Vec::new(), + position: None, + }; + let json = serde_json::to_string(&heading).unwrap(); + assert!(json.contains("\"depth\":3"), "{json}"); + assert_eq!(serde_json::from_str::<Heading>(&json).unwrap(), heading); + assert!(serde_json::from_str::<Heading>(&json.replace("\"depth\":3", "\"depth\":7")).is_err()); +} diff --git a/crates/mq-markdown/src/node_kind.rs b/crates/mq-markdown/src/node_kind.rs index b63d119d2..d8d56369b 100644 --- a/crates/mq-markdown/src/node_kind.rs +++ b/crates/mq-markdown/src/node_kind.rs @@ -255,8 +255,8 @@ impl Node { Node::Embed(_) => NodeKind::Embed, Node::Definition(_) => NodeKind::Definition, Node::Delete(_) => NodeKind::Delete, - Node::Heading(Heading { depth, .. }) => match depth { - 0 | 1 => NodeKind::H1, + Node::Heading(Heading { depth, .. }) => match depth.get() { + 1 => NodeKind::H1, 2 => NodeKind::H2, 3 => NodeKind::H3, 4 => NodeKind::H4, diff --git a/crates/mq-markdown/src/parser/block.rs b/crates/mq-markdown/src/parser/block.rs index 81fe0ee6b..93cda599d 100644 --- a/crates/mq-markdown/src/parser/block.rs +++ b/crates/mq-markdown/src/parser/block.rs @@ -15,7 +15,7 @@ use super::table; mod rule; use super::tree::{Block, FootnoteBlock, InlineBlock, InlineKind, InlineSource, Item, ListBlock, QuoteBlock}; -use crate::node::{Html, ListMarker, Node, Point, Position, Toml, Yaml}; +use crate::node::{HeadingDepth, Html, ListMarker, Node, Point, Position, Toml, Yaml}; use rule::{BlockRule, Cx, Rules, fallback}; /// Lines indented by this many columns or more are code, not other blocks. @@ -134,7 +134,7 @@ fn parse_blocks(lines: &[Line<'_>], depth: usize, interrupting: Interrupt) -> Ve pub(super) enum LineStart<'a> { Fence(Fence<'a>), /// An ATX heading of this depth. - Atx(u8), + Atx(HeadingDepth), ThematicBreak, Blockquote, Item(ItemMarker), @@ -749,14 +749,15 @@ fn remove_task_marker(children: &mut Vec<Block>) -> bool { } /// Returns the heading depth when `rest` (indent already removed) is an ATX heading line. -fn atx_depth(rest: &str) -> Option<u8> { +fn atx_depth(rest: &str) -> Option<HeadingDepth> { let hashes = rest.bytes().take_while(|&b| b == b'#').count(); let delimited = matches!(rest[hashes..].chars().next(), None | Some(' ' | '\t')); - ((1..=6).contains(&hashes) && delimited).then_some(hashes as u8) + let depth = u8::try_from(hashes).ok().and_then(HeadingDepth::new)?; + delimited.then_some(depth) } -fn atx_heading(line: &Line<'_>, indent: usize, depth: u8) -> Block { - let rest = &line.text[indent + depth as usize..]; +fn atx_heading(line: &Line<'_>, indent: usize, depth: HeadingDepth) -> Block { + let rest = &line.text[indent + usize::from(depth.get())..]; let content = rest.trim_matches([' ', '\t']); // A closing sequence must be preceded by whitespace, unless it is the whole content. let content = { @@ -816,7 +817,7 @@ fn is_thematic_break(line: &Line<'_>, indent: usize) -> bool { } /// Returns the setext heading depth when `line` is a setext underline. -fn setext_depth(line: &Line<'_>) -> Option<u8> { +fn setext_depth(line: &Line<'_>) -> Option<HeadingDepth> { let Indent { columns, bytes: indent } = line.indent(); let content = line.text[indent..].trim_end_matches([' ', '\t']); let marker = content.chars().next()?; @@ -824,8 +825,8 @@ fn setext_depth(line: &Line<'_>) -> Option<u8> { return None; } match marker { - '=' => Some(1), - '-' => Some(2), + '=' => Some(HeadingDepth::H1), + '-' => Some(HeadingDepth::H2), _ => None, } } diff --git a/crates/mq-markdown/src/parser/tree.rs b/crates/mq-markdown/src/parser/tree.rs index ef0050aab..356c5b9b8 100644 --- a/crates/mq-markdown/src/parser/tree.rs +++ b/crates/mq-markdown/src/parser/tree.rs @@ -7,7 +7,7 @@ use super::error::MdxError; use super::line::visual_column; use super::mdx::TagKind; use super::scan::eol_len; -use crate::node::{ListMarker, MdxAttributeContent, Node, Point, Position, TableAlignKind}; +use crate::node::{HeadingDepth, ListMarker, MdxAttributeContent, Node, Point, Position, TableAlignKind}; /// Where a line of an [`InlineSource`] starts: its offset in the text and its position in the document. #[derive(Clone)] @@ -167,7 +167,7 @@ pub(super) struct InlineBlock { #[derive(Clone)] pub(super) enum InlineKind { Paragraph, - Heading { depth: u8, position: Position }, + Heading { depth: HeadingDepth, position: Position }, } pub(super) struct QuoteBlock { diff --git a/crates/mq-run/src/output/table.rs b/crates/mq-run/src/output/table.rs index c0b1950d9..bc804457c 100644 --- a/crates/mq-run/src/output/table.rs +++ b/crates/mq-run/src/output/table.rs @@ -410,7 +410,7 @@ mod tests { #[test] fn test_table_all_markdown_nodes() { let node = mq_markdown::Node::Heading(mq_markdown::Heading { - depth: 1, + depth: mq_markdown::HeadingDepth::H1, values: vec![mq_markdown::Node::Text(mq_markdown::Text { value: "Title".to_string(), position: None, From d938af8aa2c3f2f28cb89d4375cce49803ecd185 Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Thu, 8 Oct 2026 13:09:48 +0900 Subject: [PATCH 48/88] =?UTF-8?q?=E2=9C=85=20test(markdown):=20drop=20a=20?= =?UTF-8?q?duplicate=20parser=20snapshot=20case=20and=20a=20repeated=20nam?= =?UTF-8?q?e?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit thematic_over_list had the same input as thematic_dash_spaced, and two cases were named two_paragraphs. --- crates/mq-markdown/src/parser.rs | 3 +-- crates/mq-markdown/src/parser/snapshots.txt | 5 +---- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/crates/mq-markdown/src/parser.rs b/crates/mq-markdown/src/parser.rs index f46985fb8..b56844538 100644 --- a/crates/mq-markdown/src/parser.rs +++ b/crates/mq-markdown/src/parser.rs @@ -231,7 +231,7 @@ mod tests { ("nested", "- a\n - b\n c\n"), ("nested_ordered", "1. a\n 1. b\n 2. c\n2. d\n"), ("nested_dedent", "- a\n - b\n- c\n"), - ("two_paragraphs", "- a\n\n b\n"), + ("list_item_two_paragraphs", "- a\n\n b\n"), ("two_paragraphs_then_item", "- a\n\n b\n- c\n"), ("empty_item", "-\n"), ("empty_item_content", "-\n foo\n"), @@ -245,7 +245,6 @@ mod tests { ("ordered_two_no_interrupt", "a\n2. b\n"), ("ordered_one_interrupts", "a\n1. b\n"), ("empty_item_no_interrupt", "a\n-\n"), - ("thematic_over_list", "- - -\n"), ("list_in_quote", "> - a\n> - b\n"), ("quote_in_list", "- > a\n > b\n"), ("quote_in_list_level", "- a\n > - b\n"), diff --git a/crates/mq-markdown/src/parser/snapshots.txt b/crates/mq-markdown/src/parser/snapshots.txt index dcc285c72..f53bc3d4a 100644 --- a/crates/mq-markdown/src/parser/snapshots.txt +++ b/crates/mq-markdown/src/parser/snapshots.txt @@ -199,7 +199,7 @@ html: "<ol>\n<li>a\n<ol>\n<li>b</li>\n<li>c</li>\n</ol>\n</li>\n<li>d</li>\n</ol ## nested_dedent "- a\n - b\n- c\n" nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-1:4 }), List(List { values: [Text(Text { value: "b", @2:5-2:6 })], index: 0, level: 1, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @2:5-2:6 }), List(List { values: [Text(Text { value: "c", @3:3-3:4 })], index: 1, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @3:3-3:4 })] html: "<ul>\n<li>a\n<ul>\n<li>b</li>\n</ul>\n</li>\n<li>c</li>\n</ul>\n" -## two_paragraphs "- a\n\n b\n" +## list_item_two_paragraphs "- a\n\n b\n" nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 }), Text(Text { value: "b", @3:3-3:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-3:4 })] html: "<ul>\n<li>\n<p>a</p>\n<p>b</p>\n</li>\n</ul>\n" ## two_paragraphs_then_item "- a\n\n b\n- c\n" @@ -241,9 +241,6 @@ html: "<p>a</p>\n<ol>\n<li>b</li>\n</ol>\n" ## empty_item_no_interrupt "a\n-\n" nodes: [Heading(Heading { depth: 2, values: [Text(Text { value: "a", @1:1-1:2 })], @1:1-2:2 })] html: "<h2>a</h2>\n" -## thematic_over_list "- - -\n" -nodes: [HorizontalRule(HorizontalRule { marker: Some(Dash), @1:1-1:6 })] -html: "<hr />\n" ## list_in_quote "> - a\n> - b\n" nodes: [Blockquote(Blockquote { values: [List(List { values: [Text(Text { value: "a", @1:5-1:6 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:5-1:6 }), List(List { values: [Text(Text { value: "b", @2:5-2:6 })], index: 1, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @2:5-2:6 })], @1:1-2:6 })] html: "<blockquote>\n<ul>\n<li>a</li>\n<li>b</li>\n</ul>\n</blockquote>\n" From 9453f1920a2fb214379facc9ccee0074390737e9 Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Thu, 8 Oct 2026 13:09:54 +0900 Subject: [PATCH 49/88] =?UTF-8?q?=F0=9F=93=9D=20docs(markdown):=20name=20t?= =?UTF-8?q?he=20Unicode=20and=20HTML=20standard=20tables=20as=20the=20sour?= =?UTF-8?q?ces?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit punctuation.rs holds the Unicode general categories P and S (16.0.0) and entities.rs the named character references of the HTML standard. Both tables were checked against those sources and match, so the comments no longer point at markdown-rs. --- crates/mq-markdown/src/parser/inline/entities.rs | 5 +++-- crates/mq-markdown/src/parser/inline/punctuation.rs | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/crates/mq-markdown/src/parser/inline/entities.rs b/crates/mq-markdown/src/parser/inline/entities.rs index 8b6913480..9ec791374 100644 --- a/crates/mq-markdown/src/parser/inline/entities.rs +++ b/crates/mq-markdown/src/parser/inline/entities.rs @@ -1,7 +1,8 @@ //! HTML character reference names and their values. //! -//! Generated from the table of `markdown-rs` (MIT License, Titus Wormer), which follows the -//! [HTML standard](https://html.spec.whatwg.org/multipage/named-characters.html). Sorted by name. +//! The names that end with `;` in the table of the +//! [HTML standard](https://html.spec.whatwg.org/multipage/named-characters.html), without the `;`. +//! Sorted by name. pub(super) static ENTITIES: [(&str, &str); 2125] = [ ("AElig", "Æ"), diff --git a/crates/mq-markdown/src/parser/inline/punctuation.rs b/crates/mq-markdown/src/parser/inline/punctuation.rs index e313f1515..4fdfc8383 100644 --- a/crates/mq-markdown/src/parser/inline/punctuation.rs +++ b/crates/mq-markdown/src/parser/inline/punctuation.rs @@ -1,7 +1,7 @@ //! Unicode punctuation for emphasis flanking rules. //! -//! Generated from the table of `markdown-rs` (MIT License, Titus Wormer): the general categories P -//! and S, as `CommonMark` defines punctuation. Sorted ranges. +//! The Unicode general categories P and S (Unicode 16.0.0), as `CommonMark` defines punctuation. +//! Sorted ranges, ASCII included. const RANGES: [(u32, u32); 349] = [ (0x0021, 0x002F), From b22970fbb335905877388c7fe58530216654a5b5 Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Thu, 8 Oct 2026 13:10:00 +0900 Subject: [PATCH 50/88] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor(markdown):?= =?UTF-8?q?=20restructure=20HTML=20block=20starts=20and=20autolink=20liter?= =?UTF-8?q?al=20ends?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit html_flow reads the start conditions in the order of the CommonMark list and checks a complete tag with a small cursor. literal finds the end of an autolink by cutting at whitespace, < and a closing ] and then trimming trailing punctuation, references and unbalanced ), instead of walking a state machine. Results match the previous code on 200k (html_flow) and 300k (literal) generated inputs, and parse time is the same or lower. --- crates/mq-markdown/src/parser/html_flow.rs | 200 +++++++++++------- .../mq-markdown/src/parser/inline/literal.rs | 162 +++++--------- 2 files changed, 179 insertions(+), 183 deletions(-) diff --git a/crates/mq-markdown/src/parser/html_flow.rs b/crates/mq-markdown/src/parser/html_flow.rs index 51638b689..c15bd4430 100644 --- a/crates/mq-markdown/src/parser/html_flow.rs +++ b/crates/mq-markdown/src/parser/html_flow.rs @@ -1,4 +1,4 @@ -//! HTML blocks: the seven kinds of `CommonMark`, ported from the rules of `markdown-rs`. +//! HTML blocks, with the start and end conditions of the seven kinds that `CommonMark` defines. use super::scan::skip_blanks; @@ -90,111 +90,153 @@ const BLOCK_NAMES: [&str; 62] = [ /// Recognizes the start of an HTML block in `rest`, the line without its indentation. pub(super) fn start(rest: &str) -> Option<Kind> { - let bytes = rest.as_bytes(); - if bytes.first() != Some(&b'<') { - return None; - } - - match bytes.get(1)? { - b'!' => match bytes.get(2)? { - b'-' if bytes.get(3) == Some(&b'-') => Some(Kind::Comment), - b'[' if rest[3..].starts_with("CDATA[") => Some(Kind::Cdata), - byte if byte.is_ascii_alphabetic() => Some(Kind::Declaration), - _ => None, - }, - b'?' => Some(Kind::Instruction), - b'/' => tag(rest, 2, true), - byte if byte.is_ascii_alphabetic() => tag(rest, 1, false), - _ => None, + let after = rest.strip_prefix('<')?; + if after.starts_with("!--") { + Some(Kind::Comment) + } else if after.starts_with("![CDATA[") { + Some(Kind::Cdata) + } else if after.starts_with('?') { + Some(Kind::Instruction) + } else if after + .strip_prefix('!') + .is_some_and(|name| name.starts_with(|c: char| c.is_ascii_alphabetic())) + { + Some(Kind::Declaration) + } else { + match after.strip_prefix('/') { + Some(name) => tag(name, true), + None => tag(after, false), + } } } -/// Recognizes a block tag, a raw tag or a complete tag whose name starts at `from`. -fn tag(rest: &str, from: usize, closing: bool) -> Option<Kind> { - let bytes = rest.as_bytes(); - if !bytes.get(from).is_some_and(u8::is_ascii_alphabetic) { +/// Classifies the tag whose name starts `text`, which is what follows `<` or `</`. +fn tag(text: &str, closing: bool) -> Option<Kind> { + let bytes = text.as_bytes(); + if !bytes.first().is_some_and(u8::is_ascii_alphabetic) { return None; } - let length = bytes[from..] + let name_end = bytes .iter() - .take_while(|b| b.is_ascii_alphanumeric() || **b == b'-') - .count(); - let end = from + length; - let terminator = bytes.get(end).copied(); - if !matches!(terminator, None | Some(b'\t' | b' ' | b'/' | b'>')) { + .position(|b| !(b.is_ascii_alphanumeric() || *b == b'-')) + .unwrap_or(bytes.len()); + let after = &bytes[name_end..]; + let self_closing = after.starts_with(b"/"); + if !matches!(after.first(), None | Some(b'\t' | b' ' | b'/' | b'>')) { return None; } - let slash = terminator == Some(b'/'); - let name = rest[from..end].to_ascii_lowercase(); + let name = text[..name_end].to_ascii_lowercase(); - if let Some(raw) = RAW_NAMES.iter().find(|raw| !slash && !closing && **raw == name) { + if !closing + && !self_closing + && let Some(raw) = RAW_NAMES.iter().find(|raw| **raw == name) + { Some(Kind::Raw(raw)) } else if BLOCK_NAMES.contains(&name.as_str()) { - // A slash has to be the end of a self-closing tag. - (!slash || bytes.get(end + 1) == Some(&b'>')).then_some(Kind::Basic) + // A slash has to be the start of `/>`. + (!self_closing || after.starts_with(b"/>")).then_some(Kind::Basic) } else { - complete_tag(&bytes[end..], closing).then_some(Kind::Complete) + is_complete_tag(after, closing).then_some(Kind::Complete) } } -/// Whether the rest of a tag after its name is complete and alone on the line. -fn complete_tag(bytes: &[u8], closing: bool) -> bool { - let mut index = 0; +/// Whether `rest`, what follows the name of a tag, closes the tag with nothing after it on the line. +fn is_complete_tag(rest: &[u8], closing: bool) -> bool { + let mut cursor = Cursor { bytes: rest, index: 0 }; + cursor.skip_blanks(); + if !closing { + loop { + match cursor.attribute() { + Attribute::Read => cursor.skip_blanks(), + Attribute::Absent => break, + Attribute::Invalid => return false, + } + } + cursor.eat(b'/'); + } + cursor.eat(b'>') && { + cursor.skip_blanks(); + cursor.peek().is_none() + } +} + +enum Attribute { + Read, + /// There is no attribute here. + Absent, + /// An attribute whose value is malformed. + Invalid, +} - if closing { - index = skip_blanks(bytes, index); - return bytes.get(index) == Some(&b'>') && skip_blanks(bytes, index + 1) == bytes.len(); +struct Cursor<'a> { + bytes: &'a [u8], + index: usize, +} + +impl Cursor<'_> { + fn peek(&self) -> Option<u8> { + self.bytes.get(self.index).copied() } - loop { - index = skip_blanks(bytes, index); - match bytes.get(index) { - Some(b'/') => { - return bytes.get(index + 1) == Some(&b'>') && skip_blanks(bytes, index + 2) == bytes.len(); - } - Some(b'0'..=b'9' | b':' | b'A'..=b'Z' | b'_' | b'a'..=b'z') => { - index += 1; - while matches!( - bytes.get(index), - Some(b'-' | b'.' | b'0'..=b'9' | b':' | b'A'..=b'Z' | b'_' | b'a'..=b'z') - ) { - index += 1; - } - let after = skip_blanks(bytes, index); - if bytes.get(after) == Some(&b'=') { - let Some(end) = attribute_value(bytes, skip_blanks(bytes, after + 1)) else { - return false; - }; - index = end; - } - } - Some(b'>') => return skip_blanks(bytes, index + 1) == bytes.len(), - _ => return false, + fn skip_blanks(&mut self) { + self.index = skip_blanks(self.bytes, self.index); + } + + fn eat(&mut self, byte: u8) -> bool { + let found = self.peek() == Some(byte); + self.index += usize::from(found); + found + } + + fn skip_while(&mut self, accept: impl Fn(u8) -> bool) { + while self.peek().is_some_and(&accept) { + self.index += 1; } } -} -/// The offset after the attribute value at `index`. -fn attribute_value(bytes: &[u8], index: usize) -> Option<usize> { - match bytes.get(index)? { - b'<' | b'=' | b'>' | b'`' => None, - quote @ (b'"' | b'\'') => { - let close = bytes[index + 1..].iter().position(|b| b == quote)?; - let end = index + 1 + close + 1; - matches!(bytes.get(end), Some(b'\t' | b' ' | b'/' | b'>')).then_some(end) + /// Reads an attribute name and its optional value. + fn attribute(&mut self) -> Attribute { + if !self + .peek() + .is_some_and(|b| b.is_ascii_alphanumeric() || b == b':' || b == b'_') + { + return Attribute::Absent; } - _ => { - let length = bytes[index..] - .iter() - .take_while(|b| !matches!(b, b'\t' | b' ' | b'"' | b'\'' | b'/' | b'<' | b'=' | b'>' | b'`')) - .count(); - Some(index + length) + self.skip_while(|b| b.is_ascii_alphanumeric() || matches!(b, b'-' | b'.' | b':' | b'_')); + // Blanks before an `=` belong to the value, otherwise they are left to the caller. + let name_end = self.index; + self.skip_blanks(); + if self.eat(b'=') { + self.skip_blanks(); + return if self.value() { + Attribute::Read + } else { + Attribute::Invalid + }; + } + self.index = name_end; + Attribute::Read + } + + fn value(&mut self) -> bool { + match self.peek() { + None | Some(b'<' | b'=' | b'>' | b'`') => false, + Some(quote @ (b'"' | b'\'')) => { + self.index += 1; + self.skip_while(|b| b != quote); + // The closing quote has to be followed by blanks, `/` or `>`. + self.eat(quote) && matches!(self.peek(), Some(b'\t' | b' ' | b'/' | b'>')) + } + Some(_) => { + self.skip_while(|b| !matches!(b, b'\t' | b' ' | b'"' | b'\'' | b'/' | b'<' | b'=' | b'>' | b'`')); + true + } } } } /// Where in the first line the search for the end of a block starts, so that the opening marker is -/// only reused where `markdown-rs` reuses it (`<!-->` and `<?>` are complete). +/// only reused where it can close the block (`<!-->` and `<?>` are complete). pub(super) fn first_line_offset(kind: Kind) -> usize { match kind { Kind::Comment => 2, diff --git a/crates/mq-markdown/src/parser/inline/literal.rs b/crates/mq-markdown/src/parser/inline/literal.rs index 831ab7c1c..f30102293 100644 --- a/crates/mq-markdown/src/parser/inline/literal.rs +++ b/crates/mq-markdown/src/parser/inline/literal.rs @@ -1,129 +1,82 @@ //! GFM autolink literals: `www.example.com`, `https://example.com` and `user@example.com`. //! -//! Ported from the state machines of `markdown-rs` (MIT License, Titus Wormer) so that the same -//! text is linked, including how trailing punctuation and unbalanced parentheses are excluded. +//! A link runs to the first whitespace or `<`, without the punctuation, character references and +//! unbalanced `)` at its end. use super::punctuation::is_punctuation; use super::{Item, Scanner, Value}; use crate::node::{Link, Node, Text, Url}; use crate::parser::scan::Span; -#[derive(PartialEq)] -enum Kind { - Whitespace, - Punctuation, - Other, +/// Whether `char` is a symbol of the emoji blocks, which a domain may contain. +fn is_emoji(char: char) -> bool { + matches!(char, '\u{2600}'..='\u{27BF}' | '\u{1F300}'..='\u{1FAFF}') } -/// Classifies the character at byte `index`; the end of the text counts as whitespace. -fn kind_at(src: &str, index: usize) -> Kind { - let Some(char) = src.get(index..).and_then(|rest| rest.chars().next()) else { - return Kind::Whitespace; - }; - if char.is_whitespace() { - Kind::Whitespace - } else if is_punctuation(char) { - Kind::Punctuation - } else { - Kind::Other - } +/// Whether `char` can be in a domain besides the separators `.`, `_` and `-`. +fn is_domain_char(char: char) -> bool { + is_emoji(char) || !(char.is_whitespace() || is_punctuation(char)) } -/// Whether the character at byte `index` is a symbol of the emoji blocks, which a domain may contain. -fn is_emoji(src: &str, index: usize) -> bool { - src.get(index..) - .and_then(|rest| rest.chars().next()) - .is_some_and(|char| matches!(char, '\u{2600}'..='\u{27BF}' | '\u{1F300}'..='\u{1FAFF}')) +/// Where a `]` ends the link: at the end of the text, before whitespace, or before a `(` or `[` +/// that continues a Markdown link. +fn closes_label(bytes: &[u8], index: usize) -> bool { + bytes[index] == b']' && matches!(bytes.get(index + 1), None | Some(b'\t' | b'\n' | b' ' | b'(' | b'[')) } -/// Whether the text from `index` is only trailing punctuation up to whitespace, the end or `<`. -fn trail(src: &str, mut index: usize) -> bool { +/// The end of the autolink whose domain starts at `start`, with the trailing punctuation left out. +fn autolink_end(src: &str, start: usize) -> usize { let bytes = src.as_bytes(); - loop { - match bytes.get(index) { - Some(b'!' | b'"' | b'\'' | b')' | b'*' | b',' | b'.' | b':' | b';' | b'?' | b'_' | b'~') => index += 1, - Some(b'&') => { - index += 1; - let letters = bytes[index..].iter().take_while(|b| b.is_ascii_alphabetic()).count(); - if letters == 0 || bytes.get(index + letters) != Some(&b';') { - return false; - } - index += letters + 1; - } - Some(b'<') => return true, - Some(b']') => { - index += 1; - if matches!(bytes.get(index), None | Some(b'\t' | b'\n' | b' ' | b'(' | b'[')) { - return true; - } - } - _ => return kind_at(src, index) == Kind::Whitespace, + let mut end = start; + for (offset, char) in src[start..].char_indices() { + if char.is_whitespace() || char == '<' || closes_label(bytes, start + offset) { + break; } + end = start + offset + char.len_utf8(); } -} - -/// Matches the domain that starts at `index`, returning the offset after it. -fn domain(src: &str, mut index: usize) -> Option<usize> { - let bytes = src.as_bytes(); - let (mut seen, mut marker, mut marker_before) = (false, 0u8, 0u8); - loop { - match bytes.get(index) { - Some(b'.' | b'_') => { - if trail(src, index) { - break; - } - if bytes[index] == b'_' { - marker = b'_'; + // The trailing punctuation starts where only punctuation is left up to the end. + let mut trailing = end; + while trailing > start { + match bytes[trailing - 1] { + b'!' | b'"' | b'\'' | b')' | b'*' | b',' | b'.' | b':' | b'?' | b']' | b'_' | b'~' => trailing -= 1, + b';' => { + // A character reference such as `&amp;` goes as a whole. + let letters = bytes[start..trailing - 1] + .iter() + .rev() + .take_while(|b| b.is_ascii_alphabetic()) + .count(); + let reference = trailing - 1 - letters; + trailing = if letters > 0 && reference > start && bytes[reference - 1] == b'&' { + reference - 1 } else { - marker_before = marker; - marker = 0; - } - index += 1; - } - Some(b'-' | 0x80..=0xBF) => index += 1, - _ if kind_at(src, index) == Kind::Other || is_emoji(src, index) => { - seen = true; - index += 1; + trailing - 1 + }; } _ => break, } } - // Underscores are not allowed in the last two segments. - (marker_before != b'_' && marker != b'_' && seen).then_some(index) + // A `)` that closes a `(` of the link is part of it. + let open = bytes[start..trailing].iter().filter(|b| **b == b'(').count(); + let mut closed = bytes[start..trailing].iter().filter(|b| **b == b')').count(); + while trailing < end && bytes[trailing] == b')' && closed < open { + closed += 1; + trailing += 1; + } + trailing } -/// Matches the path that starts at `index`, returning the offset after it. -fn path(src: &str, mut index: usize) -> usize { - let bytes = src.as_bytes(); - let (mut open, mut closed) = (0usize, 0usize); - - loop { - match bytes.get(index) { - None => return index, - Some(0x80..=0xBF) => index += 1, - Some(b'(') => { - open += 1; - index += 1; - } - Some( - punctuation @ (b'!' | b'"' | b'&' | b'\'' | b')' | b'*' | b',' | b'.' | b':' | b';' | b'<' | b'?' - | b']' | b'_' | b'~'), - ) => { - let unbalanced = *punctuation == b')' && closed < open; - if trail(src, index) && !unbalanced { - return index; - } - if *punctuation == b')' { - closed += 1; - } - index += 1; - } - _ if kind_at(src, index) == Kind::Whitespace => return index, - _ => index += 1, - } - } +/// Whether the domain at the start of `text` is valid: it has a character besides the separators, +/// and no `_` in its last two segments. +fn has_valid_domain(text: &str) -> bool { + let length = text + .find(|char: char| !(matches!(char, '.' | '_' | '-') || is_domain_char(char))) + .unwrap_or(text.len()); + let domain = &text[..length]; + let underscore_free = domain.rsplit('.').take(2).all(|segment| !segment.contains('_')); + underscore_free && domain.chars().any(is_domain_char) } /// Matches `http://` or `https://` followed by a domain and a path. @@ -141,8 +94,9 @@ fn protocol(src: &str, pos: usize) -> Option<usize> { if !matches!(name.as_str(), "http" | "https") || !src[pos + letters..].starts_with("://") { return None; } - let end = domain(src, pos + letters + 3)?; - Some(path(src, end)) + let start = pos + letters + 3; + let end = autolink_end(src, start); + has_valid_domain(&src[start..end]).then_some(end) } /// Matches `www.` followed by a domain and a path. @@ -160,8 +114,8 @@ fn www(src: &str, pos: usize) -> Option<usize> { if !prefix[..3].eq_ignore_ascii_case(b"www") || prefix[3] != b'.' || pos + 4 >= bytes.len() { return None; } - let end = domain(src, pos)?; - Some(path(src, end)) + let end = autolink_end(src, pos); + has_valid_domain(&src[pos..end]).then_some(end) } /// Handles `h` or `w` at the scanner position: a protocol or `www.` autolink. From 12cadb8fb45074be8f3f617665e00e06955315f1 Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Thu, 8 Oct 2026 23:32:54 +0900 Subject: [PATCH 51/88] =?UTF-8?q?=E2=9C=85=20test(lang):=20drop=20the=20se?= =?UTF-8?q?ction=20test=20that=20built=20a=20heading=20of=20depth=207?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit to_h clamps the depth to 1 to 6 now, so a heading with an invalid depth can no longer be made. --- crates/mq-lang/modules/section_test.mq | 5 ----- 1 file changed, 5 deletions(-) diff --git a/crates/mq-lang/modules/section_test.mq b/crates/mq-lang/modules/section_test.mq index d14a99a18..68889a5df 100644 --- a/crates/mq-lang/modules/section_test.mq +++ b/crates/mq-lang/modules/section_test.mq @@ -387,8 +387,3 @@ end def test_section_toc_markdown_empty(): assert_eq(section::toc_markdown([]), []) end - -def test_section_sections_invalid_heading_depth(): - let secs = section::sections([to_h("A", 7)]) - | assert(is_none(secs)) -end From 0865a7b3b70c01fa5643be5fbe4a8f9ef3b84f3c Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Fri, 9 Oct 2026 00:22:54 +0000 Subject: [PATCH 52/88] =?UTF-8?q?=E2=9C=85=20test(markdown):=20replace=20t?= =?UTF-8?q?he=20mdast=20comparison=20with=20rstest=20cases?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The comparison with the official mdast packages needed npm, the network and two JavaScript scripts. Its MDX inputs (hand-written, markdown-rs and micromark tests) and the hand-written Markdown inputs are now rstest cases whose expected tokens were taken once from the official packages. Inputs that were known to differ expect what mq does, with the reason in a comment. Each case also checks that what mq writes reads back the same. The spec examples, the Markdown tests of markdown-rs and the random documents are dropped: `just test-spec` and the property tests cover them. --- crates/mq-markdown/tests/mdast_official.rs | 135 -- .../tests/mdast_official/compare.mjs | 211 --- .../tests/mdast_official/corpus-markdown.json | 12 - .../tests/mdast_official/corpus-mdx.json | 203 --- .../tests/mdast_official/corpus.mjs | 138 -- .../tests/mdast_official/generate.mjs | 120 -- .../tests/mdast_official/known.json | 77 - .../tests/mdast_official/package-lock.json | 1579 ----------------- .../tests/mdast_official/package.json | 16 - .../mdast_official/round_trip_known.json | 8 - crates/mq-markdown/tests/mdast_tests.rs | 1036 +++++++++++ justfile | 8 - 12 files changed, 1036 insertions(+), 2507 deletions(-) delete mode 100644 crates/mq-markdown/tests/mdast_official.rs delete mode 100644 crates/mq-markdown/tests/mdast_official/compare.mjs delete mode 100644 crates/mq-markdown/tests/mdast_official/corpus-markdown.json delete mode 100644 crates/mq-markdown/tests/mdast_official/corpus-mdx.json delete mode 100644 crates/mq-markdown/tests/mdast_official/corpus.mjs delete mode 100644 crates/mq-markdown/tests/mdast_official/generate.mjs delete mode 100644 crates/mq-markdown/tests/mdast_official/known.json delete mode 100644 crates/mq-markdown/tests/mdast_official/package-lock.json delete mode 100644 crates/mq-markdown/tests/mdast_official/package.json delete mode 100644 crates/mq-markdown/tests/mdast_official/round_trip_known.json create mode 100644 crates/mq-markdown/tests/mdast_tests.rs diff --git a/crates/mq-markdown/tests/mdast_official.rs b/crates/mq-markdown/tests/mdast_official.rs deleted file mode 100644 index 26845d0b9..000000000 --- a/crates/mq-markdown/tests/mdast_official.rs +++ /dev/null @@ -1,135 +0,0 @@ -//! Writes the nodes mq makes of each input of the corpus as JSON, for `mdast_official/compare.mjs` to -//! compare with the mdast of the official packages, and checks that what mq writes back reads as the -//! same nodes. Run it with `just test-mdast-official`. -#![cfg(feature = "json")] - -use mq_markdown::Markdown; -use serde_json::{Value, json}; - -#[test] -#[ignore = "needs the official packages and the specs; run via `just test-mdast-official`"] -fn write_nodes_of_the_corpus() { - let corpus = std::fs::read_to_string(std::env::var("MDAST_CORPUS").unwrap()).unwrap(); - let corpus: Vec<Value> = serde_json::from_str(&corpus).unwrap(); - let results: Vec<Value> = corpus - .iter() - .map(|entry| { - let input = entry["input"].as_str().unwrap(); - let parsed = match entry["mode"].as_str().unwrap() { - "mdx" => Markdown::from_mdx_str(input), - _ => Markdown::from_markdown_str(input), - }; - match parsed.and_then(|markdown| markdown.to_json()) { - Ok(nodes) => json!({"error": null, "nodes": serde_json::from_str::<Value>(&nodes).unwrap()}), - Err(error) => json!({"error": error.to_string(), "nodes": []}), - } - }) - .collect(); - std::fs::write(std::env::var("MDAST_OUT").unwrap(), json!(results).to_string()).unwrap(); -} - -/// Normalizes nodes for the comparison of what is written with what was read: positions and the spelling -/// of references, fences and rules are left out, line endings are the same and adjacent text is joined. -fn normalize(value: &mut Value) { - match value { - Value::Object(object) => { - for key in ["position", "label", "fence"] { - object.remove(key); - } - if object.get("type").is_some_and(|kind| kind == "HorizontalRule") { - object.remove("marker"); - } - for (key, value) in object.iter_mut() { - normalize(value); - if let (Value::String(text), true) = (value, key == "value") { - *text = text.replace("\r\n", "\n").replace('\r', "\n"); - } - } - if object.get("type").is_some_and(|kind| kind == "Html") - && let Some(Value::String(text)) = object.get_mut("value") - { - *text = text.trim_end_matches('\n').to_string(); - } - } - Value::Array(array) => { - array.iter_mut().for_each(normalize); - let mut joined: Vec<Value> = Vec::with_capacity(array.len()); - for item in array.drain(..) { - if let Some(last) = joined.last_mut() - && item["type"] == "Text" - && last["type"] == "Text" - { - let text = format!("{}{}", last["value"].as_str().unwrap(), item["value"].as_str().unwrap()); - last["value"] = json!(text); - continue; - } - joined.push(item); - } - *array = joined; - } - _ => {} - } -} - -fn parse(mode: &str, input: &str) -> Option<Value> { - let markdown = match mode { - "mdx" => Markdown::from_mdx_str(input), - _ => Markdown::from_markdown_str(input), - } - .ok()?; - let mut nodes: Value = serde_json::from_str(&markdown.to_json().ok()?).unwrap(); - normalize(&mut nodes); - Some(nodes) -} - -/// Inputs whose written form reads as other nodes, by id, which `round_trip_known.json` lists. -#[test] -#[ignore = "needs the corpus; run via `just test-mdast-official`"] -fn the_corpus_round_trips() { - let corpus = std::fs::read_to_string(std::env::var("MDAST_CORPUS").unwrap()).unwrap(); - let corpus: Vec<Value> = serde_json::from_str(&corpus).unwrap(); - let known: Value = serde_json::from_str(include_str!("mdast_official/round_trip_known.json")).unwrap(); - let mut unexpected = Vec::new(); - let mut fixed = Vec::new(); - for entry in &corpus { - let (id, mode, input) = ( - entry["id"].as_str().unwrap(), - entry["mode"].as_str().unwrap(), - entry["input"].as_str().unwrap(), - ); - let Some(before) = parse(mode, input) else { continue }; - let written = match mode { - "mdx" => Markdown::from_mdx_str(input), - _ => Markdown::from_markdown_str(input), - } - .unwrap() - .to_string(); - let same = parse(mode, &written).as_ref() == Some(&before); - match (same, known.get(id).is_some()) { - (false, false) => unexpected.push(format!( - "{id} {input:?}\n written: {written:?}\n before: {}\n after: {}", - before, - parse(mode, &written).map_or("error".to_string(), |after| after.to_string()) - )), - (true, true) => fixed.push(id.to_string()), - _ => {} - } - } - let ids: Vec<&str> = corpus.iter().map(|entry| entry["id"].as_str().unwrap()).collect(); - for id in known.as_object().unwrap().keys() { - assert!( - ids.contains(&id.as_str()), - "round_trip_known.json has an id that is not in the corpus: {id}" - ); - } - println!("{} inputs, {} known", corpus.len(), known.as_object().unwrap().len()); - assert!( - unexpected.is_empty(), - "do not read back as the same nodes:\n{}", - unexpected.join("\n") - ); - assert!( - fixed.is_empty(), - "now the same, remove from round_trip_known.json: {fixed:?}" - ); -} diff --git a/crates/mq-markdown/tests/mdast_official/compare.mjs b/crates/mq-markdown/tests/mdast_official/compare.mjs deleted file mode 100644 index defdeacae..000000000 --- a/crates/mq-markdown/tests/mdast_official/compare.mjs +++ /dev/null @@ -1,211 +0,0 @@ -// Parses each input of the corpus (first argument) with the official packages and compares the nodes with -// those mq wrote (second argument). Both are put in the same flat form first: the kinds and values of the -// nodes in document order, with containers opened by `kind(` and closed by `)`. Paragraphs are not -// marked, since mq has none, and the items of nested lists follow their parent item, as in mq. -// Differences must be listed in known.json by the id of the input. -import fs from 'node:fs' -import {fromMarkdown} from 'mdast-util-from-markdown' -import {frontmatterFromMarkdown} from 'mdast-util-frontmatter' -import {gfmFromMarkdown} from 'mdast-util-gfm' -import {mathFromMarkdown} from 'mdast-util-math' -import {mdxFromMarkdown} from 'mdast-util-mdx' -import {frontmatter} from 'micromark-extension-frontmatter' -import {gfm} from 'micromark-extension-gfm' -import {math} from 'micromark-extension-math' -import {mdxjs} from 'micromark-extension-mdxjs' - -const read = (file) => JSON.parse(fs.readFileSync(file, 'utf8')) -const corpus = read(process.argv[2]) -const actual = read(process.argv[3]) -const known = read(new URL('known.json', import.meta.url)) - -const json = JSON.stringify -const text = (value) => `T${json(value)}` - -/** Joins adjacent text tokens. */ -function merge(tokens) { - const out = [] - for (const token of tokens) { - if (token.startsWith('T') && out.length && out.at(-1).startsWith('T')) { - out[out.length - 1] = text(JSON.parse(out.at(-1).slice(1)) + JSON.parse(token.slice(1))) - } else { - out.push(token) - } - } - return out -} - -const container = (head, inner) => [`${head}(`, ...inner, ')'] -const mdxAttribute = (attribute) => - attribute.type === 'mdxJsxExpressionAttribute' - ? `...{${attribute.value}}` - : attribute.value == null - ? attribute.name - : typeof attribute.value === 'string' - ? `${attribute.name}=lit:${json(attribute.value)}` - : `${attribute.name}=expr:{${attribute.value.value}}` -const item = (level, ordered, checked, spread) => - `item ${level} ${ordered === null ? 'u' : `o${ordered}`} ${checked === true ? 'x' : checked === false ? '_' : '-'} ${spread ? 's' : 't'}` -const alignment = (align) => `align ${json(align.map((a) => a ?? 'none'))}` - -function fromMdast(node, level = 0) { - const kids = (inner = true) => { - const tokens = (node.children ?? []).flatMap((child) => fromMdast(child, level)) - return inner ? merge(tokens) : tokens - } - switch (node.type) { - case 'root': return kids(false) - case 'paragraph': return kids() - case 'heading': return container(`heading ${node.depth}`, kids()) - case 'thematicBreak': return ['hr'] - case 'code': return [`code ${node.lang ?? ''} ${node.meta ?? ''} ${json(node.value)}`] - case 'html': return [`html ${json(node.value)}`] - case 'blockquote': return container('blockquote', kids(false)) - case 'list': - return node.children.flatMap((listItem, index) => { - const nested = [] - const body = listItem.children.flatMap((child) => - child.type === 'list' ? (nested.push(...fromMdast(child, level + 1)), []) : fromMdast(child, level), - ) - const number = node.ordered ? (node.start ?? 1) + index : null - return [...container(item(level, number, listItem.checked, node.spread), body), ...nested] - }) - case 'table': - return node.children.flatMap((row, r) => [ - ...row.children.flatMap((cell, c) => container(`cell ${r} ${c}`, merge(cell.children.flatMap((n) => fromMdast(n))))), - ...(r === 0 ? [alignment(node.align)] : []), - ]) - case 'definition': return [`definition ${node.identifier} ${node.url} ${node.title ?? ''}`] - case 'footnoteDefinition': return container(`footnote ${node.identifier}`, kids(false)) - case 'footnoteReference': return [`footnoteref ${node.identifier}`] - case 'math': return [`math ${json(node.value)}`] - case 'inlineMath': return [`imath ${json(node.value)}`] - case 'yaml': return [`yaml ${json(node.value)}`] - case 'toml': return [`toml ${json(node.value)}`] - case 'text': return [text(node.value)] - case 'emphasis': return container('em', kids()) - case 'strong': return container('strong', kids()) - case 'delete': return container('del', kids()) - case 'inlineCode': return [`code_inline ${json(node.value)}`] - case 'break': return ['br'] - case 'link': return container(`link ${node.url} ${node.title ?? ''}`, kids()) - case 'image': return [`image ${node.url} ${node.title ?? ''} ${json(node.alt)}`] - case 'linkReference': return container(`linkref ${node.identifier}`, kids()) - case 'imageReference': return [`imageref ${node.identifier} ${json(node.alt)}`] - case 'mdxJsxFlowElement': - case 'mdxJsxTextElement': { - const kind = node.type === 'mdxJsxFlowElement' ? 'flow' : 'text' - const attributes = node.attributes.map((a) => ' ' + mdxAttribute(a)).join('') - return container(`${kind} <${node.name ?? ''}${attributes}>`, kids(node.type === 'mdxJsxTextElement')) - } - case 'mdxFlowExpression': return [`flowexpr {${node.value}}`] - case 'mdxTextExpression': return [`textexpr {${node.value}}`] - case 'mdxjsEsm': return [`esm ${json(node.value)}`] - default: throw new Error(`unknown mdast node ${node.type}`) - } -} - -function mqAttribute(attribute) { - if (attribute.type === 'expression') return `...{${attribute.value}}` - const {name, value} = attribute.value - if (value === null) return name - return value.type === 'literal' ? `${name}=lit:${json(value.value)}` : `${name}=expr:{${value.value}}` -} - -function fromMq(node) { - const kids = (inner = true) => { - const tokens = (node.values ?? node.children ?? []).flatMap(fromMq) - return inner ? merge(tokens) : tokens - } - switch (node.type) { - case 'Heading': return container(`heading ${node.depth}`, kids()) - case 'HorizontalRule': return ['hr'] - case 'Code': return [`code ${node.lang ?? ''} ${node.meta ?? ''} ${json(node.value)}`] - case 'Html': return [`html ${json(node.value)}`] - case 'Blockquote': return container('blockquote', kids(false)) - case 'List': { - const number = node.ordered ? (node.start ?? 1) + node.index : null - return container(item(node.level, number, node.checked, node.spread), kids(false)) - } - case 'TableCell': return container(`cell ${node.row} ${node.column}`, kids()) - case 'TableAlign': return [alignment(node.align)] - case 'Definition': return [`definition ${node.ident} ${node.url} ${node.title ?? ''}`] - case 'Footnote': return container(`footnote ${node.ident}`, kids(false)) - case 'FootnoteRef': return [`footnoteref ${node.ident}`] - case 'Math': return [`math ${json(node.value)}`] - case 'MathInline': return [`imath ${json(node.value)}`] - case 'Yaml': return [`yaml ${json(node.value)}`] - case 'Toml': return [`toml ${json(node.value)}`] - case 'Text': return [text(node.value)] - case 'Emphasis': return container('em', kids()) - case 'Strong': return container('strong', kids()) - case 'Delete': return container('del', kids()) - case 'CodeInline': return [`code_inline ${json(node.value)}`] - case 'Break': return ['br'] - case 'Link': return container(`link ${node.url} ${node.title ?? ''}`, kids()) - case 'Image': return [`image ${node.url} ${node.title ?? ''} ${json(node.alt)}`] - case 'LinkRef': return container(`linkref ${node.ident}`, kids()) - case 'ImageRef': return [`imageref ${node.ident} ${json(node.alt)}`] - case 'MdxJsxFlowElement': - case 'MdxJsxTextElement': { - const kind = node.type === 'MdxJsxFlowElement' ? 'flow' : 'text' - const attributes = node.attributes.map((a) => ' ' + mqAttribute(a)).join('') - return container(`${kind} <${node.name ?? ''}${attributes}>`, kids(node.type === 'MdxJsxTextElement')) - } - case 'MdxFlowExpression': return [`flowexpr {${node.value}}`] - case 'MdxTextExpression': return [`textexpr {${node.value}}`] - case 'MdxJsEsm': return [`esm ${json(node.value)}`] - case 'Empty': return [] - default: throw new Error(`unknown mq node ${node.type}`) - } -} - -const OPTIONS = { - markdown: { - extensions: [gfm(), frontmatter(['yaml', 'toml']), math()], - mdastExtensions: [gfmFromMarkdown(), frontmatterFromMarkdown(['yaml', 'toml']), mathFromMarkdown()], - }, - mdx: { - extensions: [mdxjs(), frontmatter(['yaml', 'toml'])], - mdastExtensions: [mdxFromMarkdown(), frontmatterFromMarkdown(['yaml', 'toml'])], - }, -} - -function official({mode, input}) { - try { - return {error: null, tokens: fromMdast(fromMarkdown(input, OPTIONS[mode]))} - } catch (error) { - if (error.message.startsWith('unknown mdast node')) throw error - return {error: String(error.reason ?? error.message).split('\n')[0], tokens: []} - } -} - -const unexpected = [] -const fixed = [] -let differing = 0 -corpus.forEach((entry, index) => { - const expected = official(entry) - const mq = actual[index] - const got = mq.error === null ? {error: null, tokens: mq.nodes.flatMap(fromMq)} : {error: mq.error, tokens: []} - const same = - (expected.error === null) === (got.error === null) && - (expected.error !== null || json(expected.tokens) === json(got.tokens)) - if (!same) differing++ - if (!same && !(entry.id in known)) { - const at = expected.tokens.findIndex((token, i) => token !== got.tokens[i]) - unexpected.push( - `${entry.id} ${json(entry.input)}\n official: ${expected.error ?? json(expected.tokens.slice(Math.max(at, 0)))}\n mq: ${got.error ?? json(got.tokens.slice(Math.max(at, 0)))}`, - ) - } else if (same && entry.id in known) { - fixed.push(entry.id) - } -}) - -const ids = new Set(corpus.map((entry) => entry.id)) -for (const id of Object.keys(known)) { - if (!ids.has(id)) unexpected.push(`known.json has an id that is not in the corpus: ${id}`) -} -if (unexpected.length) console.error(`differs from the official packages:\n${unexpected.join('\n')}`) -if (fixed.length) console.error(`now the same, remove from known.json:\n${fixed.join('\n')}`) -console.log(`${corpus.length} inputs, ${differing} differ, ${Object.keys(known).length} known`) -process.exit(unexpected.length || fixed.length ? 1 : 0) diff --git a/crates/mq-markdown/tests/mdast_official/corpus-markdown.json b/crates/mq-markdown/tests/mdast_official/corpus-markdown.json deleted file mode 100644 index c9e817abf..000000000 --- a/crates/mq-markdown/tests/mdast_official/corpus-markdown.json +++ /dev/null @@ -1,12 +0,0 @@ -[ -"# a\n\n- x\n - y\n- [ ] z\n\n1. q\n", -"| a | b |\n|:-|-:|\n| 1 | 2 |\n", -"> t *e* **s** ~~d~~ `c` [l](u \"t\") ![i](u) [r][d] <b> a \nb\n\n[d]: /u\n", -"[^1]\n\n[^1]: note\n", -"---\ntitle: x\n---\n\n# a\n", -"+++\ntitle = 1\n+++\n\na\n", -"$$\nm\n$$\n\n$i$\n", -"```rs x\nc\n```\n", -"a\n\nb\n", -"*a*\n\n*b*\n" -] diff --git a/crates/mq-markdown/tests/mdast_official/corpus-mdx.json b/crates/mq-markdown/tests/mdast_official/corpus-mdx.json deleted file mode 100644 index 6223a95be..000000000 --- a/crates/mq-markdown/tests/mdast_official/corpus-mdx.json +++ /dev/null @@ -1,203 +0,0 @@ -[ -"<a />", -"<a b=\"c&amp;d\" e='f' g={h} {...i} j />", -"<a.b.c />", -"<a:b c:d=\"e\" />", -"<></>", -"<a>x</a>", -"<a>\n\nx\n\n</a>", -"<a>\n x\n</a>", -"a <b>c</b> d", -"a <b/> d", -"{a}", -"{a {b} c}", -"{a\nb}", -"{a\n b\n c}", -"x {a} y", -"{a} <b/>", -"<b/> {a}", -"<b/> x", -"{a}x", -"<a>{b}</a>", -"<a>b", -"<a></b>", -"a < b", -"a <3", -"<a b=>", -"<a>\n<b>\n</b>\n</a>", -"> <a>\n> x\n> </a>", -"- <a>\n x\n </a>", -"<a b='c\"d' />", -"<a/><b/>", -"<a>x</a><b>y</b>", -"a\n<b/>", -"a\n{b}", -"<b e= \"f\"/>", -"a <b e = \"f\"/>.", -"<b c\n= \"x\">c</b>", -"a <b xml :\tlang\n= \"de-CH\" foo:bar>c</b>.", -"a <b a b : c d : e = \"f\" g/>.", -"<b e= {f}/>", -"<a>\n\n# h\n\n- l\n\n</a>", -"<Zoom>\n\n![a](b)\n\n</Zoom>", -"<A>\n <B>\n x\n </B>\n</A>", -"import a from \"b\"\nexport const x = 1\n\nx\n", -"export default d\n", -" import a from \"b\"\n", -"import{a} from \"b\"\n", -"importx a\n", -"- import a from \"b\"\n", -"text\nimport a from \"b\"\n", -"a {", -"{", -"<a", -"<a b", -"<a b=", -"<a b=\"", -"<a b={", -"</a>", -"<a></a></a>", -"<a>", -"<a><b></a></b>", -"a <b c=\"d\" /> e", -"a <b>c</b>", -"a <b>c", -"<a b=\"c\nd\" />", -"<a b='&lt;' />", -"<a b=\"&#x41;\" />", -"<a {...b} {...c} />", -"<a {b} />", -"<a b={c} d={e} />", -"<a b={{c: 1}} />", -"{/* comment */}", -"{}", -"a {} b", -"a {b} {c} d", -"<a>{b}{c}</a>", -"# <a/>", -"# a {b} c", -"*<a>b</a>*", -"**a {b}**", -"[<a/>](b)", -"`<a/>`", -"`{a}`", -"<a/>\n<b/>", -"<a/> <b/>", -"<a/>x\n", -"x<a/>", -"<a>\n\nb\n\n</a>\n\n<c/>", -"<a>\nb\n</a>", -"<a>b\n</a>", -"<a>\nb</a>", -"<a>\n <b/>\n</a>", -"- a\n <b/>\n", -"> a\n> <b/>\n", -"<a>\n- b\n</a>", -"<a>\n\n- b\n\n</a>", -"<a b=\"c\" d=\"e\">\n\nf\n\n</a>", -"<a-b />", -"<a.b-c />", -"<a:b.c />", -"<a.b:c />", -"<_a />", -"<$a />", -"<1 />", -"< a />", -"<a / >", -"<a/ >", -"<a b-c=\"d\" />", -"<a b:c-d=\"e\" />", -"<a b:c:d />", -"<a ={b} />", -"<a b=c />", -"<a b='c' d />", -"<a\nb\n/>", -"<a\n b=\"c\"\n/>", -"{a}\n{b}", -"{a}\n\n{b}", -"{a} {b}", -"{a}{b}", -"x\n{a}\n", -"{\n}", -"{a}\n<b/>", -"<b/>\n{a}", -"a <b> c </b> d", -"<b>\n</b>", -"<b></b>", -"<b> </b>", -"</>", -"<>a</>", -"<>\n\na\n\n</>", -"<></a>", -"<a>{`}`}</a>", -"{'}'}", -"{\"}\"}", -"<a b=\"{c}\" />", -"<a b={\"c\"} />", -"<a b={`c`} />", -"a\\<b/>", -"a\\{b}", -"&lt;a/>", -"a &#123;b}", -"<!-- a -->", -"<a>\n<!-- b -->\n</a>", -"<https://a.b>", -"<a@b.c>", -"https://a.b", -"www.a.b", -"---\ntitle: x\n---\n\na", -"a\n---\n", -"***\n", -"```\n<a/>\n```\n", -" <a/>\n", -"<a/>\n x\n", -"<a>\n\n x\n\n</a>", -"1. <a/>\n", -"* <a>\n b\n </a>\n", -"x <a", -"x <a b", -"x <a b=", -"<a\n\nb", -"<a\nb", -"a\n<a", -"<a b=\"c", -"x <a b=\"c", -"x <", -"<", -"x <a\n\nb", -"x <a b\n\nc", -"- <a", -"> <a", -"<a>\n<b", -"<a>\n\n<b", -"<a>x</a", -"<a>x</", -"<a>x<", -"a {b", -"a {b\n\nc}", -"{a\n\nb}", -"x {a}\n\ny {", -"{a /* } */}", -"{`${'}'}`}", -"<a b={'}'} />", -"{a // }\n}", -"{\"a}\" + '{'}", -"<a b={\"}\"} c='}' />", -"{a /* { */}", -"{`a${b}c`}", -"{`a${`}`}c`}", -"{'\\'}'}", -"{a // {\n}", -"x {'}'} y", -"x {`}`} y", -"{/* } */}\n", -"<a>{'}'}</a>", -"{'a\nb'}", -"{a / b}", -"{a /* x */ + 1}", -"---\ntitle: x\n---\n\n<A />\n", -"+++\na = 1\n+++\n\nb\n", -"---\na\n", -"---\n\n---\n\nx\n", -"---\ntitle: x\n---\nimport a from \"b\"\n\n{a}\n" -] \ No newline at end of file diff --git a/crates/mq-markdown/tests/mdast_official/corpus.mjs b/crates/mq-markdown/tests/mdast_official/corpus.mjs deleted file mode 100644 index 801e7b948..000000000 --- a/crates/mq-markdown/tests/mdast_official/corpus.mjs +++ /dev/null @@ -1,138 +0,0 @@ -// Writes the inputs to compare to the file in the first argument: the hand-written corpora and the -// examples of the specs, which are fetched over the network. -import fs from 'node:fs' -import {generate} from './generate.mjs' - -const dir = new URL('.', import.meta.url) -const read = (file) => JSON.parse(fs.readFileSync(new URL(file, dir), 'utf8')) - -const SPECS = [ - ['commonmark', 'https://raw.githubusercontent.com/commonmark/commonmark-spec/0.31.2/spec.txt'], - ['gfm', 'https://raw.githubusercontent.com/github/cmark-gfm/828322d1ee4facdab56f0d3edccb13e9af90dcd2/test/spec.txt'], - ['gfm-extensions', 'https://raw.githubusercontent.com/github/cmark-gfm/828322d1ee4facdab56f0d3edccb13e9af90dcd2/test/extensions.txt'], -] - -const MARKDOWN_RS = 'https://raw.githubusercontent.com/wooorm/markdown-rs/1.0.0/tests' -// Test files whose inputs are read as MDX; the others are read as Markdown. `mdx_swc` needs a JavaScript parser. -const MDX_TESTS = ['mdx_esm', 'mdx_expression_flow', 'mdx_expression_text', 'mdx_jsx_flow', 'mdx_jsx_text'] -const MARKDOWN_RS_TESTS = [ - ...MDX_TESTS, - 'attention', 'autolink', 'block_quote', 'character_escape', 'character_reference', 'code_fenced', - 'code_indented', 'code_text', 'commonmark', 'definition', 'frontmatter', 'gfm_autolink_literal', - 'gfm_footnote', 'gfm_strikethrough', 'gfm_table', 'gfm_tagfilter', 'gfm_task_list_item', - 'hard_break_escape', 'hard_break_trailing', 'heading_atx', 'heading_setext', 'html_flow', 'html_text', - 'image', 'link_reference', 'link_resource', 'list', 'math_flow', 'math_text', 'misc_bom', - 'misc_dangerous_html', 'misc_dangerous_protocol', 'misc_line_ending', 'misc_soft_break', 'misc_tabs', - 'misc_url', 'misc_zero', 'text', 'thematic_break', -] -const MICROMARK_TESTS = [ - 'https://raw.githubusercontent.com/micromark/micromark-extension-mdx-jsx/3.0.2/test/index.js', - 'https://raw.githubusercontent.com/micromark/micromark-extension-mdx-expression/micromark-extension-mdx-expression%403.0.1/test/index.js', - 'https://raw.githubusercontent.com/micromark/micromark-extension-mdxjs-esm/3.0.0/test/index.js', -] - -const fence = '`'.repeat(32) - -const ESCAPES = {n: '\n', r: '\r', t: '\t', 0: '\0', '\\': '\\', '"': '"', "'": "'", '`': '`', $: '$'} - -/** The string literal that starts at `index` in Rust or JavaScript source, or null. */ -function literal(source, index) { - const quote = source[index] - if (!['"', "'", '`'].includes(quote)) return null - let value = '' - for (let i = index + 1; i < source.length; i++) { - const char = source[i] - if (char === quote) return value - if (char !== '\\') { - value += char - continue - } - const next = source[++i] - if (next === '\n') { - // A Rust line continuation drops the whitespace that follows. - while (/\s/.test(source[i + 1] ?? '')) i++ - } else if (next === 'u' && source[i + 1] === '{') { - const end = source.indexOf('}', i) - value += String.fromCodePoint(parseInt(source.slice(i + 2, end), 16)) - i = end - } else if (next === 'u' || next === 'x') { - const length = next === 'u' ? 4 : 2 - value += String.fromCodePoint(parseInt(source.slice(i + 1, i + 1 + length), 16)) - i += length - } else if (next in ESCAPES) { - value += ESCAPES[next] - } else { - return null - } - } - return null -} - -/** The first argument, when it is a string literal, of each call to one of `names` in `source`. */ -function firstArguments(source, names) { - const out = [] - const pattern = new RegExp(`\\b(?:${names.join('|')})\\(\\s*(?=["'\`])`, 'g') - for (const match of source.matchAll(pattern)) { - const value = literal(source, match.index + match[0].length) - if (value !== null) out.push(value) - } - return out -} - -async function fetchText(url) { - const response = await fetch(url) - if (!response.ok) throw new Error(`${url}: ${response.status}`) - return response.text() -} - -/** The markdown of each example of a spec, numbered from 1 as in the specs. */ -function examples(text) { - const lines = text.split('\n') - const out = [] - for (let i = 0; i < lines.length; i++) { - if (lines[i] !== `${fence} example`) continue - const markdown = [] - for (i++; lines[i] !== '.'; i++) markdown.push(lines[i]) - while (lines[i] !== fence) i++ - out.push(markdown.join('\n').replaceAll('→', '\t') + '\n') - } - return out -} - -const corpus = [] -for (const input of read('corpus-mdx.json')) corpus.push({id: `mdx:${input}`, mode: 'mdx', input}) -for (const input of read('corpus-markdown.json')) corpus.push({id: `markdown:${input}`, mode: 'markdown', input}) -for (const [name, url] of SPECS) { - examples(await fetchText(url)).forEach((input, index) => { - corpus.push({id: `${name}#${index + 1}`, mode: 'markdown', input}) - }) -} - -const seen = new Set(corpus.map((entry) => `${entry.mode}\0${entry.input}`)) -function addTests(name, mode, inputs) { - let count = 0 - for (const input of inputs) { - if (input.length > 5000 || !input.isWellFormed() || seen.has(`${mode}\0${input}`)) continue - seen.add(`${mode}\0${input}`) - corpus.push({id: `${name}#${++count}`, mode, input}) - } -} -for (const name of MARKDOWN_RS_TESTS) { - const source = await fetchText(`${MARKDOWN_RS}/${name}.rs`) - const mode = MDX_TESTS.includes(name) ? 'mdx' : 'markdown' - addTests(`markdown-rs/${name}`, mode, firstArguments(source, ['to_html', 'to_html_with_options', 'to_mdast'])) -} -for (const url of MICROMARK_TESTS) { - const name = url.split('/').slice(-4, -2).join('/').replace('%40', '@') - addTests(name, 'mdx', firstArguments(await fetchText(url), ['micromark'])) -} -// Random documents. MDAST_RANDOM is how many of each kind, MDAST_SEED the seed. -const random = Number(process.env.MDAST_RANDOM ?? 500) -const seed = Number(process.env.MDAST_SEED ?? 1) -for (const [mode, offset] of [['markdown', 0], ['mdx', 1]]) { - generate(random, seed + offset, mode === 'mdx').forEach((input, index) => { - corpus.push({id: `random-${mode}-${seed}#${index + 1}`, mode, input}) - }) -} -fs.writeFileSync(process.argv[2], JSON.stringify(corpus)) -console.log(`${corpus.length} inputs`) diff --git a/crates/mq-markdown/tests/mdast_official/generate.mjs b/crates/mq-markdown/tests/mdast_official/generate.mjs deleted file mode 100644 index 7289887a9..000000000 --- a/crates/mq-markdown/tests/mdast_official/generate.mjs +++ /dev/null @@ -1,120 +0,0 @@ -// Random documents built from a grammar of Markdown and MDX constructs. The same seed makes the same -// documents, so the ids of the inputs stay the same and known.json can list them. - -/** A small seeded generator (mulberry32). */ -function random(seed) { - let state = seed >>> 0 - return () => { - state = (state + 0x6d2b79f5) >>> 0 - let t = state - t = Math.imul(t ^ (t >>> 15), t | 1) - t ^= t + Math.imul(t ^ (t >>> 7), t | 61) - return ((t ^ (t >>> 14)) >>> 0) / 4294967296 - } -} - -const WORDS = ['a', 'bc', 'foo', 'x1', 'Zed', 'été', '日本', 'a_b', '1', 'q'] - -export function generate(count, seed, mdx) { - const next = random(seed) - const pick = (list) => list[Math.floor(next() * list.length)] - const chance = (p) => next() < p - const word = () => pick(WORDS) - const words = (max = 4) => Array.from({length: 1 + Math.floor(next() * max)}, word).join(' ') - - const inline = (depth = 0) => { - const parts = [] - for (let i = 0, n = 1 + Math.floor(next() * 3); i < n; i++) { - const choices = [ - () => words(), - () => `*${depth < 2 ? inline(depth + 1) : word()}*`, - () => `**${depth < 2 ? inline(depth + 1) : word()}**`, - () => `\`${word()}\``, - () => `[${word()}](/${word()}${chance(0.3) ? ` "${word()}"` : ''})`, - () => `![${word()}](/${word()}.png)`, - () => `[${word()}][r${Math.floor(next() * 2)}]`, - () => `${word()} \n${word()}`, - () => `\\${pick(['*', '_', '[', '#', '`'])}${word()}`, - () => `&${pick(['amp', 'lt', 'copy'])};`, - ] - if (!mdx) { - choices.push( - () => `~~${word()}~~`, - () => `$${word()}$`, - () => `[^f${Math.floor(next() * 2)}]`, - () => `<${pick(['b', 'span'])}>${word()}</${pick(['b', 'span'])}>`, - ) - } else { - choices.push( - () => `<${pick(['B', 'a.b', 'x:y'])}${attributes()}>${depth < 2 ? inline(depth + 1) : word()}</${pick(['B', 'a.b', 'x:y'])}>`, - () => `<${pick(['B', 'C'])}${attributes()} />`, - () => `{${expression()}}`, - ) - } - parts.push(pick(choices)()) - } - return parts.join(chance(0.7) ? ' ' : '') - } - - const expression = () => pick(['a', '1 + 1', '"x"', "'}'", '`t${a}`', 'a.b', '[1, 2]', '{a: 1}', 'f(a, b)', '/* c */ a']) - const attributes = () => - Array.from({length: Math.floor(next() * 3)}, () => - pick([` a="${word()}"`, ` b='${word()}'`, ` c={${expression()}}`, ' d', ` {...${pick(['p', 'q.r'])}}`, ` e = "${word()}"`]), - ).join('') - - const block = (depth = 0) => { - const choices = [ - () => inline(), - () => `${'#'.repeat(1 + Math.floor(next() * 6))} ${inline()}`, - () => `${inline()}\n${chance(0.5) ? '===' : '---'}`, - () => pick(['***', '---', '___', '- - -']), - () => `\`\`\`${pick(['', 'js', 'rs x'])}\n${words()}\n${chance(0.3) ? '\n' : ''}\`\`\``, - () => ` ${words()}`, - () => `> ${block(depth + 1).replaceAll('\n', '\n> ')}`, - () => list(depth), - () => `[r${Math.floor(next() * 2)}]: /${word()}${chance(0.3) ? ` "${word()}"` : ''}`, - ] - if (!mdx) { - choices.push( - () => `| ${word()} | ${word()} |\n|${pick([':-', '-', '-:', ':-:'])}|${pick(['-', ':-'])}|\n| ${word()} | ${word()} |`, - () => `$$\n${word()}\n$$`, - () => `[^f${Math.floor(next() * 2)}]: ${inline()}`, - () => `<div>\n${word()}\n</div>`, - () => `<!-- ${word()} -->`, - ) - } else { - choices.push( - () => `<${pick(['A', 'B'])}${attributes()} />`, - () => `{${expression()}}`, - () => { - const name = pick(['A', 'B', 'a.b']) - return `<${name}${attributes()}>\n\n${depth < 2 ? block(depth + 1) : word()}\n\n</${name}>` - }, - () => `<${pick(['A', 'B'])}>${inline()}</${pick(['A', 'B'])}>`, - ) - } - return pick(choices)() - } - - const list = (depth) => { - const ordered = chance(0.4) - const items = Array.from({length: 1 + Math.floor(next() * 3)}, (_, i) => { - const marker = ordered ? `${i + 1}.` : pick(['-', '*', '+']) - const task = !ordered && !mdx && chance(0.2) ? pick(['[ ] ', '[x] ']) : '' - const nested = depth < 2 && chance(0.3) ? `\n${list(depth + 1).replaceAll(/^/gm, ' ')}` : '' - return `${marker} ${task}${inline()}${nested}` - }) - return items.join(chance(0.2) ? '\n\n' : '\n') - } - - const documents = [] - for (let i = 0; i < count; i++) { - const blocks = Array.from({length: 1 + Math.floor(next() * 5)}, () => block()) - // A document that starts with a line of dashes makes the official parser read what follows as text. - if (/^(---|- - -)/.test(blocks[0])) blocks[0] = '***' - if (mdx && chance(0.15)) blocks.unshift(pick(['import a from "b"', 'export const x = 1'])) - if (!mdx && chance(0.1)) blocks.unshift(`---\ntitle: ${word()}\n---`) - documents.push(blocks.join('\n\n') + '\n') - } - return documents -} diff --git a/crates/mq-markdown/tests/mdast_official/known.json b/crates/mq-markdown/tests/mdast_official/known.json deleted file mode 100644 index 9140b1fc9..000000000 --- a/crates/mq-markdown/tests/mdast_official/known.json +++ /dev/null @@ -1,77 +0,0 @@ -{ - "mdx:{a {b} c}": "Expressions are not parsed as JavaScript, so invalid JavaScript is accepted.", - "mdx:{a\nb}": "Expressions are not parsed as JavaScript, so invalid JavaScript is accepted.", - "mdx:{a\n b\n c}": "Expressions are not parsed as JavaScript, so invalid JavaScript is accepted.", - "mdx:<a {b} />": "Expressions are not parsed as JavaScript, so invalid JavaScript is accepted.", - "mdx:{a\n\nb}": "Expressions are not parsed as JavaScript, so invalid JavaScript is accepted.", - "mdx:{'a\nb'}": "Expressions are not parsed as JavaScript, so invalid JavaScript is accepted.", - "mdx:<a": "A tag that is not closed before the end of its text is text here and in markdown-rs, and an error in the JavaScript packages.", - "mdx:<a b": "A tag that is not closed before the end of its text is text here and in markdown-rs, and an error in the JavaScript packages.", - "mdx:<a b=": "A tag that is not closed before the end of its text is text here and in markdown-rs, and an error in the JavaScript packages.", - "mdx:x <a": "A tag that is not closed before the end of its text is text here and in markdown-rs, and an error in the JavaScript packages.", - "mdx:x <a b": "A tag that is not closed before the end of its text is text here and in markdown-rs, and an error in the JavaScript packages.", - "mdx:x <a b=": "A tag that is not closed before the end of its text is text here and in markdown-rs, and an error in the JavaScript packages.", - "mdx:<a\n\nb": "A tag that is not closed before the end of its text is text here and in markdown-rs, and an error in the JavaScript packages.", - "mdx:<a\nb": "A tag that is not closed before the end of its text is text here and in markdown-rs, and an error in the JavaScript packages.", - "mdx:a\n<a": "A tag that is not closed before the end of its text is text here and in markdown-rs, and an error in the JavaScript packages.", - "mdx:x <": "A tag that is not closed before the end of its text is text here and in markdown-rs, and an error in the JavaScript packages.", - "mdx:<": "A tag that is not closed before the end of its text is text here and in markdown-rs, and an error in the JavaScript packages.", - "mdx:x <a\n\nb": "A tag that is not closed before the end of its text is text here and in markdown-rs, and an error in the JavaScript packages.", - "mdx:x <a b\n\nc": "A tag that is not closed before the end of its text is text here and in markdown-rs, and an error in the JavaScript packages.", - "mdx:- <a": "A tag that is not closed before the end of its text is text here and in markdown-rs, and an error in the JavaScript packages.", - "mdx:> <a": "A tag that is not closed before the end of its text is text here and in markdown-rs, and an error in the JavaScript packages.", - "commonmark#606": "GFM autolink literals: an email after a backslash escape is text here, as in the CommonMark spec.", - "gfm#602": "GFM autolink literals: an email after a backslash escape is text here, as in the CommonMark spec.", - "gfm-extensions#19": "GFM autolink literals: the text of a `mailto:` link includes the scheme here, as in the GFM spec.", - "gfm-extensions#20": "GFM autolink literals: the spec only requires that this input does not crash.", - "markdown-rs/mdx_esm#13": "Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted.", - "markdown-rs/mdx_esm#14": "Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted.", - "markdown-rs/mdx_esm#17": "Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted.", - "markdown-rs/mdx_esm#19": "Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted.", - "markdown-rs/mdx_esm#20": "Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted.", - "markdown-rs/mdx_esm#25": "Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted.", - "markdown-rs/mdx_expression_flow#23": "Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted.", - "markdown-rs/mdx_expression_flow#26": "Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted.", - "markdown-rs/mdx_expression_flow#27": "Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted.", - "markdown-rs/mdx_expression_flow#28": "Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted.", - "markdown-rs/mdx_expression_text#16": "Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted.", - "markdown-rs/mdx_expression_text#18": "Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted.", - "markdown-rs/mdx_expression_text#19": "Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted.", - "markdown-rs/mdx_expression_text#27": "Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted.", - "markdown-rs/mdx_jsx_flow#4": "Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted.", - "markdown-rs/mdx_jsx_text#30": "Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted.", - "markdown-rs/mdx_jsx_text#35": "Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted.", - "markdown-rs/mdx_jsx_text#36": "Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted.", - "markdown-rs/mdx_jsx_text#37": "Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted.", - "markdown-rs/mdx_jsx_text#122": "Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted.", - "markdown-rs/mdx_jsx_text#123": "Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted.", - "markdown-rs/mdx_jsx_text#128": "Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted.", - "micromark-extension-mdx-expression/micromark-extension-mdx-expression@3.0.1#6": "Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted.", - "micromark-extension-mdx-expression/micromark-extension-mdx-expression@3.0.1#7": "Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted.", - "micromark-extension-mdx-expression/micromark-extension-mdx-expression@3.0.1#9": "Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted.", - "micromark-extension-mdxjs-esm/3.0.0#11": "Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted.", - "micromark-extension-mdxjs-esm/3.0.0#12": "Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted.", - "markdown-rs/code_fenced#34": "A null character is kept in values, as in markdown-rs, and replaced with U+FFFD only in HTML; the JavaScript packages replace it in values too.", - "markdown-rs/hard_break_trailing#11": "A null character is kept in values, as in markdown-rs, and replaced with U+FFFD only in HTML; the JavaScript packages replace it in values too.", - "markdown-rs/hard_break_trailing#12": "A null character is kept in values, as in markdown-rs, and replaced with U+FFFD only in HTML; the JavaScript packages replace it in values too.", - "markdown-rs/link_resource#72": "A null character is kept in values, as in markdown-rs, and replaced with U+FFFD only in HTML; the JavaScript packages replace it in values too.", - "markdown-rs/misc_zero#2": "A null character is kept in values, as in markdown-rs, and replaced with U+FFFD only in HTML; the JavaScript packages replace it in values too.", - "micromark-extension-mdx-expression/micromark-extension-mdx-expression@3.0.1#5": "A null character is kept in values, as in markdown-rs, and replaced with U+FFFD only in HTML; the JavaScript packages replace it in values too.", - "markdown-rs/autolink#14": "GFM autolink literals: an email after a backslash escape is text here, as in the CommonMark spec.", - "markdown-rs/gfm_autolink_literal#28": "GFM autolink literals: www after punctuation other than the ASCII ones of the GFM spec is text here.", - "markdown-rs/gfm_autolink_literal#33": "GFM autolink literals: the text of a `mailto:` or `xmpp:` link includes the scheme here, as in the GFM spec.", - "markdown-rs/code_fenced#36": "Blank lines at the end of an unclosed fence in a quote are one fewer in the value of the official packages.", - "markdown-rs/math_flow#35": "Blank lines at the end of an unclosed fence in a quote are one fewer in the value of the official packages.", - "markdown-rs/code_indented#23": "The lines of inline HTML lose all their indent here, as in CommonMark, and all but a space in the official packages.", - "markdown-rs/gfm_task_list_item#6": "The text after a checkbox that ends its line starts with a line ending in the JavaScript packages, and not in markdown-rs.", - "markdown-rs/misc_tabs#15": "The info string of a fence is trimmed here, as in CommonMark, and the official packages keep the trailing whitespace as the meta.", - "markdown-rs/misc_tabs#16": "The info string of a fence is trimmed here, as in CommonMark, and the official packages keep the trailing whitespace as the meta.", - "markdown-rs/html_flow#147": "The official packages do not start HTML flow on a line that continues a paragraph in a quote, which CommonMark does.", - "markdown-rs/link_reference#46": "A reference followed by an empty label is a shortcut reference here, as in CommonMark, and text in the official packages.", - "random-markdown-1#90": "The official packages match a delimiter run that is partly used without the rule of three, which CommonMark applies to the length of the whole run.", - "random-markdown-1#200": "The official packages match a delimiter run that is partly used without the rule of three, which CommonMark applies to the length of the whole run.", - "random-markdown-1#298": "The official packages match a delimiter run that is partly used without the rule of three, which CommonMark applies to the length of the whole run.", - "random-mdx-1#102": "The official packages match a delimiter run that is partly used without the rule of three, which CommonMark applies to the length of the whole run.", - "random-markdown-1#162": "The official packages mark a list as spread when a blank line follows its last item, before a list with another marker.", - "random-markdown-1#205": "The official packages mark a list as spread when a blank line follows its last item, before a list with another marker." -} \ No newline at end of file diff --git a/crates/mq-markdown/tests/mdast_official/package-lock.json b/crates/mq-markdown/tests/mdast_official/package-lock.json deleted file mode 100644 index 45f0b43d3..000000000 --- a/crates/mq-markdown/tests/mdast_official/package-lock.json +++ /dev/null @@ -1,1579 +0,0 @@ -{ - "name": "mdast_official", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "dependencies": { - "mdast-util-from-markdown": "2.1.0", - "mdast-util-frontmatter": "^2.0.1", - "mdast-util-gfm": "^3.1.0", - "mdast-util-math": "^3.0.0", - "mdast-util-mdx": "3.0.0", - "micromark-extension-frontmatter": "^2.0.0", - "micromark-extension-gfm": "^3.0.0", - "micromark-extension-math": "^3.1.0", - "micromark-extension-mdxjs": "3.0.0" - } - }, - "node_modules/@types/debug": { - "version": "4.1.13", - "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.13.tgz", - "integrity": "sha512-KSVgmQmzMwPlmtljOomayoR89W4FynCAi3E8PPs7vmDVPe84hT+vGPKkJfThkmXs0x0jAaa9U8uW8bbfyS2fWw==", - "license": "MIT", - "dependencies": { - "@types/ms": "*" - } - }, - "node_modules/@types/estree": { - "version": "1.0.9", - "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz", - "integrity": "sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==", - "license": "MIT" - }, - "node_modules/@types/estree-jsx": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/@types/estree-jsx/-/estree-jsx-1.0.5.tgz", - "integrity": "sha512-52CcUVNFyfb1A2ALocQw/Dd1BQFNmSdkuC3BkZ6iqhdMfQz7JWOFRuJFloOzjk+6WijU56m9oKXFAXc7o3Towg==", - "license": "MIT", - "dependencies": { - "@types/estree": "*" - } - }, - "node_modules/@types/hast": { - "version": "3.0.5", - "resolved": "https://registry.npmjs.org/@types/hast/-/hast-3.0.5.tgz", - "integrity": "sha512-rp/ezSWaD1m44dPKICGhiskI13nVr7qTloFwDa/IYkhhf5nzwP+zIQcIJh3WIFSBOy/H1PzB40jPjMDksN4F+g==", - "license": "MIT", - "dependencies": { - "@types/unist": "*" - } - }, - "node_modules/@types/katex": { - "version": "0.16.8", - "resolved": "https://registry.npmjs.org/@types/katex/-/katex-0.16.8.tgz", - "integrity": "sha512-trgaNyfU+Xh2Tc+ABIb44a5AYUpicB3uwirOioeOkNPPbmgRNtcWyDeeFRzjPZENO9Vq8gvVqfhaaXWLlevVwg==", - "license": "MIT" - }, - "node_modules/@types/mdast": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/@types/mdast/-/mdast-4.0.4.tgz", - "integrity": "sha512-kGaNbPh1k7AFzgpud/gMdvIm5xuECykRR+JnWKQno9TAXVa6WIVCGTPvYGekIDL4uwCZQSYbUxNBSb1aUo79oA==", - "license": "MIT", - "dependencies": { - "@types/unist": "*" - } - }, - "node_modules/@types/ms": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/@types/ms/-/ms-2.1.0.tgz", - "integrity": "sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA==", - "license": "MIT" - }, - "node_modules/@types/unist": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.3.tgz", - "integrity": "sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q==", - "license": "MIT" - }, - "node_modules/acorn": { - "version": "8.19.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.19.0.tgz", - "integrity": "sha512-oJlA3XiRm7Cyk6qFD2Jn8ak9B7jSy0qy00ADO3+8dpT0LSjFihQYv4C02LFCSYJV3Q37xYwwRy5m+IpIiUzqWw==", - "license": "MIT", - "peer": true, - "bin": { - "acorn": "bin/acorn" - }, - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/acorn-jsx": { - "version": "5.3.2", - "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", - "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", - "license": "MIT", - "peerDependencies": { - "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" - } - }, - "node_modules/ccount": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/ccount/-/ccount-2.0.1.tgz", - "integrity": "sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg==", - "license": "MIT", - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } - }, - "node_modules/character-entities": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/character-entities/-/character-entities-2.0.2.tgz", - "integrity": "sha512-shx7oQ0Awen/BRIdkjkvz54PnEEI/EjwXDSIZp86/KKdbafHh1Df/RYGBhn4hbe2+uKC9FnT5UCEdyPz3ai9hQ==", - "license": "MIT", - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } - }, - "node_modules/character-entities-html4": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/character-entities-html4/-/character-entities-html4-2.1.0.tgz", - "integrity": "sha512-1v7fgQRj6hnSwFpq1Eu0ynr/CDEw0rXo2B61qXrLNdHZmPKgb7fqS1a2JwF0rISo9q77jDI8VMEHoApn8qDoZA==", - "license": "MIT", - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } - }, - "node_modules/character-entities-legacy": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/character-entities-legacy/-/character-entities-legacy-3.0.0.tgz", - "integrity": "sha512-RpPp0asT/6ufRm//AJVwpViZbGM/MkjQFxJccQRHmISF/22NBtsHqAWmL+/pmkPWoIUJdWyeVleTl1wydHATVQ==", - "license": "MIT", - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } - }, - "node_modules/character-reference-invalid": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/character-reference-invalid/-/character-reference-invalid-2.0.1.tgz", - "integrity": "sha512-iBZ4F4wRbyORVsu0jPV7gXkOsGYjGHPmAyv+HiHG8gi5PtC9KI2j1+v8/tlibRvjoWX027ypmG/n0HtO5t7unw==", - "license": "MIT", - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } - }, - "node_modules/commander": { - "version": "8.3.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-8.3.0.tgz", - "integrity": "sha512-OkTL9umf+He2DZkUq8f8J9of7yL6RJKI24dVITBmNfZBmri9zYZQrKkuXiKhyfPSu8tUhnVBB1iKXevvnlR4Ww==", - "license": "MIT", - "engines": { - "node": ">= 12" - } - }, - "node_modules/debug": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", - "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/decode-named-character-reference": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/decode-named-character-reference/-/decode-named-character-reference-1.3.0.tgz", - "integrity": "sha512-GtpQYB283KrPp6nRw50q3U9/VfOutZOe103qlN7BPP6Ad27xYnOIWv4lPzo8HCAL+mMZofJ9KEy30fq6MfaK6Q==", - "license": "MIT", - "dependencies": { - "character-entities": "^2.0.0" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } - }, - "node_modules/dequal": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", - "integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==", - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/devlop": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/devlop/-/devlop-1.1.0.tgz", - "integrity": "sha512-RWmIqhcFf1lRYBvNmr7qTNuyCt/7/ns2jbpp1+PalgE/rDQcBT0fioSMUpJ93irlUhC5hrg4cYqe6U+0ImW0rA==", - "license": "MIT", - "dependencies": { - "dequal": "^2.0.0" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } - }, - "node_modules/escape-string-regexp": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-5.0.0.tgz", - "integrity": "sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw==", - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/estree-util-is-identifier-name": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/estree-util-is-identifier-name/-/estree-util-is-identifier-name-3.0.0.tgz", - "integrity": "sha512-hFtqIDZTIUZ9BXLb8y4pYGyk6+wekIivNVTcmvk8NoOh+VeRn5y6cEHzbURrWbfp1fIqdVipilzj+lfaadNZmg==", - "license": "MIT", - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/estree-util-visit": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/estree-util-visit/-/estree-util-visit-2.0.0.tgz", - "integrity": "sha512-m5KgiH85xAhhW8Wta0vShLcUvOsh3LLPI2YVwcbio1l7E09NTLL1EyMZFM1OyWowoH0skScNbhOPl4kcBgzTww==", - "license": "MIT", - "dependencies": { - "@types/estree-jsx": "^1.0.0", - "@types/unist": "^3.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/fault": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/fault/-/fault-2.0.1.tgz", - "integrity": "sha512-WtySTkS4OKev5JtpHXnib4Gxiurzh5NCGvWrFaZ34m6JehfTUhKZvn9njTfw48t6JumVQOmrKqpmGcdwxnhqBQ==", - "license": "MIT", - "dependencies": { - "format": "^0.2.0" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } - }, - "node_modules/format": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/format/-/format-0.2.2.tgz", - "integrity": "sha512-wzsgA6WOq+09wrU1tsJ09udeR/YZRaeArL9e1wPbFg3GG2yDnC2ldKpxs4xunpFF9DgqCqOIra3bc1HWrJ37Ww==", - "engines": { - "node": ">=0.4.x" - } - }, - "node_modules/is-alphabetical": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/is-alphabetical/-/is-alphabetical-2.0.1.tgz", - "integrity": "sha512-FWyyY60MeTNyeSRpkM2Iry0G9hpr7/9kD40mD/cGQEuilcZYS4okz8SN2Q6rLCJ8gbCt6fN+rC+6tMGS99LaxQ==", - "license": "MIT", - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } - }, - "node_modules/is-alphanumerical": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/is-alphanumerical/-/is-alphanumerical-2.0.1.tgz", - "integrity": "sha512-hmbYhX/9MUMF5uh7tOXyK/n0ZvWpad5caBA17GsC6vyuCqaWliRG5K1qS9inmUhEMaOBIW7/whAnSwveW/LtZw==", - "license": "MIT", - "dependencies": { - "is-alphabetical": "^2.0.0", - "is-decimal": "^2.0.0" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } - }, - "node_modules/is-decimal": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/is-decimal/-/is-decimal-2.0.1.tgz", - "integrity": "sha512-AAB9hiomQs5DXWcRB1rqsxGUstbRroFOPPVAomNk/3XHR5JyEZChOyTWe2oayKnsSsr/kcGqF+z6yuH6HHpN0A==", - "license": "MIT", - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } - }, - "node_modules/is-hexadecimal": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/is-hexadecimal/-/is-hexadecimal-2.0.1.tgz", - "integrity": "sha512-DgZQp241c8oO6cA1SbTEWiXeoxV42vlcJxgH+B3hi1AiqqKruZR3ZGF8In3fj4+/y/7rHvlOZLZtgJ/4ttYGZg==", - "license": "MIT", - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } - }, - "node_modules/katex": { - "version": "0.16.47", - "resolved": "https://registry.npmjs.org/katex/-/katex-0.16.47.tgz", - "integrity": "sha512-Eeo8Ys1doU1z+x8AZsPpQu+p/QcZBI5PeOo7QGQdy2x2m0MU/hYagBbGOmXwr5KVbEfVuWv9LpnQWeehogurjg==", - "funding": [ - "https://opencollective.com/katex", - "https://github.com/sponsors/katex" - ], - "license": "MIT", - "dependencies": { - "commander": "^8.3.0" - }, - "bin": { - "katex": "cli.js" - } - }, - "node_modules/longest-streak": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/longest-streak/-/longest-streak-3.1.0.tgz", - "integrity": "sha512-9Ri+o0JYgehTaVBBDoMqIl8GXtbWg711O3srftcHhZ0dqnETqLaoIK0x17fUw9rFSlK/0NlsKe0Ahhyl5pXE2g==", - "license": "MIT", - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } - }, - "node_modules/markdown-table": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/markdown-table/-/markdown-table-3.0.4.tgz", - "integrity": "sha512-wiYz4+JrLyb/DqW2hkFJxP7Vd7JuTDm77fvbM8VfEQdmSMqcImWeeRbHwZjBjIFki/VaMK2BhFi7oUUZeM5bqw==", - "license": "MIT", - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } - }, - "node_modules/mdast-util-find-and-replace": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/mdast-util-find-and-replace/-/mdast-util-find-and-replace-3.0.3.tgz", - "integrity": "sha512-xPgpDNl0/OXHsI7TlaIs22lWnH3KlpvZdXG1ET/m/YT2Hhdkx8lJV4kLphE6l9simyM4KPDFBwas08BgEob6Jw==", - "license": "MIT", - "dependencies": { - "@types/mdast": "^4.0.0", - "escape-string-regexp": "^5.0.0", - "unist-util-is": "^6.0.0", - "unist-util-visit-parents": "^6.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/mdast-util-from-markdown": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/mdast-util-from-markdown/-/mdast-util-from-markdown-2.1.0.tgz", - "integrity": "sha512-KRLgmPaP2H1ul2bpoJkOS+9pAcDAAj9Nu1mqLnh8NLeJG5tlvj42TH80PIGBpROWlRXLk0lDrWNuZCFy/tsB4A==", - "license": "MIT", - "dependencies": { - "@types/mdast": "^4.0.0", - "@types/unist": "^3.0.0", - "decode-named-character-reference": "^1.0.0", - "devlop": "^1.0.0", - "mdast-util-to-string": "^4.0.0", - "micromark": "^4.0.0", - "micromark-util-decode-numeric-character-reference": "^2.0.0", - "micromark-util-decode-string": "^2.0.0", - "micromark-util-normalize-identifier": "^2.0.0", - "micromark-util-symbol": "^2.0.0", - "micromark-util-types": "^2.0.0", - "unist-util-stringify-position": "^4.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/mdast-util-frontmatter": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/mdast-util-frontmatter/-/mdast-util-frontmatter-2.0.1.tgz", - "integrity": "sha512-LRqI9+wdgC25P0URIJY9vwocIzCcksduHQ9OF2joxQoyTNVduwLAFUzjoopuRJbJAReaKrNQKAZKL3uCMugWJA==", - "license": "MIT", - "dependencies": { - "@types/mdast": "^4.0.0", - "devlop": "^1.0.0", - "escape-string-regexp": "^5.0.0", - "mdast-util-from-markdown": "^2.0.0", - "mdast-util-to-markdown": "^2.0.0", - "micromark-extension-frontmatter": "^2.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/mdast-util-gfm": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/mdast-util-gfm/-/mdast-util-gfm-3.1.0.tgz", - "integrity": "sha512-0ulfdQOM3ysHhCJ1p06l0b0VKlhU0wuQs3thxZQagjcjPrlFRqY215uZGHHJan9GEAXd9MbfPjFJz+qMkVR6zQ==", - "license": "MIT", - "dependencies": { - "mdast-util-from-markdown": "^2.0.0", - "mdast-util-gfm-autolink-literal": "^2.0.0", - "mdast-util-gfm-footnote": "^2.0.0", - "mdast-util-gfm-strikethrough": "^2.0.0", - "mdast-util-gfm-table": "^2.0.0", - "mdast-util-gfm-task-list-item": "^2.0.0", - "mdast-util-to-markdown": "^2.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/mdast-util-gfm-autolink-literal": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/mdast-util-gfm-autolink-literal/-/mdast-util-gfm-autolink-literal-2.0.1.tgz", - "integrity": "sha512-5HVP2MKaP6L+G6YaxPNjuL0BPrq9orG3TsrZ9YXbA3vDw/ACI4MEsnoDpn6ZNm7GnZgtAcONJyPhOP8tNJQavQ==", - "license": "MIT", - "dependencies": { - "@types/mdast": "^4.0.0", - "ccount": "^2.0.0", - "devlop": "^1.0.0", - "mdast-util-find-and-replace": "^3.0.0", - "micromark-util-character": "^2.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/mdast-util-gfm-footnote": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/mdast-util-gfm-footnote/-/mdast-util-gfm-footnote-2.1.0.tgz", - "integrity": "sha512-sqpDWlsHn7Ac9GNZQMeUzPQSMzR6Wv0WKRNvQRg0KqHh02fpTz69Qc1QSseNX29bhz1ROIyNyxExfawVKTm1GQ==", - "license": "MIT", - "dependencies": { - "@types/mdast": "^4.0.0", - "devlop": "^1.1.0", - "mdast-util-from-markdown": "^2.0.0", - "mdast-util-to-markdown": "^2.0.0", - "micromark-util-normalize-identifier": "^2.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/mdast-util-gfm-strikethrough": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/mdast-util-gfm-strikethrough/-/mdast-util-gfm-strikethrough-2.0.1.tgz", - "integrity": "sha512-OuJHqvr455pwu2OaOrir7dbzqs4jlWKOtlu9L6GjcIjjQwNyw02VntQsr/Ek6/A13vgnXfUhBekWRUP1OkNivw==", - "license": "MIT", - "dependencies": { - "@types/mdast": "^4.0.0", - "mdast-util-from-markdown": "^2.0.0", - "mdast-util-to-markdown": "^2.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/mdast-util-gfm-table": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/mdast-util-gfm-table/-/mdast-util-gfm-table-2.0.0.tgz", - "integrity": "sha512-78UEvebzz/rJIxLvE7ZtDd/vIQ0RHv+3Mh5DR96p7cS7HsBhYIICDBCu8csTNWNO6tBWfqXPWekRuj2FNOGOZg==", - "license": "MIT", - "dependencies": { - "@types/mdast": "^4.0.0", - "devlop": "^1.0.0", - "markdown-table": "^3.0.0", - "mdast-util-from-markdown": "^2.0.0", - "mdast-util-to-markdown": "^2.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/mdast-util-gfm-task-list-item": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/mdast-util-gfm-task-list-item/-/mdast-util-gfm-task-list-item-2.0.0.tgz", - "integrity": "sha512-IrtvNvjxC1o06taBAVJznEnkiHxLFTzgonUdy8hzFVeDun0uTjxxrRGVaNFqkU1wJR3RBPEfsxmU6jDWPofrTQ==", - "license": "MIT", - "dependencies": { - "@types/mdast": "^4.0.0", - "devlop": "^1.0.0", - "mdast-util-from-markdown": "^2.0.0", - "mdast-util-to-markdown": "^2.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/mdast-util-math": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/mdast-util-math/-/mdast-util-math-3.0.0.tgz", - "integrity": "sha512-Tl9GBNeG/AhJnQM221bJR2HPvLOSnLE/T9cJI9tlc6zwQk2nPk/4f0cHkOdEixQPC/j8UtKDdITswvLAy1OZ1w==", - "license": "MIT", - "dependencies": { - "@types/hast": "^3.0.0", - "@types/mdast": "^4.0.0", - "devlop": "^1.0.0", - "longest-streak": "^3.0.0", - "mdast-util-from-markdown": "^2.0.0", - "mdast-util-to-markdown": "^2.1.0", - "unist-util-remove-position": "^5.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/mdast-util-mdx": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/mdast-util-mdx/-/mdast-util-mdx-3.0.0.tgz", - "integrity": "sha512-JfbYLAW7XnYTTbUsmpu0kdBUVe+yKVJZBItEjwyYJiDJuZ9w4eeaqks4HQO+R7objWgS2ymV60GYpI14Ug554w==", - "license": "MIT", - "dependencies": { - "mdast-util-from-markdown": "^2.0.0", - "mdast-util-mdx-expression": "^2.0.0", - "mdast-util-mdx-jsx": "^3.0.0", - "mdast-util-mdxjs-esm": "^2.0.0", - "mdast-util-to-markdown": "^2.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/mdast-util-mdx-expression": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/mdast-util-mdx-expression/-/mdast-util-mdx-expression-2.0.1.tgz", - "integrity": "sha512-J6f+9hUp+ldTZqKRSg7Vw5V6MqjATc+3E4gf3CFNcuZNWD8XdyI6zQ8GqH7f8169MM6P7hMBRDVGnn7oHB9kXQ==", - "license": "MIT", - "dependencies": { - "@types/estree-jsx": "^1.0.0", - "@types/hast": "^3.0.0", - "@types/mdast": "^4.0.0", - "devlop": "^1.0.0", - "mdast-util-from-markdown": "^2.0.0", - "mdast-util-to-markdown": "^2.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/mdast-util-mdx-jsx": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/mdast-util-mdx-jsx/-/mdast-util-mdx-jsx-3.2.0.tgz", - "integrity": "sha512-lj/z8v0r6ZtsN/cGNNtemmmfoLAFZnjMbNyLzBafjzikOM+glrjNHPlf6lQDOTccj9n5b0PPihEBbhneMyGs1Q==", - "license": "MIT", - "dependencies": { - "@types/estree-jsx": "^1.0.0", - "@types/hast": "^3.0.0", - "@types/mdast": "^4.0.0", - "@types/unist": "^3.0.0", - "ccount": "^2.0.0", - "devlop": "^1.1.0", - "mdast-util-from-markdown": "^2.0.0", - "mdast-util-to-markdown": "^2.0.0", - "parse-entities": "^4.0.0", - "stringify-entities": "^4.0.0", - "unist-util-stringify-position": "^4.0.0", - "vfile-message": "^4.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/mdast-util-mdxjs-esm": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/mdast-util-mdxjs-esm/-/mdast-util-mdxjs-esm-2.0.1.tgz", - "integrity": "sha512-EcmOpxsZ96CvlP03NghtH1EsLtr0n9Tm4lPUJUBccV9RwUOneqSycg19n5HGzCf+10LozMRSObtVr3ee1WoHtg==", - "license": "MIT", - "dependencies": { - "@types/estree-jsx": "^1.0.0", - "@types/hast": "^3.0.0", - "@types/mdast": "^4.0.0", - "devlop": "^1.0.0", - "mdast-util-from-markdown": "^2.0.0", - "mdast-util-to-markdown": "^2.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/mdast-util-phrasing": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/mdast-util-phrasing/-/mdast-util-phrasing-4.1.0.tgz", - "integrity": "sha512-TqICwyvJJpBwvGAMZjj4J2n0X8QWp21b9l0o7eXyVJ25YNWYbJDVIyD1bZXE6WtV6RmKJVYmQAKWa0zWOABz2w==", - "license": "MIT", - "dependencies": { - "@types/mdast": "^4.0.0", - "unist-util-is": "^6.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/mdast-util-to-markdown": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/mdast-util-to-markdown/-/mdast-util-to-markdown-2.2.0.tgz", - "integrity": "sha512-2Jn/ADrHFFyyniBMi0bncTbJqU78K3/TEYvzvwrWWzD9MYPUa/Gm4r3nOeKFQg/PDM5BsbDYdc8ackW4NTY6NA==", - "license": "MIT", - "dependencies": { - "@types/mdast": "^4.0.0", - "@types/unist": "^3.0.0", - "longest-streak": "^3.0.0", - "mdast-util-phrasing": "^4.0.0", - "mdast-util-to-string": "^4.0.0", - "micromark-util-character": "^2.0.0", - "micromark-util-classify-character": "^2.0.0", - "micromark-util-decode-string": "^2.0.0", - "micromark-util-html-tag-name": "^2.0.0", - "unist-util-visit": "^5.0.0", - "zwitch": "^2.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/mdast-util-to-string": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/mdast-util-to-string/-/mdast-util-to-string-4.0.0.tgz", - "integrity": "sha512-0H44vDimn51F0YwvxSJSm0eCDOJTRlmN0R1yBh4HLj9wiV1Dn0QoXGbvFAWj2hSItVTlCmBF1hqKlIyUBVFLPg==", - "license": "MIT", - "dependencies": { - "@types/mdast": "^4.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/micromark": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/micromark/-/micromark-4.0.3.tgz", - "integrity": "sha512-oGYfQzHSG5dOMovQcJ3fyTmZlWAWpi0XA0sJwJs+i6OT88o1+Jtw/8z0CmdowSLxGhhFK89rQ/oXph/wN02PNw==", - "funding": [ - { - "type": "GitHub Sponsors", - "url": "https://github.com/sponsors/unifiedjs" - }, - { - "type": "OpenCollective", - "url": "https://opencollective.com/unified" - } - ], - "license": "MIT", - "dependencies": { - "@types/debug": "^4.0.0", - "debug": "^4.0.0", - "decode-named-character-reference": "^1.0.0", - "devlop": "^1.0.0", - "micromark-core-commonmark": "^2.0.0", - "micromark-factory-space": "^2.0.0", - "micromark-util-character": "^2.0.0", - "micromark-util-chunked": "^2.0.0", - "micromark-util-combine-extensions": "^2.0.0", - "micromark-util-decode-numeric-character-reference": "^2.0.0", - "micromark-util-edit-map": "^1.0.0", - "micromark-util-encode": "^2.0.0", - "micromark-util-normalize-identifier": "^2.0.0", - "micromark-util-resolve-all": "^2.0.0", - "micromark-util-sanitize-uri": "^2.0.0", - "micromark-util-subtokenize": "^2.0.0", - "micromark-util-symbol": "^2.0.0", - "micromark-util-types": "^2.0.0" - } - }, - "node_modules/micromark-core-commonmark": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/micromark-core-commonmark/-/micromark-core-commonmark-2.0.4.tgz", - "integrity": "sha512-wxEeE8v8XVvOrxn1TZj74qYhAtTQsSqufHVS3uNVyT1MupXxhyaHk8/9xDNPh9BmL77QNDgWTkZ3RsYHB2ry7w==", - "funding": [ - { - "type": "GitHub Sponsors", - "url": "https://github.com/sponsors/unifiedjs" - }, - { - "type": "OpenCollective", - "url": "https://opencollective.com/unified" - } - ], - "license": "MIT", - "dependencies": { - "decode-named-character-reference": "^1.0.0", - "devlop": "^1.0.0", - "micromark-factory-destination": "^2.0.0", - "micromark-factory-label": "^2.0.0", - "micromark-factory-space": "^2.1.0", - "micromark-factory-title": "^2.0.0", - "micromark-factory-whitespace": "^2.0.0", - "micromark-util-character": "^2.0.0", - "micromark-util-chunked": "^2.0.0", - "micromark-util-classify-character": "^2.0.0", - "micromark-util-edit-map": "^1.0.0", - "micromark-util-html-tag-name": "^2.0.0", - "micromark-util-normalize-identifier": "^2.0.0", - "micromark-util-resolve-all": "^2.0.0", - "micromark-util-subtokenize": "^2.0.0", - "micromark-util-symbol": "^2.0.0", - "micromark-util-types": "^2.0.3" - } - }, - "node_modules/micromark-extension-frontmatter": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/micromark-extension-frontmatter/-/micromark-extension-frontmatter-2.0.0.tgz", - "integrity": "sha512-C4AkuM3dA58cgZha7zVnuVxBhDsbttIMiytjgsM2XbHAB2faRVaHRle40558FBN+DJcrLNCoqG5mlrpdU4cRtg==", - "license": "MIT", - "dependencies": { - "fault": "^2.0.0", - "micromark-util-character": "^2.0.0", - "micromark-util-symbol": "^2.0.0", - "micromark-util-types": "^2.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/micromark-extension-gfm": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/micromark-extension-gfm/-/micromark-extension-gfm-3.0.0.tgz", - "integrity": "sha512-vsKArQsicm7t0z2GugkCKtZehqUm31oeGBV/KVSorWSy8ZlNAv7ytjFhvaryUiCUJYqs+NoE6AFhpQvBTM6Q4w==", - "license": "MIT", - "dependencies": { - "micromark-extension-gfm-autolink-literal": "^2.0.0", - "micromark-extension-gfm-footnote": "^2.0.0", - "micromark-extension-gfm-strikethrough": "^2.0.0", - "micromark-extension-gfm-table": "^2.0.0", - "micromark-extension-gfm-tagfilter": "^2.0.0", - "micromark-extension-gfm-task-list-item": "^2.0.0", - "micromark-util-combine-extensions": "^2.0.0", - "micromark-util-types": "^2.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/micromark-extension-gfm-autolink-literal": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/micromark-extension-gfm-autolink-literal/-/micromark-extension-gfm-autolink-literal-2.1.0.tgz", - "integrity": "sha512-oOg7knzhicgQ3t4QCjCWgTmfNhvQbDDnJeVu9v81r7NltNCVmhPy1fJRX27pISafdjL+SVc4d3l48Gb6pbRypw==", - "license": "MIT", - "dependencies": { - "micromark-util-character": "^2.0.0", - "micromark-util-sanitize-uri": "^2.0.0", - "micromark-util-symbol": "^2.0.0", - "micromark-util-types": "^2.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/micromark-extension-gfm-footnote": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/micromark-extension-gfm-footnote/-/micromark-extension-gfm-footnote-2.1.0.tgz", - "integrity": "sha512-/yPhxI1ntnDNsiHtzLKYnE3vf9JZ6cAisqVDauhp4CEHxlb4uoOTxOCJ+9s51bIB8U1N1FJ1RXOKTIlD5B/gqw==", - "license": "MIT", - "dependencies": { - "devlop": "^1.0.0", - "micromark-core-commonmark": "^2.0.0", - "micromark-factory-space": "^2.0.0", - "micromark-util-character": "^2.0.0", - "micromark-util-normalize-identifier": "^2.0.0", - "micromark-util-sanitize-uri": "^2.0.0", - "micromark-util-symbol": "^2.0.0", - "micromark-util-types": "^2.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/micromark-extension-gfm-strikethrough": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/micromark-extension-gfm-strikethrough/-/micromark-extension-gfm-strikethrough-2.1.0.tgz", - "integrity": "sha512-ADVjpOOkjz1hhkZLlBiYA9cR2Anf8F4HqZUO6e5eDcPQd0Txw5fxLzzxnEkSkfnD0wziSGiv7sYhk/ktvbf1uw==", - "license": "MIT", - "dependencies": { - "devlop": "^1.0.0", - "micromark-util-chunked": "^2.0.0", - "micromark-util-classify-character": "^2.0.0", - "micromark-util-resolve-all": "^2.0.0", - "micromark-util-symbol": "^2.0.0", - "micromark-util-types": "^2.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/micromark-extension-gfm-table": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/micromark-extension-gfm-table/-/micromark-extension-gfm-table-2.1.2.tgz", - "integrity": "sha512-pRzm4kDTu0MjlmBkxmS9yYhw60nncfcEwu9NNdPFSQEFXS95ZKyIIyTSHu/o3ReBUrLKYEq+7YaXCRn/bPB4MA==", - "license": "MIT", - "dependencies": { - "devlop": "^1.0.0", - "micromark-factory-space": "^2.0.0", - "micromark-util-character": "^2.0.0", - "micromark-util-symbol": "^2.0.0", - "micromark-util-types": "^2.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/micromark-extension-gfm-tagfilter": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/micromark-extension-gfm-tagfilter/-/micromark-extension-gfm-tagfilter-2.0.0.tgz", - "integrity": "sha512-xHlTOmuCSotIA8TW1mDIM6X2O1SiX5P9IuDtqGonFhEK0qgRI4yeC6vMxEV2dgyr2TiD+2PQ10o+cOhdVAcwfg==", - "license": "MIT", - "dependencies": { - "micromark-util-types": "^2.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/micromark-extension-gfm-task-list-item": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/micromark-extension-gfm-task-list-item/-/micromark-extension-gfm-task-list-item-2.1.0.tgz", - "integrity": "sha512-qIBZhqxqI6fjLDYFTBIa4eivDMnP+OZqsNwmQ3xNLE4Cxwc+zfQEfbs6tzAo2Hjq+bh6q5F+Z8/cksrLFYWQQw==", - "license": "MIT", - "dependencies": { - "devlop": "^1.0.0", - "micromark-factory-space": "^2.0.0", - "micromark-util-character": "^2.0.0", - "micromark-util-symbol": "^2.0.0", - "micromark-util-types": "^2.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/micromark-extension-math": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/micromark-extension-math/-/micromark-extension-math-3.1.0.tgz", - "integrity": "sha512-lvEqd+fHjATVs+2v/8kg9i5Q0AP2k85H0WUOwpIVvUML8BapsMvh1XAogmQjOCsLpoKRCVQqEkQBB3NhVBcsOg==", - "license": "MIT", - "dependencies": { - "@types/katex": "^0.16.0", - "devlop": "^1.0.0", - "katex": "^0.16.0", - "micromark-factory-space": "^2.0.0", - "micromark-util-character": "^2.0.0", - "micromark-util-symbol": "^2.0.0", - "micromark-util-types": "^2.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/micromark-extension-mdx-expression": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/micromark-extension-mdx-expression/-/micromark-extension-mdx-expression-3.0.1.tgz", - "integrity": "sha512-dD/ADLJ1AeMvSAKBwO22zG22N4ybhe7kFIZ3LsDI0GlsNr2A3KYxb0LdC1u5rj4Nw+CHKY0RVdnHX8vj8ejm4Q==", - "funding": [ - { - "type": "GitHub Sponsors", - "url": "https://github.com/sponsors/unifiedjs" - }, - { - "type": "OpenCollective", - "url": "https://opencollective.com/unified" - } - ], - "license": "MIT", - "dependencies": { - "@types/estree": "^1.0.0", - "devlop": "^1.0.0", - "micromark-factory-mdx-expression": "^2.0.0", - "micromark-factory-space": "^2.0.0", - "micromark-util-character": "^2.0.0", - "micromark-util-events-to-acorn": "^2.0.0", - "micromark-util-symbol": "^2.0.0", - "micromark-util-types": "^2.0.0" - } - }, - "node_modules/micromark-extension-mdx-jsx": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/micromark-extension-mdx-jsx/-/micromark-extension-mdx-jsx-3.0.2.tgz", - "integrity": "sha512-e5+q1DjMh62LZAJOnDraSSbDMvGJ8x3cbjygy2qFEi7HCeUT4BDKCvMozPozcD6WmOt6sVvYDNBKhFSz3kjOVQ==", - "license": "MIT", - "dependencies": { - "@types/estree": "^1.0.0", - "devlop": "^1.0.0", - "estree-util-is-identifier-name": "^3.0.0", - "micromark-factory-mdx-expression": "^2.0.0", - "micromark-factory-space": "^2.0.0", - "micromark-util-character": "^2.0.0", - "micromark-util-events-to-acorn": "^2.0.0", - "micromark-util-symbol": "^2.0.0", - "micromark-util-types": "^2.0.0", - "vfile-message": "^4.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/micromark-extension-mdx-md": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/micromark-extension-mdx-md/-/micromark-extension-mdx-md-2.0.0.tgz", - "integrity": "sha512-EpAiszsB3blw4Rpba7xTOUptcFeBFi+6PY8VnJ2hhimH+vCQDirWgsMpz7w1XcZE7LVrSAUGb9VJpG9ghlYvYQ==", - "license": "MIT", - "dependencies": { - "micromark-util-types": "^2.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/micromark-extension-mdxjs": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/micromark-extension-mdxjs/-/micromark-extension-mdxjs-3.0.0.tgz", - "integrity": "sha512-A873fJfhnJ2siZyUrJ31l34Uqwy4xIFmvPY1oj+Ean5PHcPBYzEsvqvWGaWcfEIr11O5Dlw3p2y0tZWpKHDejQ==", - "license": "MIT", - "dependencies": { - "acorn": "^8.0.0", - "acorn-jsx": "^5.0.0", - "micromark-extension-mdx-expression": "^3.0.0", - "micromark-extension-mdx-jsx": "^3.0.0", - "micromark-extension-mdx-md": "^2.0.0", - "micromark-extension-mdxjs-esm": "^3.0.0", - "micromark-util-combine-extensions": "^2.0.0", - "micromark-util-types": "^2.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/micromark-extension-mdxjs-esm": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/micromark-extension-mdxjs-esm/-/micromark-extension-mdxjs-esm-3.0.0.tgz", - "integrity": "sha512-DJFl4ZqkErRpq/dAPyeWp15tGrcrrJho1hKK5uBS70BCtfrIFg81sqcTVu3Ta+KD1Tk5vAtBNElWxtAa+m8K9A==", - "license": "MIT", - "dependencies": { - "@types/estree": "^1.0.0", - "devlop": "^1.0.0", - "micromark-core-commonmark": "^2.0.0", - "micromark-util-character": "^2.0.0", - "micromark-util-events-to-acorn": "^2.0.0", - "micromark-util-symbol": "^2.0.0", - "micromark-util-types": "^2.0.0", - "unist-util-position-from-estree": "^2.0.0", - "vfile-message": "^4.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/micromark-factory-destination": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/micromark-factory-destination/-/micromark-factory-destination-2.0.1.tgz", - "integrity": "sha512-Xe6rDdJlkmbFRExpTOmRj9N3MaWmbAgdpSrBQvCFqhezUn4AHqJHbaEnfbVYYiexVSs//tqOdY/DxhjdCiJnIA==", - "funding": [ - { - "type": "GitHub Sponsors", - "url": "https://github.com/sponsors/unifiedjs" - }, - { - "type": "OpenCollective", - "url": "https://opencollective.com/unified" - } - ], - "license": "MIT", - "dependencies": { - "micromark-util-character": "^2.0.0", - "micromark-util-symbol": "^2.0.0", - "micromark-util-types": "^2.0.0" - } - }, - "node_modules/micromark-factory-label": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/micromark-factory-label/-/micromark-factory-label-2.0.1.tgz", - "integrity": "sha512-VFMekyQExqIW7xIChcXn4ok29YE3rnuyveW3wZQWWqF4Nv9Wk5rgJ99KzPvHjkmPXF93FXIbBp6YdW3t71/7Vg==", - "funding": [ - { - "type": "GitHub Sponsors", - "url": "https://github.com/sponsors/unifiedjs" - }, - { - "type": "OpenCollective", - "url": "https://opencollective.com/unified" - } - ], - "license": "MIT", - "dependencies": { - "devlop": "^1.0.0", - "micromark-util-character": "^2.0.0", - "micromark-util-symbol": "^2.0.0", - "micromark-util-types": "^2.0.0" - } - }, - "node_modules/micromark-factory-mdx-expression": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/micromark-factory-mdx-expression/-/micromark-factory-mdx-expression-2.0.3.tgz", - "integrity": "sha512-kQnEtA3vzucU2BkrIa8/VaSAsP+EJ3CKOvhMuJgOEGg9KDC6OAY6nSnNDVRiVNRqj7Y4SlSzcStaH/5jge8JdQ==", - "funding": [ - { - "type": "GitHub Sponsors", - "url": "https://github.com/sponsors/unifiedjs" - }, - { - "type": "OpenCollective", - "url": "https://opencollective.com/unified" - } - ], - "license": "MIT", - "dependencies": { - "@types/estree": "^1.0.0", - "devlop": "^1.0.0", - "micromark-factory-space": "^2.0.0", - "micromark-util-character": "^2.0.0", - "micromark-util-events-to-acorn": "^2.0.0", - "micromark-util-symbol": "^2.0.0", - "micromark-util-types": "^2.0.0", - "unist-util-position-from-estree": "^2.0.0", - "vfile-message": "^4.0.0" - } - }, - "node_modules/micromark-factory-space": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/micromark-factory-space/-/micromark-factory-space-2.1.0.tgz", - "integrity": "sha512-fS8hnLIjnjvdQIj39Geug8wWsR0HrYZ43KShKxNfwT7t2LOHo/LbWZEzEaSOjwtjdCsjoE6syHhonEzW0zv0+Q==", - "funding": [ - { - "type": "GitHub Sponsors", - "url": "https://github.com/sponsors/unifiedjs" - }, - { - "type": "OpenCollective", - "url": "https://opencollective.com/unified" - } - ], - "license": "MIT", - "dependencies": { - "micromark-util-character": "^2.0.0", - "micromark-util-types": "^2.0.0" - } - }, - "node_modules/micromark-factory-title": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/micromark-factory-title/-/micromark-factory-title-2.0.1.tgz", - "integrity": "sha512-5bZ+3CjhAd9eChYTHsjy6TGxpOFSKgKKJPJxr293jTbfry2KDoWkhBb6TcPVB4NmzaPhMs1Frm9AZH7OD4Cjzw==", - "funding": [ - { - "type": "GitHub Sponsors", - "url": "https://github.com/sponsors/unifiedjs" - }, - { - "type": "OpenCollective", - "url": "https://opencollective.com/unified" - } - ], - "license": "MIT", - "dependencies": { - "micromark-factory-space": "^2.0.0", - "micromark-util-character": "^2.0.0", - "micromark-util-symbol": "^2.0.0", - "micromark-util-types": "^2.0.0" - } - }, - "node_modules/micromark-factory-whitespace": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/micromark-factory-whitespace/-/micromark-factory-whitespace-2.0.1.tgz", - "integrity": "sha512-Ob0nuZ3PKt/n0hORHyvoD9uZhr+Za8sFoP+OnMcnWK5lngSzALgQYKMr9RJVOWLqQYuyn6ulqGWSXdwf6F80lQ==", - "funding": [ - { - "type": "GitHub Sponsors", - "url": "https://github.com/sponsors/unifiedjs" - }, - { - "type": "OpenCollective", - "url": "https://opencollective.com/unified" - } - ], - "license": "MIT", - "dependencies": { - "micromark-factory-space": "^2.0.0", - "micromark-util-character": "^2.0.0", - "micromark-util-symbol": "^2.0.0", - "micromark-util-types": "^2.0.0" - } - }, - "node_modules/micromark-util-character": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/micromark-util-character/-/micromark-util-character-2.1.1.tgz", - "integrity": "sha512-wv8tdUTJ3thSFFFJKtpYKOYiGP2+v96Hvk4Tu8KpCAsTMs6yi+nVmGh1syvSCsaxz45J6Jbw+9DD6g97+NV67Q==", - "funding": [ - { - "type": "GitHub Sponsors", - "url": "https://github.com/sponsors/unifiedjs" - }, - { - "type": "OpenCollective", - "url": "https://opencollective.com/unified" - } - ], - "license": "MIT", - "dependencies": { - "micromark-util-symbol": "^2.0.0", - "micromark-util-types": "^2.0.0" - } - }, - "node_modules/micromark-util-chunked": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/micromark-util-chunked/-/micromark-util-chunked-2.0.1.tgz", - "integrity": "sha512-QUNFEOPELfmvv+4xiNg2sRYeS/P84pTW0TCgP5zc9FpXetHY0ab7SxKyAQCNCc1eK0459uoLI1y5oO5Vc1dbhA==", - "funding": [ - { - "type": "GitHub Sponsors", - "url": "https://github.com/sponsors/unifiedjs" - }, - { - "type": "OpenCollective", - "url": "https://opencollective.com/unified" - } - ], - "license": "MIT", - "dependencies": { - "micromark-util-symbol": "^2.0.0" - } - }, - "node_modules/micromark-util-classify-character": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/micromark-util-classify-character/-/micromark-util-classify-character-2.0.1.tgz", - "integrity": "sha512-K0kHzM6afW/MbeWYWLjoHQv1sgg2Q9EccHEDzSkxiP/EaagNzCm7T/WMKZ3rjMbvIpvBiZgwR3dKMygtA4mG1Q==", - "funding": [ - { - "type": "GitHub Sponsors", - "url": "https://github.com/sponsors/unifiedjs" - }, - { - "type": "OpenCollective", - "url": "https://opencollective.com/unified" - } - ], - "license": "MIT", - "dependencies": { - "micromark-util-character": "^2.0.0", - "micromark-util-symbol": "^2.0.0", - "micromark-util-types": "^2.0.0" - } - }, - "node_modules/micromark-util-combine-extensions": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/micromark-util-combine-extensions/-/micromark-util-combine-extensions-2.0.1.tgz", - "integrity": "sha512-OnAnH8Ujmy59JcyZw8JSbK9cGpdVY44NKgSM7E9Eh7DiLS2E9RNQf0dONaGDzEG9yjEl5hcqeIsj4hfRkLH/Bg==", - "funding": [ - { - "type": "GitHub Sponsors", - "url": "https://github.com/sponsors/unifiedjs" - }, - { - "type": "OpenCollective", - "url": "https://opencollective.com/unified" - } - ], - "license": "MIT", - "dependencies": { - "micromark-util-chunked": "^2.0.0", - "micromark-util-types": "^2.0.0" - } - }, - "node_modules/micromark-util-decode-numeric-character-reference": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/micromark-util-decode-numeric-character-reference/-/micromark-util-decode-numeric-character-reference-2.0.2.tgz", - "integrity": "sha512-ccUbYk6CwVdkmCQMyr64dXz42EfHGkPQlBj5p7YVGzq8I7CtjXZJrubAYezf7Rp+bjPseiROqe7G6foFd+lEuw==", - "funding": [ - { - "type": "GitHub Sponsors", - "url": "https://github.com/sponsors/unifiedjs" - }, - { - "type": "OpenCollective", - "url": "https://opencollective.com/unified" - } - ], - "license": "MIT", - "dependencies": { - "micromark-util-symbol": "^2.0.0" - } - }, - "node_modules/micromark-util-decode-string": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/micromark-util-decode-string/-/micromark-util-decode-string-2.0.1.tgz", - "integrity": "sha512-nDV/77Fj6eH1ynwscYTOsbK7rR//Uj0bZXBwJZRfaLEJ1iGBR6kIfNmlNqaqJf649EP0F3NWNdeJi03elllNUQ==", - "funding": [ - { - "type": "GitHub Sponsors", - "url": "https://github.com/sponsors/unifiedjs" - }, - { - "type": "OpenCollective", - "url": "https://opencollective.com/unified" - } - ], - "license": "MIT", - "dependencies": { - "decode-named-character-reference": "^1.0.0", - "micromark-util-character": "^2.0.0", - "micromark-util-decode-numeric-character-reference": "^2.0.0", - "micromark-util-symbol": "^2.0.0" - } - }, - "node_modules/micromark-util-edit-map": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/micromark-util-edit-map/-/micromark-util-edit-map-1.0.0.tgz", - "integrity": "sha512-Pa2ljlsEL6sVwFaYeyrOLSYbQt73JvGbPOYFq+9AElXiSnfI5Q4465RRZ1sHv7lDjDOnGRDDaqPXqClqWK/q1Q==", - "funding": [ - { - "type": "GitHub Sponsors", - "url": "https://github.com/sponsors/unifiedjs" - }, - { - "type": "OpenCollective", - "url": "https://opencollective.com/unified" - } - ], - "license": "MIT", - "dependencies": { - "micromark-util-types": "^2.0.0" - } - }, - "node_modules/micromark-util-encode": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/micromark-util-encode/-/micromark-util-encode-2.0.1.tgz", - "integrity": "sha512-c3cVx2y4KqUnwopcO9b/SCdo2O67LwJJ/UyqGfbigahfegL9myoEFoDYZgkT7f36T0bLrM9hZTAaAyH+PCAXjw==", - "funding": [ - { - "type": "GitHub Sponsors", - "url": "https://github.com/sponsors/unifiedjs" - }, - { - "type": "OpenCollective", - "url": "https://opencollective.com/unified" - } - ], - "license": "MIT" - }, - "node_modules/micromark-util-events-to-acorn": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/micromark-util-events-to-acorn/-/micromark-util-events-to-acorn-2.0.3.tgz", - "integrity": "sha512-jmsiEIiZ1n7X1Rr5k8wVExBQCg5jy4UXVADItHmNk1zkwEVhBuIUKRu3fqv+hs4nxLISi2DQGlqIOGiFxgbfHg==", - "funding": [ - { - "type": "GitHub Sponsors", - "url": "https://github.com/sponsors/unifiedjs" - }, - { - "type": "OpenCollective", - "url": "https://opencollective.com/unified" - } - ], - "license": "MIT", - "dependencies": { - "@types/estree": "^1.0.0", - "@types/unist": "^3.0.0", - "devlop": "^1.0.0", - "estree-util-visit": "^2.0.0", - "micromark-util-symbol": "^2.0.0", - "micromark-util-types": "^2.0.0", - "vfile-message": "^4.0.0" - } - }, - "node_modules/micromark-util-html-tag-name": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/micromark-util-html-tag-name/-/micromark-util-html-tag-name-2.0.1.tgz", - "integrity": "sha512-2cNEiYDhCWKI+Gs9T0Tiysk136SnR13hhO8yW6BGNyhOC4qYFnwF1nKfD3HFAIXA5c45RrIG1ub11GiXeYd1xA==", - "funding": [ - { - "type": "GitHub Sponsors", - "url": "https://github.com/sponsors/unifiedjs" - }, - { - "type": "OpenCollective", - "url": "https://opencollective.com/unified" - } - ], - "license": "MIT" - }, - "node_modules/micromark-util-normalize-identifier": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/micromark-util-normalize-identifier/-/micromark-util-normalize-identifier-2.0.1.tgz", - "integrity": "sha512-sxPqmo70LyARJs0w2UclACPUUEqltCkJ6PhKdMIDuJ3gSf/Q+/GIe3WKl0Ijb/GyH9lOpUkRAO2wp0GVkLvS9Q==", - "funding": [ - { - "type": "GitHub Sponsors", - "url": "https://github.com/sponsors/unifiedjs" - }, - { - "type": "OpenCollective", - "url": "https://opencollective.com/unified" - } - ], - "license": "MIT", - "dependencies": { - "micromark-util-symbol": "^2.0.0" - } - }, - "node_modules/micromark-util-resolve-all": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/micromark-util-resolve-all/-/micromark-util-resolve-all-2.0.1.tgz", - "integrity": "sha512-VdQyxFWFT2/FGJgwQnJYbe1jjQoNTS4RjglmSjTUlpUMa95Htx9NHeYW4rGDJzbjvCsl9eLjMQwGeElsqmzcHg==", - "funding": [ - { - "type": "GitHub Sponsors", - "url": "https://github.com/sponsors/unifiedjs" - }, - { - "type": "OpenCollective", - "url": "https://opencollective.com/unified" - } - ], - "license": "MIT", - "dependencies": { - "micromark-util-types": "^2.0.0" - } - }, - "node_modules/micromark-util-sanitize-uri": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/micromark-util-sanitize-uri/-/micromark-util-sanitize-uri-2.0.1.tgz", - "integrity": "sha512-9N9IomZ/YuGGZZmQec1MbgxtlgougxTodVwDzzEouPKo3qFWvymFHWcnDi2vzV1ff6kas9ucW+o3yzJK9YB1AQ==", - "funding": [ - { - "type": "GitHub Sponsors", - "url": "https://github.com/sponsors/unifiedjs" - }, - { - "type": "OpenCollective", - "url": "https://opencollective.com/unified" - } - ], - "license": "MIT", - "dependencies": { - "micromark-util-character": "^2.0.0", - "micromark-util-encode": "^2.0.0", - "micromark-util-symbol": "^2.0.0" - } - }, - "node_modules/micromark-util-subtokenize": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/micromark-util-subtokenize/-/micromark-util-subtokenize-2.1.0.tgz", - "integrity": "sha512-XQLu552iSctvnEcgXw6+Sx75GflAPNED1qx7eBJ+wydBb2KCbRZe+NwvIEEMM83uml1+2WSXpBAcp9IUCgCYWA==", - "funding": [ - { - "type": "GitHub Sponsors", - "url": "https://github.com/sponsors/unifiedjs" - }, - { - "type": "OpenCollective", - "url": "https://opencollective.com/unified" - } - ], - "license": "MIT", - "dependencies": { - "devlop": "^1.0.0", - "micromark-util-chunked": "^2.0.0", - "micromark-util-symbol": "^2.0.0", - "micromark-util-types": "^2.0.0" - } - }, - "node_modules/micromark-util-symbol": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/micromark-util-symbol/-/micromark-util-symbol-2.0.1.tgz", - "integrity": "sha512-vs5t8Apaud9N28kgCrRUdEed4UJ+wWNvicHLPxCa9ENlYuAY31M0ETy5y1vA33YoNPDFTghEbnh6efaE8h4x0Q==", - "funding": [ - { - "type": "GitHub Sponsors", - "url": "https://github.com/sponsors/unifiedjs" - }, - { - "type": "OpenCollective", - "url": "https://opencollective.com/unified" - } - ], - "license": "MIT" - }, - "node_modules/micromark-util-types": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/micromark-util-types/-/micromark-util-types-2.0.3.tgz", - "integrity": "sha512-oxB2Ik03hI0gv+VNn9tnh1t1YEe9MDPptViAEgfdf3YQHsn0pzGTgCdlSCJXcwhqm8phaEuM7zeEu3QQzVBrPg==", - "funding": [ - { - "type": "GitHub Sponsors", - "url": "https://github.com/sponsors/unifiedjs" - }, - { - "type": "OpenCollective", - "url": "https://opencollective.com/unified" - } - ], - "license": "MIT" - }, - "node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "license": "MIT" - }, - "node_modules/parse-entities": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/parse-entities/-/parse-entities-4.0.2.tgz", - "integrity": "sha512-GG2AQYWoLgL877gQIKeRPGO1xF9+eG1ujIb5soS5gPvLQ1y2o8FL90w2QWNdf9I361Mpp7726c+lj3U0qK1uGw==", - "license": "MIT", - "dependencies": { - "@types/unist": "^2.0.0", - "character-entities-legacy": "^3.0.0", - "character-reference-invalid": "^2.0.0", - "decode-named-character-reference": "^1.0.0", - "is-alphanumerical": "^2.0.0", - "is-decimal": "^2.0.0", - "is-hexadecimal": "^2.0.0" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } - }, - "node_modules/parse-entities/node_modules/@types/unist": { - "version": "2.0.11", - "resolved": "https://registry.npmjs.org/@types/unist/-/unist-2.0.11.tgz", - "integrity": "sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA==", - "license": "MIT" - }, - "node_modules/stringify-entities": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/stringify-entities/-/stringify-entities-4.0.4.tgz", - "integrity": "sha512-IwfBptatlO+QCJUo19AqvrPNqlVMpW9YEL2LIVY+Rpv2qsjCGxaDLNRgeGsQWJhfItebuJhsGSLjaBbNSQ+ieg==", - "license": "MIT", - "dependencies": { - "character-entities-html4": "^2.0.0", - "character-entities-legacy": "^3.0.0" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } - }, - "node_modules/unist-util-is": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/unist-util-is/-/unist-util-is-6.0.1.tgz", - "integrity": "sha512-LsiILbtBETkDz8I9p1dQ0uyRUWuaQzd/cuEeS1hoRSyW5E5XGmTzlwY1OrNzzakGowI9Dr/I8HVaw4hTtnxy8g==", - "license": "MIT", - "dependencies": { - "@types/unist": "^3.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/unist-util-position-from-estree": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/unist-util-position-from-estree/-/unist-util-position-from-estree-2.0.0.tgz", - "integrity": "sha512-KaFVRjoqLyF6YXCbVLNad/eS4+OfPQQn2yOd7zF/h5T/CSL2v8NpN6a5TPvtbXthAGw5nG+PuTtq+DdIZr+cRQ==", - "license": "MIT", - "dependencies": { - "@types/unist": "^3.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/unist-util-remove-position": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/unist-util-remove-position/-/unist-util-remove-position-5.0.0.tgz", - "integrity": "sha512-Hp5Kh3wLxv0PHj9m2yZhhLt58KzPtEYKQQ4yxfYFEO7EvHwzyDYnduhHnY1mDxoqr7VUwVuHXk9RXKIiYS1N8Q==", - "license": "MIT", - "dependencies": { - "@types/unist": "^3.0.0", - "unist-util-visit": "^5.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/unist-util-stringify-position": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/unist-util-stringify-position/-/unist-util-stringify-position-4.0.0.tgz", - "integrity": "sha512-0ASV06AAoKCDkS2+xw5RXJywruurpbC4JZSm7nr7MOt1ojAzvyyaO+UxZf18j8FCF6kmzCZKcAgN/yu2gm2XgQ==", - "license": "MIT", - "dependencies": { - "@types/unist": "^3.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/unist-util-visit": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/unist-util-visit/-/unist-util-visit-5.1.0.tgz", - "integrity": "sha512-m+vIdyeCOpdr/QeQCu2EzxX/ohgS8KbnPDgFni4dQsfSCtpz8UqDyY5GjRru8PDKuYn7Fq19j1CQ+nJSsGKOzg==", - "license": "MIT", - "dependencies": { - "@types/unist": "^3.0.0", - "unist-util-is": "^6.0.0", - "unist-util-visit-parents": "^6.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/unist-util-visit-parents": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/unist-util-visit-parents/-/unist-util-visit-parents-6.0.2.tgz", - "integrity": "sha512-goh1s1TBrqSqukSc8wrjwWhL0hiJxgA8m4kFxGlQ+8FYQ3C/m11FcTs4YYem7V664AhHVvgoQLk890Ssdsr2IQ==", - "license": "MIT", - "dependencies": { - "@types/unist": "^3.0.0", - "unist-util-is": "^6.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/vfile-message": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/vfile-message/-/vfile-message-4.0.3.tgz", - "integrity": "sha512-QTHzsGd1EhbZs4AsQ20JX1rC3cOlt/IWJruk893DfLRr57lcnOeMaWG4K0JrRta4mIJZKth2Au3mM3u03/JWKw==", - "license": "MIT", - "dependencies": { - "@types/unist": "^3.0.0", - "unist-util-stringify-position": "^4.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/zwitch": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/zwitch/-/zwitch-2.0.4.tgz", - "integrity": "sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A==", - "license": "MIT", - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } - } - } -} diff --git a/crates/mq-markdown/tests/mdast_official/package.json b/crates/mq-markdown/tests/mdast_official/package.json deleted file mode 100644 index 983c55c0f..000000000 --- a/crates/mq-markdown/tests/mdast_official/package.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "private": true, - "type": "module", - "description": "Reference parser for tests/mdx_official.rs: the mdast of the official MDX packages.", - "dependencies": { - "mdast-util-from-markdown": "2.1.0", - "mdast-util-frontmatter": "^2.0.1", - "mdast-util-gfm": "^3.1.0", - "mdast-util-math": "^3.0.0", - "mdast-util-mdx": "3.0.0", - "micromark-extension-frontmatter": "^2.0.0", - "micromark-extension-gfm": "^3.0.0", - "micromark-extension-math": "^3.1.0", - "micromark-extension-mdxjs": "3.0.0" - } -} diff --git a/crates/mq-markdown/tests/mdast_official/round_trip_known.json b/crates/mq-markdown/tests/mdast_official/round_trip_known.json deleted file mode 100644 index 4b52db244..000000000 --- a/crates/mq-markdown/tests/mdast_official/round_trip_known.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "gfm-extensions#19": "Text that reads as an email address or a URL once written is linked, as it cannot be written so that it stays text.", - "markdown-rs/gfm_autolink_literal#31": "Text that reads as an email address or a URL once written is linked, as it cannot be written so that it stays text.", - "gfm-extensions#23": "A footnote definition with blocks, such as code, is written on one line.", - "markdown-rs/misc_line_ending#20": "Line endings of a lone carriage return are not kept when written.", - "random-markdown-1#311": "Emphasis next to a run of asterisks that closes a strong and an image is written so that the runs fuse.", - "markdown-rs/gfm_task_list_item#6": "Line endings of a lone carriage return are not kept when written." -} \ No newline at end of file diff --git a/crates/mq-markdown/tests/mdast_tests.rs b/crates/mq-markdown/tests/mdast_tests.rs new file mode 100644 index 000000000..aa294ac4c --- /dev/null +++ b/crates/mq-markdown/tests/mdast_tests.rs @@ -0,0 +1,1036 @@ +//! The nodes mq reads from MDX and Markdown, flattened to tokens. The expected tokens are those of the +//! mdast that the official packages (`mdast-util-from-markdown` with `mdast-util-mdx`, `mdast-util-gfm`, +//! `mdast-util-math` and `mdast-util-frontmatter`) make of the same input, except where a comment says +//! how mq differs. The inputs come from the tests of markdown-rs and micromark and from hand-written cases. + +use mq_markdown::{Markdown, MdxAttributeContent, MdxAttributeValue, Node, TableAlignKind}; +use rstest::rstest; + +enum Token { + Text(String), + Other(String), +} + +/// Quotes `value` as `JSON.stringify` does. +fn quote(value: &str) -> String { + let mut out = String::from('"'); + for char in value.chars() { + match char { + '"' => out.push_str("\\\""), + '\\' => out.push_str("\\\\"), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + '\t' => out.push_str("\\t"), + '\u{8}' => out.push_str("\\b"), + '\u{c}' => out.push_str("\\f"), + char if char < ' ' => out.push_str(&format!("\\u{:04x}", char as u32)), + char => out.push(char), + } + } + out.push('"'); + out +} + +fn other(token: impl Into<String>) -> Token { + Token::Other(token.into()) +} + +fn container(head: String, inner: Vec<Token>) -> Vec<Token> { + let mut tokens = vec![other(format!("{head}("))]; + tokens.extend(inner); + tokens.push(other(")")); + tokens +} + +/// The tokens of `nodes`, with adjacent text joined when `inline`. +fn children(nodes: &[Node], inline: bool) -> Vec<Token> { + let mut tokens: Vec<Token> = Vec::new(); + for token in nodes.iter().flat_map(node_tokens) { + match (tokens.last_mut(), token) { + (Some(Token::Text(last)), Token::Text(text)) if inline => last.push_str(&text), + (_, token) => tokens.push(token), + } + } + tokens +} + +fn attributes(attributes: &[MdxAttributeContent]) -> String { + attributes + .iter() + .map(|attribute| match attribute { + MdxAttributeContent::Expression(value) => format!(" ...{{{value}}}"), + MdxAttributeContent::Property(property) => match &property.value { + None => format!(" {}", property.name), + Some(MdxAttributeValue::Literal(value)) => format!(" {}=lit:{}", property.name, quote(value)), + Some(MdxAttributeValue::Expression(value)) => format!(" {}=expr:{{{value}}}", property.name), + }, + }) + .collect() +} + +fn or_empty<T: ToString>(value: &Option<T>) -> String { + value.as_ref().map(ToString::to_string).unwrap_or_default() +} + +/// Paragraphs have no token, since mq has none, and list items are flat, as in mq. +fn node_tokens(node: &Node) -> Vec<Token> { + match node { + Node::Heading(heading) => container(format!("heading {}", heading.depth), children(&heading.values, true)), + Node::HorizontalRule(_) => vec![other("hr")], + Node::Code(code) => vec![other(format!( + "code {} {} {}", + or_empty(&code.lang), + or_empty(&code.meta), + quote(&code.value) + ))], + Node::Html(html) => vec![other(format!("html {}", quote(&html.value)))], + Node::Blockquote(quote) => container("blockquote".into(), children(&quote.values, false)), + Node::List(list) => { + let number = if list.ordered { + format!("o{}", list.start.unwrap_or(1) as usize + list.index) + } else { + "u".into() + }; + let checked = match list.checked { + Some(true) => "x", + Some(false) => "_", + None => "-", + }; + let spread = if list.spread { "s" } else { "t" }; + container( + format!("item {} {number} {checked} {spread}", list.level), + children(&list.values, false), + ) + } + Node::TableCell(cell) => container( + format!("cell {} {}", cell.row, cell.column), + children(&cell.values, true), + ), + Node::TableAlign(align) => { + let align = align + .align + .iter() + .map(|kind| match kind { + TableAlignKind::Left => "\"left\"", + TableAlignKind::Right => "\"right\"", + TableAlignKind::Center => "\"center\"", + TableAlignKind::None => "\"none\"", + }) + .collect::<Vec<_>>(); + vec![other(format!("align [{}]", align.join(",")))] + } + Node::Definition(definition) => vec![other(format!( + "definition {} {} {}", + definition.ident, + definition.url.as_str(), + or_empty(&definition.title) + ))], + Node::Footnote(footnote) => container( + format!("footnote {}", footnote.ident), + children(&footnote.values, false), + ), + Node::FootnoteRef(reference) => vec![other(format!("footnoteref {}", reference.ident))], + Node::Math(math) => vec![other(format!("math {}", quote(&math.value)))], + Node::MathInline(math) => vec![other(format!("imath {}", quote(&math.value)))], + Node::Yaml(yaml) => vec![other(format!("yaml {}", quote(&yaml.value)))], + Node::Toml(toml) => vec![other(format!("toml {}", quote(&toml.value)))], + Node::Text(text) => vec![Token::Text(text.value.clone())], + Node::Emphasis(emphasis) => container("em".into(), children(&emphasis.values, true)), + Node::Strong(strong) => container("strong".into(), children(&strong.values, true)), + Node::Delete(delete) => container("del".into(), children(&delete.values, true)), + Node::CodeInline(code) => vec![other(format!("code_inline {}", quote(&code.value)))], + Node::Break(_) => vec![other("br")], + Node::Link(link) => container( + format!("link {} {}", link.url.as_str(), or_empty(&link.title)), + children(&link.values, true), + ), + Node::Image(image) => vec![other(format!( + "image {} {} {}", + image.url, + or_empty(&image.title), + quote(&image.alt) + ))], + Node::LinkRef(reference) => container( + format!("linkref {}", reference.ident), + children(&reference.values, true), + ), + Node::ImageRef(reference) => vec![other(format!("imageref {} {}", reference.ident, quote(&reference.alt)))], + Node::MdxJsxFlowElement(element) => container( + format!("flow <{}{}>", or_empty(&element.name), attributes(&element.attributes)), + children(&element.children, false), + ), + Node::MdxJsxTextElement(element) => container( + format!("text <{}{}>", or_empty(&element.name), attributes(&element.attributes)), + children(&element.children, true), + ), + Node::MdxFlowExpression(expression) => vec![other(format!("flowexpr {{{}}}", expression.value))], + Node::MdxTextExpression(expression) => vec![other(format!("textexpr {{{}}}", expression.value))], + Node::MdxJsEsm(esm) => vec![other(format!("esm {}", quote(&esm.value)))], + Node::Empty => Vec::new(), + node => panic!("unexpected node {}", node.name()), + } +} + +fn tokens(markdown: &Markdown) -> Vec<String> { + children(&markdown.nodes, false) + .into_iter() + .map(|token| match token { + Token::Text(text) => format!("T{}", quote(&text)), + Token::Other(token) => token, + }) + .collect() +} + +/// Checks the tokens of `input`, and that what mq writes of it reads back as the same tokens. +fn check(input: &str, expected: &[&str], parse: fn(&str) -> miette::Result<Markdown>) { + let markdown = parse(input).unwrap_or_else(|error| panic!("{input:?}: {error}")); + assert_eq!(tokens(&markdown), expected, "{input:?}"); + let written = markdown.to_string(); + let reparsed = parse(&written).unwrap_or_else(|error| panic!("{input:?} written as {written:?}: {error}")); + assert_eq!(tokens(&reparsed), expected, "{input:?} written as {written:?}"); +} + +#[rstest] +// Hand-written. +// should support a self-closing element +#[case("<a />", &["flow <a>(", ")"])] +#[case(r#"<a b="c&amp;d" e='f' g={h} {...i} j />"#, &[r#"flow <a b=lit:"c&d" e=lit:"f" g=expr:{h} ...{...i} j>("#, ")"])] +#[case("<a.b.c />", &["flow <a.b.c>(", ")"])] +#[case(r#"<a:b c:d="e" />"#, &[r#"flow <a:b c:d=lit:"e">("#, ")"])] +#[case("<></>", &["flow <>(", ")"])] +#[case("<a>x</a>", &["text <a>(", r#"T"x""#, ")"])] +#[case("<a>\n\nx\n\n</a>", &["flow <a>(", r#"T"x""#, ")"])] +#[case("<a>\n x\n</a>", &["flow <a>(", r#"T"x""#, ")"])] +#[case("a <b>c</b> d", &[r#"T"a ""#, "text <b>(", r#"T"c""#, ")", r#"T" d""#])] +#[case("a <b/> d", &[r#"T"a ""#, "text <b>(", ")", r#"T" d""#])] +// should support an expression +#[case("{a}", &["flowexpr {a}"])] +// Differs from the official packages: Expressions are not parsed as JavaScript, so invalid JavaScript is accepted. +#[case("{a {b} c}", &["flowexpr {a {b} c}"])] +// Differs from the official packages: Expressions are not parsed as JavaScript, so invalid JavaScript is accepted. +#[case("{a\nb}", &["flowexpr {a\nb}"])] +// Differs from the official packages: Expressions are not parsed as JavaScript, so invalid JavaScript is accepted. +#[case("{a\n b\n c}", &["flowexpr {a\nb\n c}"])] +#[case("x {a} y", &[r#"T"x ""#, "textexpr {a}", r#"T" y""#])] +#[case("{a} <b/>", &["flowexpr {a}", "flow <b>(", ")"])] +#[case("<b/> {a}", &["flow <b>(", ")", "flowexpr {a}"])] +#[case("<b/> x", &["text <b>(", ")", r#"T" x""#])] +#[case("{a}x", &["textexpr {a}", r#"T"x""#])] +#[case("<a>{b}</a>", &["flow <a>(", "flowexpr {b}", ")"])] +#[case("a < b", &[r#"T"a < b""#])] +#[case("<a>\n<b>\n</b>\n</a>", &["flow <a>(", "flow <b>(", ")", ")"])] +#[case("> <a>\n> x\n> </a>", &["blockquote(", "flow <a>(", r#"T"x""#, ")", ")"])] +#[case("- <a>\n x\n </a>", &["item 0 u - t(", "flow <a>(", r#"T"x""#, ")", ")"])] +#[case(r#"<a b='c"d' />"#, &[r#"flow <a b=lit:"c\"d">("#, ")"])] +#[case("<a/><b/>", &["flow <a>(", ")", "flow <b>(", ")"])] +#[case("<a>x</a><b>y</b>", &["text <a>(", r#"T"x""#, ")", "text <b>(", r#"T"y""#, ")"])] +#[case("a\n<b/>", &[r#"T"a""#, "flow <b>(", ")"])] +#[case("a\n{b}", &[r#"T"a""#, "flowexpr {b}"])] +#[case(r#"<b e= "f"/>"#, &[r#"flow <b e=lit:"f">("#, ")"])] +#[case(r#"a <b e = "f"/>."#, &[r#"T"a ""#, r#"text <b e=lit:"f">("#, ")", r#"T".""#])] +#[case("<b c\n= \"x\">c</b>", &[r#"text <b c=lit:"x">("#, r#"T"c""#, ")"])] +// should support prefixed attributes +#[case("a <b xml :\tlang\n= \"de-CH\" foo:bar>c</b>.", &[r#"T"a ""#, r#"text <b xml:lang=lit:"de-CH" foo:bar>("#, r#"T"c""#, ")", r#"T".""#])] +// should support prefixed and normal attributes +#[case(r#"a <b a b : c d : e = "f" g/>."#, &[r#"T"a ""#, r#"text <b a b:c d:e=lit:"f" g>("#, ")", r#"T".""#])] +#[case("<b e= {f}/>", &["flow <b e=expr:{f}>(", ")"])] +#[case("<a>\n\n# h\n\n- l\n\n</a>", &["flow <a>(", "heading 1(", r#"T"h""#, ")", "item 0 u - t(", r#"T"l""#, ")", ")"])] +#[case("<Zoom>\n\n![a](b)\n\n</Zoom>", &["flow <Zoom>(", r#"image b "a""#, ")"])] +#[case("<A>\n <B>\n x\n </B>\n</A>", &["flow <A>(", "flow <B>(", r#"T"x""#, ")", ")"])] +#[case("import a from \"b\"\nexport const x = 1\n\nx\n", &[r#"esm "import a from \"b\"\nexport const x = 1""#, r#"T"x""#])] +#[case("export default d\n", &[r#"esm "export default d""#])] +#[case(" import a from \"b\"\n", &[r#"T"import a from \"b\"""#])] +#[case("import{a} from \"b\"\n", &[r#"T"import""#, "textexpr {a}", r#"T" from \"b\"""#])] +#[case("importx a\n", &[r#"T"importx a""#])] +#[case("- import a from \"b\"\n", &["item 0 u - t(", r#"T"import a from \"b\"""#, ")"])] +#[case("text\nimport a from \"b\"\n", &[r#"T"text\nimport a from \"b\"""#])] +// Differs from the official packages: A tag that is not closed before the end of its text is text here and in markdown-rs, and an error in the JavaScript packages. +#[case("<a", &[r#"T"<a""#])] +// Differs from the official packages: A tag that is not closed before the end of its text is text here and in markdown-rs, and an error in the JavaScript packages. +#[case("<a b", &[r#"T"<a b""#])] +// Differs from the official packages: A tag that is not closed before the end of its text is text here and in markdown-rs, and an error in the JavaScript packages. +#[case("<a b=", &[r#"T"<a b=""#])] +#[case(r#"a <b c="d" /> e"#, &[r#"T"a ""#, r#"text <b c=lit:"d">("#, ")", r#"T" e""#])] +#[case("a <b>c</b>", &[r#"T"a ""#, "text <b>(", r#"T"c""#, ")"])] +#[case("<a b=\"c\nd\" />", &[r#"flow <a b=lit:"c\nd">("#, ")"])] +#[case("<a b='&lt;' />", &[r#"flow <a b=lit:"<">("#, ")"])] +#[case(r#"<a b="&#x41;" />"#, &[r#"flow <a b=lit:"A">("#, ")"])] +#[case("<a {...b} {...c} />", &["flow <a ...{...b} ...{...c}>(", ")"])] +// should crash if not a spread +// Differs from the official packages: Expressions are not parsed as JavaScript, so invalid JavaScript is accepted. +#[case("<a {b} />", &["flow <a ...{b}>(", ")"])] +#[case("<a b={c} d={e} />", &["flow <a b=expr:{c} d=expr:{e}>(", ")"])] +#[case("<a b={{c: 1}} />", &["flow <a b=expr:{{c: 1}}>(", ")"])] +#[case("{/* comment */}", &["flowexpr {/* comment */}"])] +// should support an empty expression +#[case("{}", &["flowexpr {}"])] +// should support an empty expression (1) +#[case("a {} b", &[r#"T"a ""#, "textexpr {}", r#"T" b""#])] +#[case("a {b} {c} d", &[r#"T"a ""#, "textexpr {b}", r#"T" ""#, "textexpr {c}", r#"T" d""#])] +#[case("<a>{b}{c}</a>", &["text <a>(", "textexpr {b}", "textexpr {c}", ")"])] +#[case("# <a/>", &["heading 1(", "text <a>(", ")", ")"])] +#[case("# a {b} c", &["heading 1(", r#"T"a ""#, "textexpr {b}", r#"T" c""#, ")"])] +#[case("*<a>b</a>*", &["em(", "text <a>(", r#"T"b""#, ")", ")"])] +#[case("**a {b}**", &["strong(", r#"T"a ""#, "textexpr {b}", ")"])] +#[case("[<a/>](b)", &["link b (", "text <a>(", ")", ")"])] +#[case("`<a/>`", &[r#"code_inline "<a/>""#])] +#[case("`{a}`", &[r#"code_inline "{a}""#])] +#[case("<a/>\n<b/>", &["flow <a>(", ")", "flow <b>(", ")"])] +#[case("<a/> <b/>", &["flow <a>(", ")", "flow <b>(", ")"])] +#[case("<a/>x\n", &["text <a>(", ")", r#"T"x""#])] +#[case("x<a/>", &[r#"T"x""#, "text <a>(", ")"])] +#[case("<a>\n\nb\n\n</a>\n\n<c/>", &["flow <a>(", r#"T"b""#, ")", "flow <c>(", ")"])] +// should support an element w/ content +#[case("<a>\nb\n</a>", &["flow <a>(", r#"T"b""#, ")"])] +#[case("<a>\n <b/>\n</a>", &["flow <a>(", "flow <b>(", ")", ")"])] +#[case("- a\n <b/>\n", &["item 0 u - t(", r#"T"a""#, "flow <b>(", ")", ")"])] +#[case("> a\n> <b/>\n", &["blockquote(", r#"T"a""#, "flow <b>(", ")", ")"])] +// should support an element w/ containers as content +#[case("<a>\n- b\n</a>", &["flow <a>(", "item 0 u - t(", r#"T"b""#, ")", ")"])] +#[case("<a>\n\n- b\n\n</a>", &["flow <a>(", "item 0 u - t(", r#"T"b""#, ")", ")"])] +#[case("<a b=\"c\" d=\"e\">\n\nf\n\n</a>", &[r#"flow <a b=lit:"c" d=lit:"e">("#, r#"T"f""#, ")"])] +#[case("<a-b />", &["flow <a-b>(", ")"])] +#[case("<a.b-c />", &["flow <a.b-c>(", ")"])] +#[case("<_a />", &["flow <_a>(", ")"])] +#[case("<$a />", &["flow <$a>(", ")"])] +#[case("< a />", &[r#"T"< a />""#])] +#[case("<a / >", &["flow <a>(", ")"])] +#[case("<a/ >", &["flow <a>(", ")"])] +#[case(r#"<a b-c="d" />"#, &[r#"flow <a b-c=lit:"d">("#, ")"])] +#[case(r#"<a b:c-d="e" />"#, &[r#"flow <a b:c-d=lit:"e">("#, ")"])] +#[case("<a b='c' d />", &[r#"flow <a b=lit:"c" d>("#, ")"])] +#[case("<a\nb\n/>", &["flow <a b>(", ")"])] +#[case("<a\n b=\"c\"\n/>", &[r#"flow <a b=lit:"c">("#, ")"])] +#[case("{a}\n{b}", &["flowexpr {a}", "flowexpr {b}"])] +#[case("{a}\n\n{b}", &["flowexpr {a}", "flowexpr {b}"])] +#[case("{a} {b}", &["textexpr {a}", r#"T" ""#, "textexpr {b}"])] +#[case("{a}{b}", &["textexpr {a}", "textexpr {b}"])] +#[case("x\n{a}\n", &[r#"T"x""#, "flowexpr {a}"])] +#[case("{\n}", &["flowexpr {\n}"])] +#[case("{a}\n<b/>", &["flowexpr {a}", "flow <b>(", ")"])] +#[case("<b/>\n{a}", &["flow <b>(", ")", "flowexpr {a}"])] +// should support a closed element +#[case("a <b> c </b> d", &[r#"T"a ""#, "text <b>(", r#"T" c ""#, ")", r#"T" d""#])] +#[case("<b>\n</b>", &["flow <b>(", ")"])] +#[case("<b></b>", &["flow <b>(", ")"])] +#[case("<b> </b>", &["flow <b>(", ")"])] +#[case("<>a</>", &["text <>(", r#"T"a""#, ")"])] +#[case("<>\n\na\n\n</>", &["flow <>(", r#"T"a""#, ")"])] +#[case("<a>{`}`}</a>", &["flow <a>(", "flowexpr {`}`}", ")"])] +#[case("{'}'}", &["flowexpr {'}'}"])] +#[case(r#"{"}"}"#, &[r#"flowexpr {"}"}"#])] +#[case(r#"<a b="{c}" />"#, &[r#"flow <a b=lit:"{c}">("#, ")"])] +#[case(r#"<a b={"c"} />"#, &[r#"flow <a b=expr:{"c"}>("#, ")"])] +#[case("<a b={`c`} />", &["flow <a b=expr:{`c`}>(", ")"])] +#[case(r#"a\<b/>"#, &[r#"T"a<b/>""#])] +#[case(r#"a\{b}"#, &[r#"T"a{b}""#])] +#[case("&lt;a/>", &[r#"T"<a/>""#])] +#[case("a &#123;b}", &[r#"T"a {b}""#])] +#[case("https://a.b", &[r#"T"https://a.b""#])] +#[case("www.a.b", &[r#"T"www.a.b""#])] +#[case("---\ntitle: x\n---\n\na", &[r#"yaml "title: x""#, r#"T"a""#])] +#[case("a\n---\n", &["heading 2(", r#"T"a""#, ")"])] +#[case("***\n", &["hr"])] +#[case("```\n<a/>\n```\n", &[r#"code "<a/>""#])] +#[case(" <a/>\n", &["flow <a>(", ")"])] +#[case("<a/>\n x\n", &["flow <a>(", ")", r#"T"x""#])] +#[case("<a>\n\n x\n\n</a>", &["flow <a>(", r#"T"x""#, ")"])] +#[case("1. <a/>\n", &["item 0 o1 - t(", "flow <a>(", ")", ")"])] +#[case("* <a>\n b\n </a>\n", &["item 0 u - t(", "flow <a>(", r#"T"b""#, ")", ")"])] +// Differs from the official packages: A tag that is not closed before the end of its text is text here and in markdown-rs, and an error in the JavaScript packages. +#[case("x <a", &[r#"T"x <a""#])] +// Differs from the official packages: A tag that is not closed before the end of its text is text here and in markdown-rs, and an error in the JavaScript packages. +#[case("x <a b", &[r#"T"x <a b""#])] +// Differs from the official packages: A tag that is not closed before the end of its text is text here and in markdown-rs, and an error in the JavaScript packages. +#[case("x <a b=", &[r#"T"x <a b=""#])] +// Differs from the official packages: A tag that is not closed before the end of its text is text here and in markdown-rs, and an error in the JavaScript packages. +#[case("<a\n\nb", &[r#"T"<a""#, r#"T"b""#])] +// Differs from the official packages: A tag that is not closed before the end of its text is text here and in markdown-rs, and an error in the JavaScript packages. +#[case("<a\nb", &[r#"T"<a\nb""#])] +// Differs from the official packages: A tag that is not closed before the end of its text is text here and in markdown-rs, and an error in the JavaScript packages. +#[case("a\n<a", &[r#"T"a\n<a""#])] +// Differs from the official packages: A tag that is not closed before the end of its text is text here and in markdown-rs, and an error in the JavaScript packages. +#[case("x <", &[r#"T"x <""#])] +// Differs from the official packages: A tag that is not closed before the end of its text is text here and in markdown-rs, and an error in the JavaScript packages. +#[case("<", &[r#"T"<""#])] +// Differs from the official packages: A tag that is not closed before the end of its text is text here and in markdown-rs, and an error in the JavaScript packages. +#[case("x <a\n\nb", &[r#"T"x <a""#, r#"T"b""#])] +// Differs from the official packages: A tag that is not closed before the end of its text is text here and in markdown-rs, and an error in the JavaScript packages. +#[case("x <a b\n\nc", &[r#"T"x <a b""#, r#"T"c""#])] +// Differs from the official packages: A tag that is not closed before the end of its text is text here and in markdown-rs, and an error in the JavaScript packages. +#[case("- <a", &["item 0 u - t(", r#"T"<a""#, ")"])] +// Differs from the official packages: A tag that is not closed before the end of its text is text here and in markdown-rs, and an error in the JavaScript packages. +#[case("> <a", &["blockquote(", r#"T"<a""#, ")"])] +// Differs from the official packages: Expressions are not parsed as JavaScript, so invalid JavaScript is accepted. +#[case("{a\n\nb}", &["flowexpr {a\n\nb}"])] +#[case("{a /* } */}", &["flowexpr {a /* } */}"])] +#[case("{`${'}'}`}", &["flowexpr {`${'}'}`}"])] +#[case("<a b={'}'} />", &["flow <a b=expr:{'}'}>(", ")"])] +#[case("{a // }\n}", &["flowexpr {a // }\n}"])] +#[case(r#"{"a}" + '{'}"#, &[r#"flowexpr {"a}" + '{'}"#])] +#[case(r#"<a b={"}"} c='}' />"#, &[r#"flow <a b=expr:{"}"} c=lit:"}">("#, ")"])] +#[case("{a /* { */}", &["flowexpr {a /* { */}"])] +#[case("{`a${b}c`}", &["flowexpr {`a${b}c`}"])] +#[case("{`a${`}`}c`}", &["flowexpr {`a${`}`}c`}"])] +#[case(r#"{'\'}'}"#, &[r#"flowexpr {'\'}'}"#])] +#[case("{a // {\n}", &["flowexpr {a // {\n}"])] +#[case("x {'}'} y", &[r#"T"x ""#, "textexpr {'}'}", r#"T" y""#])] +#[case("x {`}`} y", &[r#"T"x ""#, "textexpr {`}`}", r#"T" y""#])] +#[case("{/* } */}\n", &["flowexpr {/* } */}"])] +#[case("<a>{'}'}</a>", &["flow <a>(", "flowexpr {'}'}", ")"])] +// Differs from the official packages: Expressions are not parsed as JavaScript, so invalid JavaScript is accepted. +#[case("{'a\nb'}", &["flowexpr {'a\nb'}"])] +#[case("{a / b}", &["flowexpr {a / b}"])] +#[case("{a /* x */ + 1}", &["flowexpr {a /* x */ + 1}"])] +#[case("---\ntitle: x\n---\n\n<A />\n", &[r#"yaml "title: x""#, "flow <A>(", ")"])] +#[case("+++\na = 1\n+++\n\nb\n", &[r#"toml "a = 1""#, r#"T"b""#])] +#[case("---\na\n", &["hr", r#"T"a""#])] +#[case("---\n\n---\n\nx\n", &[r#"yaml """#, r#"T"x""#])] +#[case("---\ntitle: x\n---\nimport a from \"b\"\n\n{a}\n", &[r#"yaml "title: x""#, r#"esm "import a from \"b\"""#, "flowexpr {a}"])] +// From markdown-rs, `tests/mdx_esm.rs`. +// should support an import +#[case("import a from 'b'\n\nc", &[r#"esm "import a from 'b'""#, r#"T"c""#])] +// should support an export +#[case("export default a\n\nb", &[r#"esm "export default a""#, r#"T"b""#])] +// should not support other keywords (`impossible`) +#[case("impossible", &[r#"T"impossible""#])] +// should not support other keywords (`exporting`) +#[case("exporting", &[r#"T"exporting""#])] +// should not support a non-whitespace after the keyword +#[case("import.", &[r#"T"import.""#])] +// should not support a non-whitespace after the keyword (import-as-a-function) +#[case("import('a')", &[r#"T"import('a')""#])] +// should not support an indent +#[case(" import a from 'b'\n export default c", &[r#"T"import a from 'b'\nexport default c""#])] +// should not support keywords in containers +#[case("- import a from 'b'\n> export default c", &["item 0 u - t(", r#"T"import a from 'b'""#, ")", "blockquote(", r#"T"export default c""#, ")"])] +// should support imports and exports in the same “block” +#[case("import a from 'b'\nexport default c", &[r#"esm "import a from 'b'\nexport default c""#])] +// should support imports and exports in separate “blocks” +#[case("import a from 'b'\n\nexport default c", &[r#"esm "import a from 'b'""#, r#"esm "export default c""#])] +// should support imports and exports in between other constructs +#[case("a\n\nimport a from 'b'\n\nb\n\nexport default c", &[r#"T"a""#, r#"esm "import a from 'b'""#, r#"T"b""#, r#"esm "export default c""#])] +// should not support import/exports when interrupting paragraphs +#[case("a\nimport a from 'b'\n\nb\nexport default c", &[r#"T"a\nimport a from 'b'""#, r#"T"b\nexport default c""#])] +// should crash on invalid import/exports (1) +// Differs from the official packages: Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted. +#[case("import a", &[r#"esm "import a""#])] +// should crash on invalid import/exports (2) +// Differs from the official packages: Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted. +#[case("import 1/1", &[r#"esm "import 1/1""#])] +// should support line endings in import/exports +#[case("export {\n a\n} from 'b'\n\nc", &[r#"esm "export {\n a\n} from 'b'""#, r#"T"c""#])] +// should support blank lines in import/exports +#[case("export {\n\n a\n\n} from 'b'\n\nc", &[r#"esm "export {\n\n a\n\n} from 'b'""#, r#"T"c""#])] +// should crash on markdown after import/export w/o blank line +// Differs from the official packages: Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted. +#[case("import a from 'b'\n*md*?", &[r#"esm "import a from 'b'\n*md*?""#])] +// should support comments in “blocks” +#[case("export var a = 1\n// b\n/* c */\n\nd", &[r#"esm "export var a = 1\n// b\n/* c */""#, r#"T"d""#])] +// should crash on other statements in “blocks” +// Differs from the official packages: Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted. +#[case("export var a = 1\nvar b\n\nc", &[r#"esm "export var a = 1\nvar b""#, r#"T"c""#])] +// should crash on import-as-a-function with a space `import (x)` +// Differs from the official packages: Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted. +#[case("import ('a')\n\nb", &[r#"esm "import ('a')""#, r#"T"b""#])] +// should support a reexport from another import +#[case("import a from 'b'\nexport {a}\n\nc", &[r#"esm "import a from 'b'\nexport {a}""#, r#"T"c""#])] +// should support a reexport from another import w/ semicolons +#[case("import a from 'b';\nexport {a};\n\nc", &[r#"esm "import a from 'b';\nexport {a};""#, r#"T"c""#])] +// should support a reexport default from another import +#[case("import a from 'b'\nexport {a as default}\n\nc", &[r#"esm "import a from 'b'\nexport {a as default}""#, r#"T"c""#])] +// should support JSX by default +#[case("export var a = () => <b />", &[r#"esm "export var a = () => <b />""#])] +// should support EOF after EOL +// Differs from the official packages: Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted. +#[case("export {a}\n", &[r#"esm "export {a}""#])] +// should support a reexport from another esm block (1) +#[case("import a from 'b'\n\nexport {a}\n\nc", &[r#"esm "import a from 'b'""#, r#"esm "export {a}""#, r#"T"c""#])] +// should support a reexport from another esm block (2) +#[case("import a from 'b'\n\nexport {a}\n\n# c", &[r#"esm "import a from 'b'""#, r#"esm "export {a}""#, "heading 1(", r#"T"c""#, ")"])] +// should support mdx esm as `MdxjsEsm`s in mdast +#[case("import a from 'b'\nexport {a}", &[r#"esm "import a from 'b'\nexport {a}""#])] +// From markdown-rs, `tests/mdx_expression_flow.rs`. +// should prefer indented code over expressions if it’s enabled +#[case(" {}", &["flowexpr {}"])] +// should support indented expressions if indented code is enabled +#[case(" {}", &["flowexpr {}"])] +// should support a line ending in an expression +#[case("{\n}\na", &["flowexpr {\n}", r#"T"a""#])] +// should support expressions followed by spaces +#[case("{ a } \t\nb", &["flowexpr { a }", r#"T"b""#])] +// should support expressions preceded by spaces +#[case(" { a }\nb", &["flowexpr { a }", r#"T"b""#])] +// should support lists after non-expressions (wooorm/markdown-rs#11) +#[case("a\n\n* b", &[r#"T"a""#, "item 0 u - t(", r#"T"b""#, ")"])] +// should not support lazyness (2) +#[case("> a\n{b}", &["blockquote(", r#"T"a""#, ")", "flowexpr {b}"])] +// should not support lazyness (3) +#[case("> {a}\nb", &["blockquote(", "flowexpr {a}", ")", r#"T"b""#])] +// should support mdx expressions (flow) as `MdxFlowExpression`s in mdast +#[case("{alpha +\nbravo}", &["flowexpr {alpha +\nbravo}"])] +// should support indent in `MdxFlowExpression` in mdast +#[case(" {`\n a\n `}", &["flowexpr {`\n a\n`}"])] +// should support expressions padded w/ parens +#[case("a{(b)}c", &[r#"T"a""#, "textexpr {(b)}", r#"T"c""#])] +// should support expressions padded w/ parens and comments +#[case("a{/* b */ ( (c) /* d */ + (e) )}f", &[r#"T"a""#, "textexpr {/* b */ ( (c) /* d */ + (e) )}", r#"T"f""#])] +// should use correct positional info when tabs are used (1, indent) +#[case("{`\n\t`}", &["flowexpr {`\n `}"])] +// should use correct positional info when tabs are used (2, content) +#[case("{`\nalpha\t`}", &["flowexpr {`\nalpha\t`}"])] +// should support template strings in JSX (text) in block quotes +#[case("> aaa <b c={`\n> d\n> `} /> eee", &["blockquote(", r#"T"aaa ""#, "text <b c=expr:{`\n d\n`}>(", ")", r#"T" eee""#, ")"])] +// should use correct positional when there are virtual spaces due to a block quote +#[case("> ab {`\n>\t`}", &["blockquote(", r#"T"ab ""#, "textexpr {`\n`}", ")"])] +// should keep the correct number of spaces in a blockquote (flow) +#[case("> {`\n> alpha\n> bravo\n> charlie\n> delta\n> `}", &["blockquote(", "flowexpr {`\nalpha\nbravo\ncharlie\n delta\n`}", ")"])] +// should support a spread +#[case("<a {...b} />", &["flow <a ...{...b}>(", ")"])] +// should crash on an incorrect spread +// Differs from the official packages: Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted. +#[case("<a {...?} />", &["flow <a ...{...?}>(", ")"])] +// should crash if not an identifier +// Differs from the official packages: Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted. +#[case("<a {b=c} />", &["flow <a ...{b=c}>(", ")"])] +// should crash on an empty spread +// Differs from the official packages: Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted. +#[case("<a {} />", &["flow <a ...{}>(", ")"])] +// should crash on a comment spread +// Differs from the official packages: Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted. +#[case("<a {/* b */} />", &["flow <a ...{/* b */}>(", ")"])] +// From markdown-rs, `tests/mdx_expression_text.rs`. +// should support an empty expression (2) +#[case("a { \t\r\n} b", &[r#"T"a ""#, "textexpr { \t\r\n}", r#"T" b""#])] +// should support a multiline comment (1) +#[case("a {/**/} b", &[r#"T"a ""#, "textexpr {/**/}", r#"T" b""#])] +// should support a multiline comment (2) +#[case("a { /*\n*/\t} b", &[r#"T"a ""#, "textexpr { /*\n*/\t}", r#"T" b""#])] +// should support a multiline comment (3) +#[case("a {/*b*//*c*/} d", &[r#"T"a ""#, "textexpr {/*b*//*c*/}", r#"T" d""#])] +// should support a multiline comment (4) +#[case("a {b/*c*/} d", &[r#"T"a ""#, "textexpr {b/*c*/}", r#"T" d""#])] +// should support a multiline comment (5) +#[case("a {/*b*/c} d", &[r#"T"a ""#, "textexpr {/*b*/c}", r#"T" d""#])] +// should support a line comment followed by a line ending +#[case("a {//\n} b", &[r#"T"a ""#, "textexpr {//\n}", r#"T" b""#])] +// should support a line comment followed by a line ending and an expression +#[case("a {// b\nc} d", &[r#"T"a ""#, "textexpr {// b\nc}", r#"T" d""#])] +// should support an expression followed by a line comment and a line ending +#[case("a {b// c\n} d", &[r#"T"a ""#, "textexpr {b// c\n}", r#"T" d""#])] +// should support comments +#[case("a {/*b*/ // c\n} d", &[r#"T"a ""#, "textexpr {/*b*/ // c\n}", r#"T" d""#])] +// should support expression statements (1) +#[case("a {b.c} d", &[r#"T"a ""#, "textexpr {b.c}", r#"T" d""#])] +// should support expression statements (2) +#[case("a {1 + 1} b", &[r#"T"a ""#, "textexpr {1 + 1}", r#"T" b""#])] +// should support expression statements (3) +#[case("a {function () {}} b", &[r#"T"a ""#, "textexpr {function () {}}", r#"T" b""#])] +// should crash on non-expressions +// Differs from the official packages: Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted. +#[case(r#"a {var b = "c"} d"#, &[r#"T"a ""#, r#"textexpr {var b = "c"}"#, r#"T" d""#])] +// should support expressions in containers +#[case("> a {\n> b} c", &["blockquote(", r#"T"a ""#, "textexpr {\nb}", r#"T" c""#, ")"])] +// should crash on incorrect expressions in containers (1) +// Differs from the official packages: Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted. +#[case("> a {\n> b<} c", &["blockquote(", r#"T"a ""#, "textexpr {\nb<}", r#"T" c""#, ")"])] +// should crash on incorrect expressions in containers (2) +// Differs from the official packages: Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted. +#[case("> a {\n> b\n> c} d", &["blockquote(", r#"T"a ""#, "textexpr {\nb\nc}", r#"T" d""#, ")"])] +// should support an expression +#[case("a {b} c", &[r#"T"a ""#, "textexpr {b}", r#"T" c""#])] +// should support a line ending in an expression +#[case("a {\n} b", &[r#"T"a ""#, "textexpr {\n}", r#"T" b""#])] +// should support just a closing brace +#[case("a } b", &[r#"T"a } b""#])] +// should support expressions as the first thing when following by other things +#[case("{ a } b", &["textexpr { a }", r#"T" b""#])] +// should support mdx expressions (text) as `MdxTextExpression`s in mdast +#[case("a {alpha} b.", &[r#"T"a ""#, "textexpr {alpha}", r#"T" b.""#])] +// should crash on an incorrect expression +// Differs from the official packages: Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted. +#[case("a {??} b", &[r#"T"a ""#, "textexpr {??}", r#"T" b""#])] +// should support an unbalanced opening brace (if JS permits) +#[case("a { /* { */ } b", &[r#"T"a ""#, "textexpr { /* { */ }", r#"T" b""#])] +// should support an unbalanced closing brace (if JS permits) +#[case("a { /* } */ } b", &[r#"T"a ""#, "textexpr { /* } */ }", r#"T" b""#])] +// should keep the correct number of spaces in a blockquote (text) +#[case("> alpha {`\n> bravo\n> charlie\n> delta\n> echo\n> `} foxtrot.", &["blockquote(", r#"T"alpha ""#, "textexpr {`\nbravo\ncharlie\ndelta\n echo\n`}", r#"T" foxtrot.""#, ")"])] +// From markdown-rs, `tests/mdx_jsx_flow.rs`. +// should prefer indented code over jsx if it’s enabled +#[case(" <a />", &["flow <a>(", ")"])] +// should support indented jsx if indented code is enabled +#[case(" <a />", &["flow <a>(", ")"])] +// should support a closed element +#[case("<a></a>", &["flow <a>(", ")"])] +// should support attributes +// Differs from the official packages: Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted. +#[case(r#"<a b c:d e="" f={/* g */} {...h} />"#, &[r#"flow <a b c:d e=lit:"" f=expr:{/* g */} ...{...h}>("#, ")"])] +// should support a dangling `>` in a tag (not a block quote) +#[case("<x\n y\n> \nb\n </x>", &["flow <x y>(", r#"T"b""#, ")"])] +// should support trailing initial and final whitespace around tags +#[case("<a> \nb\n </a>", &["flow <a>(", r#"T"b""#, ")"])] +// should support tags after tags +#[case("<a> <b>\t\nc\n </b> </a>", &["flow <a>(", "flow <b>(", r#"T"c""#, ")", ")"])] +// should not support lazy flow (8) +#[case("> a\n<X />", &["blockquote(", r#"T"a""#, ")", "flow <X>(", ")"])] +// should support mdx jsx (flow) as `MdxJsxFlowElement`s in mdast +#[case("<>\n * a\n</>", &["flow <>(", "item 0 u - t(", r#"T"a""#, ")", ")"])] +// should support tags and expressions (unaware) +#[case("<div>\n{1}\n</div>", &["flow <div>(", "flowexpr {1}", ")"])] +// should support tags and expressions (aware) +#[case("<div>\n{'}'}\n</div>", &["flow <div>(", "flowexpr {'}'}", ")"])] +// should support tags and expressions with text before (text) +#[case("x<em>{1}</em>", &[r#"T"x""#, "text <em>(", "textexpr {1}", ")"])] +// should support tags and expressions with text between, early (text) +#[case("<em>x{1}</em>", &["text <em>(", r#"T"x""#, "textexpr {1}", ")"])] +// should support tags and expressions with text between, late (text) +#[case("<em>{1}x</em>", &["text <em>(", "textexpr {1}", r#"T"x""#, ")"])] +// should support tags and expressions with text after (text) +#[case("<em>{1}</em>x", &["text <em>(", "textexpr {1}", ")", r#"T"x""#])] +// should support a tag and then an expression (flow) +#[case("<x/>{1}", &["flow <x>(", ")", "flowexpr {1}"])] +// should support a tag, an expression, then text (text) +#[case("<x/>{1}x", &["text <x>(", ")", "textexpr {1}", r#"T"x""#])] +// should support text, a tag, then an expression (text) +#[case("x<x/>{1}", &[r#"T"x""#, "text <x>(", ")", "textexpr {1}"])] +// should support an expression and then a tag (flow) +#[case("{1}<x/>", &["flowexpr {1}", "flow <x>(", ")"])] +// should support an expression, a tag, then text (text) +#[case("{1}<x/>x", &["textexpr {1}", "text <x>(", ")", r#"T"x""#])] +// should support text, an expression, then a tag (text) +#[case("x{1}<x/>", &[r#"T"x""#, "textexpr {1}", "text <x>(", ")"])] +// should nicely interleaf (micromark/micromark-extension-mdx-jsx#9) +#[case("<x>{[\n'',\n{c:''}\n]}</x>", &["flow <x>(", "flowexpr {[\n'',\n{c:''}\n]}", ")"])] +// should nicely interleaf (mdx-js/mdx#1945) +#[case("\n<style>{`\n .foo {}\n .bar {}\n`}</style>\n ", &["flow <style>(", "flowexpr {`\n.foo {}\n.bar {}\n`}", ")"])] +// From markdown-rs, `tests/mdx_jsx_text.rs`. +// should support a self-closing element +#[case("a <b/> c.", &[r#"T"a ""#, "text <b>(", ")", r#"T" c.""#])] +// should support a closed element +#[case("a <b></b> c.", &[r#"T"a ""#, "text <b>(", ")", r#"T" c.""#])] +// should support fragments +#[case("a <></> c.", &[r#"T"a ""#, "text <>(", ")", r#"T" c.""#])] +// should support markdown inside elements +#[case("a <b>*b*</b> c.", &[r#"T"a ""#, "text <b>(", "em(", r#"T"b""#, ")", ")", r#"T" c.""#])] +// should support mdx jsx (text) with expression child +#[case("{1}<a/>", &["flowexpr {1}", "flow <a>(", ")"])] +// should support mdx jsx (text) with expression child +#[case("<a>{1}</a>", &["flow <a>(", "flowexpr {1}", ")"])] +// should support mdx jsx (text) as `MdxJsxTextElement`s in mdast (self-closing) +#[case("a <b /> c.", &[r#"T"a ""#, "text <b>(", ")", r#"T" c.""#])] +// should support mdx jsx (text) as `MdxJsxTextElement`s in mdast (matched open and close tags) +#[case("a <b>*c*</b> d.", &[r#"T"a ""#, "text <b>(", "em(", r#"T"c""#, ")", ")", r#"T" d.""#])] +// should support mdx jsx (text) as `MdxJsxTextElement`s in mdast (namespace in tag name) +#[case("<a:b />.", &["text <a:b>(", ")", r#"T".""#])] +// should support mdx jsx (text) as `MdxJsxTextElement`s in mdast (members in tag name) +#[case("<a.b.c />.", &["text <a.b.c>(", ")", r#"T".""#])] +// should support mdx jsx (text) as `MdxJsxTextElement`s in mdast (attribute expression) +#[case("<a {...b} />.", &["text <a ...{...b}>(", ")", r#"T".""#])] +// should support mdx jsx (text) as `MdxJsxTextElement`s in mdast (property names) +#[case("<a b c:d />.", &["text <a b c:d>(", ")", r#"T".""#])] +// should support mdx jsx (text) as `MdxJsxTextElement`s in mdast (attribute values) +#[case(r#"<a b='c' d="e" f={g} />."#, &[r#"text <a b=lit:"c" d=lit:"e" f=expr:{g}>("#, ")", r#"T".""#])] +// should support character references (HTML 4, named) in JSX attribute values +#[case("<a b='&nbsp; &amp; &copy; &AElig; &Dcaron; &frac34; &HilbertSpace; &DifferentialD; &ClockwiseContourIntegral; &ngE;' />.", &[r#"text <a b=lit:"  & © Æ Ď ¾ ℋ ⅆ ∲ ≧̸">("#, ")", r#"T".""#])] +// should support character references (numeric) in JSX attribute values +#[case("<a b='&#35; &#1234; &#992; &#0;' c='&#X22; &#XD06; &#xcab;' />.", &["text <a b=lit:\"# Ӓ Ϡ �\" c=lit:\"\\\" ആ ಫ\">(", ")", r#"T".""#])] +// should not support things that look like character references but aren’t +#[case("<a b='&nbsp &x; &#; &#x; &#987654321; &#abcdef0; &ThisIsNotDefined; &hi?;' />.", &[r#"text <a b=lit:"&nbsp &x; &#; &#x; � &#abcdef0; &ThisIsNotDefined; &hi?;">("#, ")", r#"T".""#])] +// should support unicode whitespace in a lot of places +#[case("<a b  c  d />.", &["text <a b c d>(", ")", r#"T".""#])] +// should support line endings in a lot of places +#[case("<a\nb \nc\n d\n/>.", &["text <a b c d>(", ")", r#"T".""#])] +// should support a self-closing element +#[case("a <b /> c", &[r#"T"a ""#, "text <b>(", ")", r#"T" c""#])] +// should support an attribute expression +// Differs from the official packages: Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted. +#[case("a <b {1 + 1} /> c", &[r#"T"a ""#, "text <b ...{1 + 1}>(", ")", r#"T" c""#])] +// should support an attribute value expression +#[case("a <b c={1 + 1} /> d", &[r#"T"a ""#, "text <b c=expr:{1 + 1}>(", ")", r#"T" d""#])] +// should support an attribute expression +#[case("a <b {...c} /> d", &[r#"T"a ""#, "text <b ...{...c}>(", ")", r#"T" d""#])] +// should support more complex attribute expression (1) +#[case("a <b {...{c: 1, d: Infinity, e: false}} /> f", &[r#"T"a ""#, "text <b ...{...{c: 1, d: Infinity, e: false}}>(", ")", r#"T" f""#])] +// should support more complex attribute expression (2) +#[case("a <b {...[1, Infinity, false]} /> d", &[r#"T"a ""#, "text <b ...{...[1, Infinity, false]}>(", ")", r#"T" d""#])] +// should crash on an empty attribute value expression +// Differs from the official packages: Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted. +#[case("a <b c={} /> d", &[r#"T"a ""#, "text <b c=expr:{}>(", ")", r#"T" d""#])] +// should crash on invalid JS in an attribute value expression +// Differs from the official packages: Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted. +#[case("a <b c={?} /> d", &[r#"T"a ""#, "text <b c=expr:{?}>(", ")", r#"T" d""#])] +// should crash on invalid JS in an attribute expression +// Differs from the official packages: Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted. +#[case("a <b {?} /> c", &[r#"T"a ""#, "text <b ...{?}>(", ")", r#"T" c""#])] +// should support parenthesized expressions +#[case("a <b c={(2)} d={<e />} /> f", &[r#"T"a ""#, "text <b c=expr:{(2)} d=expr:{<e />}>(", ")", r#"T" f""#])] +// should support non-ascii identifier start characters +#[case("a <π /> b.", &[r#"T"a ""#, "text <π>(", ")", r#"T" b.""#])] +// should support non-ascii identifier continuation characters +#[case("a <a‌b /> b.", &[r#"T"a ""#, "text <a‌b>(", ")", r#"T" b.""#])] +// should support dashes in names +#[case("a <a-->b</a-->.", &[r#"T"a ""#, "text <a-->(", r#"T"b""#, ")", r#"T".""#])] +// should support dots in names for method names +#[case("a <abc . def.ghi>b</abc.def . ghi>.", &[r#"T"a ""#, "text <abc.def.ghi>(", r#"T"b""#, ")", r#"T".""#])] +// should support colons in names for local names +#[case("a <svg: rect>b</ svg :rect>.", &[r#"T"a ""#, "text <svg:rect>(", r#"T"b""#, ")", r#"T".""#])] +// should support attribute expressions +#[case("a <b {...props} {...rest}>c</b>.", &[r#"T"a ""#, "text <b ...{...props} ...{...rest}>(", r#"T"c""#, ")", r#"T".""#])] +// should support nested balanced braces in attribute expressions +#[case(r#"a <b {...{"a": "b"}}>c</b>."#, &[r#"T"a ""#, r#"text <b ...{...{"a": "b"}}>("#, r#"T"c""#, ")", r#"T".""#])] +// should support attribute expressions directly after a name +#[case("<a{...b}/>.", &["text <a ...{...b}>(", ")", r#"T".""#])] +// should support attribute expressions directly after a member name +#[case("<a.b{...c}/>.", &["text <a.b ...{...c}>(", ")", r#"T".""#])] +// should support attribute expressions directly after a local name +#[case("<a:b{...c}/>.", &["text <a:b ...{...c}>(", ")", r#"T".""#])] +// should support attribute expressions directly after boolean attributes +#[case("a <b c{...d}/>.", &[r#"T"a ""#, "text <b c ...{...d}>(", ")", r#"T".""#])] +// should support attribute expressions directly after boolean qualified attributes +#[case("a <b c:d{...e}/>.", &[r#"T"a ""#, "text <b c:d ...{...e}>(", ")", r#"T".""#])] +// should support attribute expressions and normal attributes +#[case("a <b a {...props} b>c</b>.", &[r#"T"a ""#, "text <b a ...{...props} b>(", r#"T"c""#, ")", r#"T".""#])] +// should support attributes +#[case("a <b c d=\"d\"\t\tefg=\"e\">c</b>.", &[r#"T"a ""#, r#"text <b c d=lit:"d" efg=lit:"e">("#, r#"T"c""#, ")", r#"T".""#])] +// should support attribute value expressions +#[case("a <b c={1 + 1}>c</b>.", &[r#"T"a ""#, "text <b c=expr:{1 + 1}>(", r#"T"c""#, ")", r#"T".""#])] +// should support nested balanced braces in attribute value expressions +#[case("a <b c={1 + ({a: 1}).a}>c</b>.", &[r#"T"a ""#, "text <b c=expr:{1 + ({a: 1}).a}>(", r#"T"c""#, ")", r#"T".""#])] +// should support an attribute directly after a value +#[case(r#"<a b=""c/>."#, &[r#"text <a b=lit:"" c>("#, ")", r#"T".""#])] +// should support an attribute directly after an attribute expression +#[case("<a{...b}c/>.", &["text <a ...{...b} c>(", ")", r#"T".""#])] +// should support whitespace directly after closing slash +#[case("<a/ \t>.", &["text <a>(", ")", r#"T".""#])] +// should *not* crash on closing angle in text +#[case("a > c.", &[r#"T"a > c.""#])] +// should *not* crash on opening angle in tick code in an element +#[case("a <>`<`</> c.", &[r#"T"a ""#, "text <>(", r#"code_inline "<""#, ")", r#"T" c.""#])] +// should *not* crash on ticks in tick code in an element +#[case("a <>`` ``` ``</>", &[r#"T"a ""#, "text <>(", r#"code_inline "```""#, ")"])] +// should support nested tags +#[case("a <>b <>c</> d</>.", &[r#"T"a ""#, "text <>(", r#"T"b ""#, "text <>(", r#"T"c""#, ")", r#"T" d""#, ")", r#"T".""#])] +// should support character references in attribute values +#[case(r#"<x y="Character references can be used: &quot;, &apos;, &lt;, &gt;, &#x7B;, and &#x7D;, they can be named, decimal, or hexadecimal: &copy; &#8800; &#x1D306;" />."#, &[r#"text <x y=lit:"Character references can be used: \", ', <, >, {, and }, they can be named, decimal, or hexadecimal: © ≠ 𝌆">("#, ")", r#"T".""#])] +// should support character references in text +#[case("<x>Character references can be used: &quot;, &apos;, &lt;, &gt;, &#x7B;, and &#x7D;, they can be named, decimal, or hexadecimal: &copy; &#8800; &#x1D306;</x>.", &["text <x>(", r#"T"Character references can be used: \", ', <, >, {, and }, they can be named, decimal, or hexadecimal: © ≠ 𝌆""#, ")", r#"T".""#])] +// should support as text if the closing tag is not the last thing +#[case("<x />.", &["text <x>(", ")", r#"T".""#])] +// should support as text if the opening is not the first thing +#[case("a <x />", &[r#"T"a ""#, "text <x>(", ")"])] +// should support line endings in elements +#[case("> a <b>\n> c </b> d.", &["blockquote(", r#"T"a ""#, "text <b>(", r#"T"\nc ""#, ")", r#"T" d.""#, ")"])] +// should support line endings in attribute values +#[case("> a <b c=\"d\ne\" /> f", &["blockquote(", r#"T"a ""#, r#"text <b c=lit:"d\ne">("#, ")", r#"T" f""#, ")"])] +// should support line endings in attribute value expressions +// Differs from the official packages: Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted. +#[case("> a <b c={d\ne} /> f", &["blockquote(", r#"T"a ""#, "text <b c=expr:{d\ne}>(", ")", r#"T" f""#, ")"])] +// should support line endings in attribute expressions +// Differs from the official packages: Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted. +#[case("> a <b {c\nd} /> e", &["blockquote(", r#"T"a ""#, "text <b ...{c\nd}>(", ")", r#"T" e""#, ")"])] +// should support lazy text (1) +#[case("> a <b\n/> c", &["blockquote(", r#"T"a ""#, "text <b>(", ")", r#"T" c""#, ")"])] +// should support lazy text (2) +#[case("> a <b c='\nd'/> e", &["blockquote(", r#"T"a ""#, r#"text <b c=lit:"\nd">("#, ")", r#"T" e""#, ")"])] +// should support lazy text (3) +#[case("> a <b c='d\n'/> e", &["blockquote(", r#"T"a ""#, r#"text <b c=lit:"d\n">("#, ")", r#"T" e""#, ")"])] +// should support lazy text (4) +#[case("> a <b c='d\ne'/> f", &["blockquote(", r#"T"a ""#, r#"text <b c=lit:"d\ne">("#, ")", r#"T" f""#, ")"])] +// should support lazy text (5) +// Differs from the official packages: Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted. +#[case("> a <b c={d\ne}/> f", &["blockquote(", r#"T"a ""#, "text <b c=expr:{d\ne}>(", ")", r#"T" f""#, ")"])] +// should allow `<` followed by markdown whitespace as text in markdown +#[case("1 < 3", &[r#"T"1 < 3""#])] +// From `micromark-extension-mdx-jsx`. +// should support attributes +#[case("a <b c d=\"d\"\t\tefg='e'>c</b>.", &[r#"T"a ""#, r#"text <b c d=lit:"d" efg=lit:"e">("#, r#"T"c""#, ")", r#"T".""#])] +#[case("> <a b={`\n>\t`}/>", &["blockquote(", "flow <a b=expr:{`\n`}>(", ")", ")"])] +#[case("> <a b={`\n> alpha\n> bravo\n> charlie\n> delta\n> `}/>", &["blockquote(", "flow <a b=expr:{`\nalpha\nbravo\ncharlie\n delta\n`}>(", ")", ")"])] +// From `micromark-extension-mdx-expression`. +// should not support JSX by default +#[case("a {<b />} c", &[r#"T"a ""#, "textexpr {<b />}", r#"T" c""#])] +// should support `acornOptions` (1) +#[case("a {(() => {})()} c", &[r#"T"a ""#, "textexpr {(() => {})()}", r#"T" c""#])] +// should support `acornOptions` (2) +#[case("a {(function () {})()} c", &[r#"T"a ""#, "textexpr {(function () {})()}", r#"T" c""#])] +#[case("a {} c", &[r#"T"a ""#, "textexpr {}", r#"T" c""#])] +// should support `\0` and `\r` in expressions +// Differs from the official packages: A null character is kept in values, as in markdown-rs, and replaced with U+FFFD only in HTML; the JavaScript packages replace it in values too. +#[case("{`a\0b\rc\nd\r\ne`}", &["flowexpr {`a\0b\rc\nd\r\ne`}"])] +// should support a spread +// Differs from the official packages: Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted. +#[case("a {...b} c", &[r#"T"a ""#, "textexpr {...b}", r#"T" c""#])] +// should crash on an incorrect spread +// Differs from the official packages: Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted. +#[case("a {...?} c", &[r#"T"a ""#, "textexpr {...?}", r#"T" c""#])] +#[case("a {b=c}={} d", &[r#"T"a ""#, "textexpr {b=c}", r#"T"=""#, "textexpr {}", r#"T" d""#])] +// should crash if a spread and other things +// Differs from the official packages: Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted. +#[case("a {...b,c} d", &[r#"T"a ""#, "textexpr {...b,c}", r#"T" d""#])] +// should crash if not an identifier +#[case("a {b=c} d", &[r#"T"a ""#, "textexpr {b=c}", r#"T" d""#])] +#[case("a {/* b */} c", &[r#"T"a ""#, "textexpr {/* b */}", r#"T" c""#])] +#[case("{a=b}", &["flowexpr {a=b}"])] +// From `micromark-extension-mdxjs-esm`. +// should throw if `acorn` is not passed in +#[case("import a from \"b\"\n\nc", &[r#"esm "import a from \"b\"""#, r#"T"c""#])] +#[case(r#"import("a")"#, &[r#"T"import(\"a\")""#])] +// should not support an indent +#[case(" import a from \"b\"\n export default c", &[r#"T"import a from \"b\"\nexport default c""#])] +// should not support keywords in containers +#[case("- import a from \"b\"\n> export default c", &["item 0 u - t(", r#"T"import a from \"b\"""#, ")", "blockquote(", r#"T"export default c""#, ")"])] +#[case("import a from \"b\"\nexport default c", &[r#"esm "import a from \"b\"\nexport default c""#])] +#[case("import a from \"b\"\n\nexport default c", &[r#"esm "import a from \"b\"""#, r#"esm "export default c""#])] +#[case("a\n\nimport a from \"b\"\n\nb\n\nexport default c", &[r#"T"a""#, r#"esm "import a from \"b\"""#, r#"T"b""#, r#"esm "export default c""#])] +#[case("a\nimport a from \"b\"\n\nb\nexport default c", &[r#"T"a\nimport a from \"b\"""#, r#"T"b\nexport default c""#])] +#[case("export {\n a\n} from \"b\"\n\nc", &[r#"esm "export {\n a\n} from \"b\"""#, r#"T"c""#])] +#[case("export {\n\n a\n\n} from \"b\"\n\nc", &[r#"esm "export {\n\n a\n\n} from \"b\"""#, r#"T"c""#])] +// Differs from the official packages: Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted. +#[case("import a from \"b\"\n*md*?", &[r#"esm "import a from \"b\"\n*md*?""#])] +// Differs from the official packages: Expressions and ESM are not parsed as JavaScript, so invalid JavaScript is accepted. +#[case("import (\"a\")\n\nb", &[r#"esm "import (\"a\")""#, r#"T"b""#])] +#[case("import a from \"b\"\nexport {a}\n\nc", &[r#"esm "import a from \"b\"\nexport {a}""#, r#"T"c""#])] +#[case("import a from \"b\";\nexport {a};\n\nc", &[r#"esm "import a from \"b\";\nexport {a};""#, r#"T"c""#])] +#[case("import a from \"b\"\nexport {a as default}\n\nc", &[r#"esm "import a from \"b\"\nexport {a as default}""#, r#"T"c""#])] +#[case("export var a = () => <b />\n\nc", &[r#"esm "export var a = () => <b />""#, r#"T"c""#])] +// should support `acornOptions` (1) +#[case("export var a = () => {}\n\nb", &[r#"esm "export var a = () => {}""#, r#"T"b""#])] +#[case("import a from \"b\"\n\nexport {a}\n\nc", &[r#"esm "import a from \"b\"""#, r#"esm "export {a}""#, r#"T"c""#])] +#[case("import a from \"b\"\n\nexport {a}\n\n# c", &[r#"esm "import a from \"b\"""#, r#"esm "export {a}""#, "heading 1(", r#"T"c""#, ")"])] +fn mdx(#[case] input: &str, #[case] expected: &[&str]) { + check(input, expected, Markdown::from_mdx_str); +} + +#[rstest] +// Hand-written. +#[case("<a>b")] +#[case("<a></b>")] +#[case("a <3")] +#[case("<a b=>")] +#[case("a {")] +#[case("{")] +#[case(r#"<a b=""#)] +#[case("<a b={")] +#[case("</a>")] +#[case("<a></a></a>")] +// should crash when building the ast on mismatched interleaving (4) +#[case("<a>")] +#[case("<a><b></a></b>")] +#[case("a <b>c")] +#[case("<a>b\n</a>")] +#[case("<a>\nb</a>")] +#[case("<a:b.c />")] +#[case("<a.b:c />")] +#[case("<1 />")] +#[case("<a b:c:d />")] +#[case("<a ={b} />")] +#[case("<a b=c />")] +#[case("</>")] +#[case("<></a>")] +#[case("<!-- a -->")] +#[case("<a>\n<!-- b -->\n</a>")] +#[case("<https://a.b>")] +#[case("<a@b.c>")] +#[case(r#"<a b="c"#)] +#[case(r#"x <a b="c"#)] +#[case("<a>\n<b")] +#[case("<a>\n\n<b")] +#[case("<a>x</a")] +#[case("<a>x</")] +#[case("<a>x<")] +#[case("a {b")] +#[case("a {b\n\nc}")] +#[case("x {a}\n\ny {")] +// From markdown-rs, `tests/mdx_expression_flow.rs`. +// should crash if no closing brace is found (1) +#[case("{a")] +// should crash if no closing brace is found (2) +#[case("{b { c }")] +// should not support lazyness (1) +#[case("> {a\nb}")] +// should not support lazyness (4) +#[case("> {\n> a\nb}")] +// should crash on an incorrect spread that looks like an assignment +#[case("<a {b=c}={} d>")] +// should crash if a spread and other things +#[case("<a {...b,c} d>")] +// From markdown-rs, `tests/mdx_expression_text.rs`. +// should crash on an incorrect line comment (1) +#[case("a {//} b")] +// should crash on an incorrect line comment (2) +#[case("a { // b } c")] +// should crash if no closing brace is found (1) +#[case("a {b c")] +// should crash if no closing brace is found (2) +#[case("a {b { c } d")] +// From markdown-rs, `tests/mdx_jsx_flow.rs`. +// should handle crash in containers gracefully +#[case("* <!a>\n1. b")] +// should not support lazy flow (1) +#[case("> <X\n/>")] +// should not support lazy flow (2) +#[case("> a\n> <X\n/>")] +// should not support lazy flow (3) +#[case("> <a b='\nc'/>")] +// should not support lazy flow (4) +#[case("> <a b='c\n'/>")] +// should not support lazy flow (5) +#[case("> <a b='c\nd'/>")] +// should not support lazy flow (6) +#[case("> <a b={c\nd}/>")] +// should not support lazy flow (7) +#[case("> <a {b\nc}/>")] +// From markdown-rs, `tests/mdx_jsx_text.rs`. +// should support mdx jsx (text) if enabled +#[case("a <b> c")] +// should crash when building the ast on a closing tag if none is open +#[case("a </b> c")] +// should crash when building the ast on a closing tag with a self-closing slash +#[case("a <b> c </b/> d")] +// should crash when building the ast on a closing tag with an attribute +#[case("a <b> c </b d> e")] +// should crash when building the ast on mismatched tags (1) +#[case("a <>b</c> d")] +// should crash when building the ast on mismatched tags (2) +#[case("a <b>c</> d")] +// should crash when building the ast on mismatched interleaving (1) +#[case("*a <b>c* d</b>.")] +// should crash when building the ast on mismatched interleaving (2) +#[case("<a>b *c</a> d*.")] +// should crash when building the ast on mismatched interleaving (3) +#[case("a <b>.")] +// should crash on unclosed jsx after closed jsx +#[case("<a><b></b>")] +// should crash on invalid JS in an attribute expression (2) +#[case("a <b{c=d}={}/> f")] +// should support an unclosed fragment +#[case("a <> c")] +// should *not* support whitespace in the opening tag (fragment) +#[case("a < \t>b</>")] +// should *not* support whitespace in the opening tag (named) +#[case("a < \nb\t>b</b>")] +// should crash on a nonconforming start identifier +#[case("a <!> b")] +// should crash on a nonconforming start identifier in a closing tag +#[case("a </(> b.")] +// should crash on non-conforming non-ascii identifier start characters +#[case("a <© /> b.")] +// should crash nicely on what might be a comment +#[case("a <!--b-->")] +// should crash nicely on JS line comments inside tags (1) +#[case("a <// b\nc/>")] +// should crash nicely JS line comments inside tags (2) +#[case("a <b// c\nd/>")] +// should crash nicely JS multiline comments inside tags (1) +#[case("a </*b*/c>")] +// should crash nicely JS multiline comments inside tags (2) +#[case("a <b/*c*/>")] +// should crash on non-conforming non-ascii identifier continuation characters +#[case("a <a¬ /> b.")] +// should crash nicely on what might be an email link +#[case("a <b@c.d>")] +// should crash on nonconforming identifier continuation characters +#[case("a <a?> c.")] +// should crash nicely on what might be an email link in member names +#[case("a <b.c@d.e>")] +// should crash on a nonconforming character to start a local name +#[case("a <a:+> c.")] +// should crash nicely on what might be a protocol in local names +#[case("a <http://example.com>")] +// should crash nicely on what might be a protocol in local names +#[case("a <http: >")] +// should crash on a nonconforming character in a local name +#[case("a <a:b|> c.")] +// should crash on a nonconforming character to start a member name +#[case("a <a..> c.")] +// should crash on a nonconforming character in a member name +#[case("a <a.b,> c.")] +// should crash on a nonconforming character after a local name +#[case("a <a:b .> c.")] +// should crash on a nonconforming character after a member name +#[case("a <a.b :> c.")] +// should crash on a nonconforming character after name +#[case("a <a => c.")] +// should crash on a nonconforming character before an attribute name +#[case("a <b {...p}~>c</b>.")] +// should crash on a missing closing brace in attribute expression +#[case("a <b {...")] +// should crash on a nonconforming character in attribute name +#[case("a <a b@> c.")] +// should crash on a nonconforming character after an attribute name +#[case("a <a b 1> c.")] +// should crash on a nonconforming character to start a local attribute name +#[case("a <a b:#> c.")] +// should crash on a nonconforming character in a local attribute name +#[case("a <a b:c%> c.")] +// should crash on a nonconforming character after a local attribute name +#[case("a <a b:c ^> c.")] +// should crash on a nonconforming character before an attribute value +#[case("a <a b=``> c.")] +// should crash nicely on what might be a fragment, element as prop value +#[case("a <a b=<c />> d.")] +// should crash on a missing closing quote in double quoted attribute value +#[case(r#"a <a b="> c."#)] +// should crash on a missing closing brace in an attribute value expression +#[case("a <a b={> c.")] +// should crash on a nonconforming character after an attribute value +#[case(r#"a <a b=""*> c."#)] +// should crash on a nonconforming character after a self-closing slash +#[case("a <a/b> c.")] +// should support a closing tag w/o open elements +#[case("a </> c.")] +// should support mismatched tags (1) +#[case("a <></b>")] +// should support mismatched tags (2) +#[case("a <b></>")] +// should support mismatched tags (3) +#[case("a <a.b></a>")] +// should support mismatched tags (4) +#[case("a <a></a.b>")] +// should support mismatched tags (5) +#[case("a <a.b></a.c>")] +// should support mismatched tags (6) +#[case("a <a:b></a>")] +// should support mismatched tags (7) +#[case("a <a></a:b>")] +// should support mismatched tags (8) +#[case("a <a:b></a:c>")] +// should support mismatched tags (9) +#[case("a <a:b></a.b>")] +// should support a closing self-closing tag +#[case("a <a>b</a/>")] +// should support a closing tag w/ attributes +#[case("a <a>b</a b>")] +// should not care about precedence between attention (emphasis) +#[case("a *open <b> close* </b> c.")] +// should not care about precedence between attention (strong) +#[case("a **open <b> close** </b> c.")] +// should not care about precedence between label (link) +#[case("a [open <b> close](c) </b> d.")] +// should not care about precedence between label (image) +#[case("a ![open <b> close](c) </b> d.")] +// From `micromark-extension-mdx-jsx`. +#[case("a <a></(> b.")] +#[case("a <a b='> c.")] +// should allow line endings in whitespace +#[case("a <b \n c> d.")] +fn invalid_mdx(#[case] input: &str) { + assert!(Markdown::from_mdx_str(input).is_err(), "{input:?}"); +} + +#[rstest] +// Hand-written. +#[case("# a\n\n- x\n - y\n- [ ] z\n\n1. q\n", &["heading 1(", r#"T"a""#, ")", "item 0 u - t(", r#"T"x""#, ")", "item 1 u - t(", r#"T"y""#, ")", "item 0 u _ t(", r#"T"z""#, ")", "item 0 o1 - t(", r#"T"q""#, ")"])] +#[case("| a | b |\n|:-|-:|\n| 1 | 2 |\n", &["cell 0 0(", r#"T"a""#, ")", "cell 0 1(", r#"T"b""#, ")", r#"align ["left","right"]"#, "cell 1 0(", r#"T"1""#, ")", "cell 1 1(", r#"T"2""#, ")"])] +#[case("> t *e* **s** ~~d~~ `c` [l](u \"t\") ![i](u) [r][d] <b> a \nb\n\n[d]: /u\n", &["blockquote(", r#"T"t ""#, "em(", r#"T"e""#, ")", r#"T" ""#, "strong(", r#"T"s""#, ")", r#"T" ""#, "del(", r#"T"d""#, ")", r#"T" ""#, r#"code_inline "c""#, r#"T" ""#, "link u t(", r#"T"l""#, ")", r#"T" ""#, r#"image u "i""#, r#"T" ""#, "linkref d(", r#"T"r""#, ")", r#"T" ""#, r#"html "<b>""#, r#"T" a""#, "br", r#"T"b""#, ")", "definition d /u "])] +#[case("[^1]\n\n[^1]: note\n", &["footnoteref 1", "footnote 1(", r#"T"note""#, ")"])] +#[case("---\ntitle: x\n---\n\n# a\n", &[r#"yaml "title: x""#, "heading 1(", r#"T"a""#, ")"])] +#[case("+++\ntitle = 1\n+++\n\na\n", &[r#"toml "title = 1""#, r#"T"a""#])] +#[case("$$\nm\n$$\n\n$i$\n", &[r#"math "m""#, r#"imath "i""#])] +#[case("```rs x\nc\n```\n", &[r#"code rs x "c""#])] +#[case("a\n\nb\n", &[r#"T"a""#, r#"T"b""#])] +#[case("*a*\n\n*b*\n", &["em(", r#"T"a""#, ")", "em(", r#"T"b""#, ")"])] +fn markdown(#[case] input: &str, #[case] expected: &[&str]) { + check(input, expected, Markdown::from_markdown_str); +} diff --git a/justfile b/justfile index 399f7e7c8..8ddf67b24 100644 --- a/justfile +++ b/justfile @@ -128,14 +128,6 @@ test-gfm-spec: test-spec: cargo test -p mq-markdown --lib -- --ignored spec_examples -# Compare the nodes of Markdown and MDX with the mdast of the official packages (installs them with npm and -# fetches the specs) -test-mdast-official: - cd crates/mq-markdown/tests/mdast_official && npm install --silent - node crates/mq-markdown/tests/mdast_official/corpus.mjs target/mdast_corpus.json - MDAST_CORPUS="$PWD/target/mdast_corpus.json" MDAST_OUT="$PWD/target/mdast_nodes.json" cargo test -p mq-markdown --features json --test mdast_official -- --ignored - node crates/mq-markdown/tests/mdast_official/compare.mjs target/mdast_corpus.json target/mdast_nodes.json - test-doc: cargo test --doc --workspace From 4838fddcbb38d33ffaf11d827689cca401893d6f Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Fri, 9 Oct 2026 00:37:07 +0000 Subject: [PATCH 53/88] =?UTF-8?q?=E2=9C=85=20test(markdown):=20replace=20t?= =?UTF-8?q?he=20parser=20snapshot=20file=20with=20rstest=20cases?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cases of the parser lived in `parser.rs` and their expected nodes and HTML in `snapshots.txt`, written from the Debug form of the nodes and compared as one test. Each case is now an rstest case in `tests/parser_tree_tests.rs` with its input, HTML and nodes side by side. Nodes are written as an indented tree of kinds, non-default fields and positions, so a field added to a node does not rewrite every case. --- crates/mq-markdown/src/parser.rs | 544 +-- crates/mq-markdown/src/parser/snapshots.txt | 1324 ------- crates/mq-markdown/tests/parser_tree_tests.rs | 3441 +++++++++++++++++ 3 files changed, 3445 insertions(+), 1864 deletions(-) delete mode 100644 crates/mq-markdown/src/parser/snapshots.txt create mode 100644 crates/mq-markdown/tests/parser_tree_tests.rs diff --git a/crates/mq-markdown/src/parser.rs b/crates/mq-markdown/src/parser.rs index b56844538..a128dfcc1 100644 --- a/crates/mq-markdown/src/parser.rs +++ b/crates/mq-markdown/src/parser.rs @@ -65,552 +65,16 @@ mod tests { use proptest::prelude::*; use proptest::test_runner::{RngAlgorithm, TestRng, TestRunner}; - const MDX_CASES: &[(&str, &str)] = &[ - ("jsx_self", "<a />"), - ("jsx_attrs", "<a b=\"c&amp;d\" e='f' g={h} {...i} j />"), - ("jsx_member", "<a.b.c />"), - ("jsx_namespace", "<a:b c:d=\"e\" />"), - ("jsx_fragment", "<></>"), - ("jsx_text_pair", "<a>x</a>"), - ("jsx_flow_pair", "<a>\n\nx\n\n</a>"), - ("jsx_flow_indented_child", "<a>\n x\n</a>"), - ("jsx_inline", "a <b>c</b> d"), - ("jsx_inline_self", "a <b/> d"), - ("expr_flow", "{a}"), - ("expr_nested", "{a {b} c}"), - ("expr_multiline", "{a\nb}"), - ("expr_multiline_indented", "{a\n b\n c}"), - ("expr_text", "x {a} y"), - ("expr_then_tag", "{a} <b/>"), - ("tag_then_expr", "<b/> {a}"), - ("tag_then_text", "<b/> x"), - ("expr_then_text", "{a}x"), - ("jsx_in_expr_child", "<a>{b}</a>"), - ("jsx_unclosed_text", "<a>b"), - ("jsx_mismatch", "<a></b>"), - ("lt_space", "a < b"), - ("lt_digit", "a <3"), - ("attr_no_value", "<a b=>"), - ("attr_space_after_eq", "<b e= \"f\"/>"), - ("attr_spaces_around_eq", "a <b e = \"f\"/>."), - ("attr_line_before_eq", "<b c\n= \"x\">c</b>"), - ("attr_namespace_spaces", "a <b xml :\tlang\n= \"de-CH\" foo:bar>c</b>."), - ("attr_spaced_names", "a <b a b : c d : e = \"f\" g/>."), - ("attr_expr_space_after_eq", "<b e= {f}/>"), - ("jsx_nested_flow", "<a>\n<b>\n</b>\n</a>"), - ("jsx_in_quote", "> <a>\n> x\n> </a>"), - ("jsx_in_list", "- <a>\n x\n </a>"), - ("attr_quote_inside", "<a b='c\"d' />"), - ("two_tags", "<a/><b/>"), - ("two_text_elements", "<a>x</a><b>y</b>"), - ("interrupt_paragraph_tag", "a\n<b/>"), - ("interrupt_paragraph_expr", "a\n{b}"), - ("two_expressions", "{a}\n{b}"), - ("expr_in_flow_element", "<a>\n\n{b}\n\n</a>"), - ("indented_tag", " <a/>"), - ("very_indented_tag", " <a/>"), - ("heading_jsx", "# h <a>b</a>"), - ("emphasis_crossing", "*<a>x</a>*"), - ("emphasis_crossing_bad", "*<a>x*</a>"), - ("link_jsx", "[<a>x</a>](y)"), - ("code_not_jsx", "`<a/>` {b}"), - ("escaped", "\\<a/> \\{b}"), - ("attr_entities", "<a b=\"&lt;&#x41;\" />"), - ("attr_multiline_literal", "<a b='x\ny' />"), - ("closing_spaces", "<a >x</a >"), - ("lt_space_name", "< a>"), - ("self_closing_space", "<a / >"), - ("attr_on_next_line", "<a\nb />"), - ("dashed_name", "<a-b />"), - ("dashed_attr", "<a b-c=\"d\" />"), - ("bad_name_char", "<A_b$.c-d />"), - ("expr_unclosed", "{a"), - ("expr_empty", "{}"), - ("expr_space", "{ }"), - ("empty_flow_element", "<a>\n</a>"), - ("text_element_unclosed_line", "<a>x\n</a>"), - ("stray_close", "</a>"), - ("stray_close_text", "x </a>"), - ("element_with_emphasis", "<a>*b*</a>"), - ("flow_element_with_emphasis", "<a>\n*b*\n</a>"), - ("flow_element_with_list", "<a>\n- b\n</a>"), - ("list_items_unbalanced", "- <a>\n- </a>"), - ("spaced_equals", "<a b = \"c\" />"), - ("heading", "# h\n\ntext *a* [b](c) `d`\n"), - ("indented_is_text", " not code\n"), - ("indented_heading", " # h\n"), - ("no_autolink", "<http://a.b>"), - ("no_gfm", "| a |\n|-|\n\n~a~ www.a.b [^a]\n\n- [ ] a\n"), - ("no_math", "$a$\n\n$$\na\n$$\n"), - ("frontmatter", "---\na\n---\n"), - ("frontmatter_toml", "+++\na = 1\n+++\n\n<A />"), - ("frontmatter_unclosed", "---\na\n"), - ("esm", "import a from 'b'\n\nexport const c = 1\n"), - ("definition", "[a]: /u\n\n[a]\n"), - ("list_deep_indent", "- a\n"), - ("fence", "```rust\ncode\n```\n"), - ("lone_cr_after_lt", "<\r/-x"), - ("crlf_after_lt", "<\r\n/-x"), - ("expression_lone_cr", "{\r }"), - ("expression_cr_indent", "{a\r b\r\n c\n d}"), - ("tilde_not_delimiter", "*~*a"), - ("tilde_strong_not_delimiter", "**~**あ"), - ("dollar_after_name", "<a$/>"), - ("dollar_after_space", "<a $b/>"), - ("cjk_tag_name", "<あ b=\"c\" />"), - ("fullwidth_underscore_name", "<a_b />"), - ("jsx_text_cjk", "あ <b>い</b> う"), - ("expression_cjk", "{あ} い"), - ]; - - const CASES: &[(&str, &str)] = &[ - ("empty", ""), - ("blank_lines", "\n\n \n"), - ("paragraph", "hello\n"), - ("paragraph_no_eol", "hello"), - ("paragraph_multiline", "a\nb\nc\n"), - ("paragraph_indented_continuation", "a\n b\n"), - ("paragraph_trailing_space", "あい\n うえ \n"), - ("two_paragraphs", "a\n\nb\n"), - ("crlf", "a\r\nb\r\n\r\nc\r\n"), - ("crlf_fence", "```\r\na\r\nb\r\n```\r\n"), - ("cr_only", "a\rb\r"), - ("atx_h1", "# title\n"), - ("atx_h6", "###### title\n"), - ("atx_seven", "####### title\n"), - ("atx_no_space", "#title\n"), - ("atx_empty", "#\n"), - ("atx_closing", "## title ##\n"), - ("atx_closing_no_space", "## title##\n"), - ("atx_indent", " ## title ## \n"), - ("atx_multibyte", "# あ h #\n"), - ("atx_interrupts_paragraph", "a\n# b\n"), - ("setext_h1", "Title\n===\n"), - ("setext_h2", "a\n ---\n"), - ("setext_multiline", "a\nb\n---\n"), - ("thematic_star", "***\n"), - ("thematic_dash_spaced", "- - -\n"), - ("thematic_underscore", " ___ \n"), - ("thematic_two", "**\n"), - ("fence_lang_meta", "```rust title\nlet a;\n```\n"), - ("fence_indented", " ```\n a\n b\n ```\n"), - ("fence_tilde", "~~~\na\n~~~\n"), - ("fence_unclosed", "```\na\n"), - ("fence_empty", "para\n\n\n```\n```\n"), - ("fence_longer_close", "```\na\n`````\n"), - ("fence_shorter_close", "````\na\n```\nb\n````\n"), - ("fence_blank_inside", "```\na\n\nb\n```\n"), - ("fence_backtick_info", "``` a`b\nc\n"), - ("fence_interrupts_paragraph", "a\n```\nb\n```\n"), - ("indented_code", " code\n\n more\n\nx\n"), - ("quote", "> a\n> b\n"), - ("quote_indented_lazy", " > a\nb\n"), - ("quote_heading", "> # h\n>\n> c\n"), - ("quote_no_space", ">a\n"), - ("quote_empty", ">\n"), - ("quote_nested", "> > a\n> b\n"), - ("quote_lazy_nested", "> > a\nb\n"), - ("quote_blank_ends", "> a\n\n> b\n"), - ("quote_fence", "> ```\n> a\n> ```\n"), - ("quote_no_lazy_after_fence", "> ```\n> a\nb\n"), - ("quote_interrupts_paragraph", "a\n> b\n"), - ("quote_thematic_not_lazy", "> a\n---\n"), - ("quote_multibyte", "> あ\n> い\n"), - ("bullet", "- a\n- b\n"), - ("bullet_loose", "- a\n\n- b\n"), - ("bullet_indent", " - a\n"), - ("bullet_mixed_markers", "* a\n+ b\n"), - ("ordered", "1. a\n2. b\n"), - ("ordered_paren", "5) a\n"), - ("ordered_zero", "0. a\n"), - ("ordered_too_long", "1234567890. a\n"), - ("ordered_mixed_delimiters", "1. a\n2) b\n"), - ("task", "- [ ] x\n- [x] y\n- [X] z\n"), - ("task_empty", "- [ ]\n"), - ("task_no_space", "- [ ]x\n"), - ("nested", "- a\n - b\n c\n"), - ("nested_ordered", "1. a\n 1. b\n 2. c\n2. d\n"), - ("nested_dedent", "- a\n - b\n- c\n"), - ("list_item_two_paragraphs", "- a\n\n b\n"), - ("two_paragraphs_then_item", "- a\n\n b\n- c\n"), - ("empty_item", "-\n"), - ("empty_item_content", "-\n foo\n"), - ("empty_item_blank_then_text", "-\n\n foo\n"), - ("empty_items", "-\n-\n"), - ("wide_marker_gap", "- a\n b\n"), - ("code_in_item", "- a\n"), - ("lazy", "- a\nb\n"), - ("lazy_after_blank", "- a\n\nb\n"), - ("list_interrupts_paragraph", "a\n- b\n"), - ("ordered_two_no_interrupt", "a\n2. b\n"), - ("ordered_one_interrupts", "a\n1. b\n"), - ("empty_item_no_interrupt", "a\n-\n"), - ("list_in_quote", "> - a\n> - b\n"), - ("quote_in_list", "- > a\n > b\n"), - ("quote_in_list_level", "- a\n > - b\n"), - ("heading_in_item", "- # h\n text\n"), - ("fence_in_item", "- ```\n a\n ```\n"), - ("multibyte_item", "- あ\n- い\n"), - ("list_after_paragraph_blank", "a\n\n- b\n"), - ("two_lists_split", "- a\n\n\n- b\n"), - ("deep_quote", ">>>>>> a\n"), - ("fence_unclosed_no_eol", "```\na"), - ("fence_unclosed_in_item", "- ```\n a\n"), - ("fence_unclosed_in_item_no_eol", "- ```\n a"), - ("fence_unclosed_in_quote_no_eol", "> ```\n> a"), - ("fence_unclosed_in_quote_eof", "> ```\n> a\n"), - ("fence_unclosed_trailing_blank", "```\na\n\n"), - ("fence_unclosed_item_then_para", "- ```\n a\nb\n"), - ("item_code_then_unindented_text", "- ===\n\n a\n==="), - ("quote_setext_dashes", "> a\n> ---\n> ---\n> a"), - ("table_basic", "| a | b |\n|---|:-:|\n| 1 | 2 |\n"), - ("table_no_edge_pipes", "a|b\n-|-\n1|2|3\n"), - ("table_short_row", "|a|b|\n|-|-|\n|1|\n"), - ("table_escaped_pipe", "| a \\| b | c |\n|--|--|\n"), - ("table_indented", " | a |\n |---|\n | b |\n"), - ("table_text_after", "| a |\n|---|\ntext after\n"), - ("table_in_quote", "> | a |\n> |---|\n> | b |\n"), - ("table_after_paragraph", "x\n| a |\n|---|\n"), - ("table_column_mismatch", "| a | b |\n|---|\n"), - ("table_bad_delimiter", "|a|\n|:|\n"), - ("table_then_blank_para", "|a|\n|-|\n\nnext\n"), - ("table_then_quote", "|a|\n|-|\n> q\n"), - ("table_then_heading", "|a|\n|-|\n# h\n"), - ("table_then_fence", "|a|\n|-|\n```\nx\n```\n"), - ("table_then_list", "|a|\n|-|\n- x\n"), - ("table_two", "|a|\n|-|\n\n|b|\n|-|\n"), - ("table_align_spaces", "|a|b|\n|:-|-:|\n| x | y|\n"), - ("table_indented_code_row", "|a|\n|-|\n|b|\n |c|\n"), - ("table_no_body", "| a |\n| - |\n"), - ("table_empty_cells", "||\n|-|\n"), - ("table_lone_pipe_row", "|a|\n|-|\n|\n"), - ("table_no_trailing_pipe", "| a | b\n|-|-\n| c | d\n"), - ("table_multibyte", "| あ | い |\n|---|---|\n| う | え |\n"), - ("not_table_setext", "Title\n---\n"), - ("not_table_no_pipe_delim", "a|b\n---\n"), - ("table_header_no_pipe", "a\n|-|\n"), - ("table_one_col_dash", "|a|\n-\n"), - ("inline_escape", "a\\*b"), - ("inline_escape_nonpunct", "a\\qb"), - ("inline_entities", "a&amp;b &copy; &#35; &#x41; &unknown; &"), - ("inline_emphasis_all", "a *b* **c** _d_ __e__ ***f***"), - ("inline_code", "`code` `` a`b `` ` a `"), - ("inline_code_unclosed", "`a ``b`"), - ("inline_hard_break_spaces", "a \nb"), - ("inline_hard_break_backslash", "a\\\nb"), - ("inline_soft_break", "a\nb"), - ("inline_soft_break_trailing_space", "a \nb"), - ("inline_emphasis_multiline", "*a\nb*"), - ("inline_strike", "a ~b~ ~~c~~ ~~~d~~~"), - ("inline_math", "$x$ $$y$$ $ a"), - ("inline_autolinks", "<http://a.b> <a@b.c> <span> <!-- c --> <br/>"), - ("inline_links", "[a](http://x \"t\") [b][c] [d][] [e] ![f](g) ![h][i]"), - ( - "inline_link_forms", - "[a](<b c> 'x') [d]( e ) [f](g (h)) [i](j \"k\\\"l\") [m](n&amp;o)", - ), - ("inline_link_multiline", "[a](b\n\"t\") [c](d"), - ("inline_image_alt", "![a *b* `c`](d) ![e ![f](g)](h)"), - ("inline_link_in_link", "[a [b](c) d](e)"), - ("inline_emphasis_link", "*[a](b)* **[c][a]**"), - ("inline_html", "<a href=\"x\">t</a> <a\nhref=x>"), - ( - "inline_html_misc", - "<http://a b> <a+b@c> <ab> </a> <?x?> <![CDATA[x]]> <!X y>", - ), - ("inline_nested_emphasis", "*a **b** c* **a *b* c**"), - ("inline_intraword", "a_b_c a*b*c _a_b"), - ("inline_unmatched", "*a **b _c ~d [e ![f"), - ("inline_rule_of_three", "*foo**bar**baz* ***a** b*"), - ("inline_heading", "# a *b* `c`\n"), - ("inline_setext_heading", "a *b*\nc\n===\n"), - ("inline_list_item", "- a *b*\n c **d**\n"), - ("inline_table_cell", "| a *b* | `c\\|d` |\n|-|-|\n"), - ("inline_multibyte", "あ*い*う `え` [お](か)"), - ("inline_crlf_break", "a \r\nb\r\n"), - ("math_multiline", "x$\nx$"), - ("decl_email", "x<!a@b.co>"), - ("code_multiline", "x`\nx`"), - ("def_basic", "[a]: b"), - ("def_title", "[a]: <b c> 'T'"), - ("def_next_lines", "[a]:\nb\n\"t\""), - ("def_title_junk_next_line", "[a]: b\n\"t\" x"), - ("def_title_junk_same_line", "[a]: b \"t\" x"), - ("def_then_paragraph", "[a]: b\nc"), - ("def_two_then_text", "[a]: b\n[c]: d\ntext"), - ("def_not_at_start", "text\n[a]: b"), - ("def_then_setext_equals", "[a]: b\n=== "), - ("def_then_setext_dashes", "[a]: b\n---"), - ("def_indent1", " [a]: b"), - ("def_indent3", " [a]: b"), - ("def_indent4", " [a]: b"), - ("def_in_quote", "> [a]: b"), - ("def_in_list", "- [a]: b"), - ("def_no_dest", "[a]:"), - ("def_empty_label", "[]: b"), - ("def_blank_label", "[ ]: b"), - ("def_escaped_label", "[a\\]b]: c"), - ("def_multiline_title", "[a]: b \"t\nt\""), - ("def_title_indented", "[a]: b\n \"t\""), - ("def_no_space", "[a]:b"), - ("def_dest_junk", "[a]: b c"), - ("def_duplicate_labels", "[A b]: c\n[a b]: d"), - ("def_empty_angle", "[a]: <>"), - ("def_quote_in_dest", "[a]: 'x"), - ("def_use_shortcut", "[a]: /u\n\n[a] [a][] [b][a] [a][b] [A b]"), - ("def_use_normalized", "[ab cd]: /u\n\n[AB CD] [ab\ncd]"), - ("def_use_before", "[a] and [b][a]\n\n[a]: /u \"T\""), - ("def_use_in_link_text", "[a]: b\n\n[a](x) [a]"), - ("def_use_image", "[x]: /u\n\n![x] ![y][x] ![x][]"), - ("def_link_in_emphasis", "[a]: b\n\n*[a]* **[c][a]**"), - ("fn_basic", "[^a]: b"), - ("fn_lazy", "[^a]: b\nc"), - ("fn_indented_continuation", "[^a]: b\n c"), - ("fn_two_paragraphs", "[^a]: b\n\n c"), - ("fn_two_paragraphs_then_text", "[^a]: b\n\n c\n\nd"), - ("fn_two_space_continuation", "[^a]: b\n c"), - ("fn_empty", "[^a]:"), - ("fn_empty_space", "[^a]: "), - ("fn_content_next_line", "[^a]:\n b"), - ("fn_two_adjacent", "[^a]: b\n[^c]: d"), - ("fn_two_separated", "[^a]: b\n\n[^c]: d"), - ("fn_label_space", "[^ a]: b"), - ("fn_label_inner_space", "[^a b]: b"), - ("fn_label_empty", "[^]: b"), - ("fn_indent1", " [^a]: b"), - ("fn_indent4", " [^a]: b"), - ("fn_in_quote", "> [^a]: b"), - ("fn_in_list", "- [^a]: b"), - ("fn_then_quote", "[^a]: b\n> q"), - ("fn_then_list", "[^a]: b\n- x"), - ("fn_heading", "[^a]: # h"), - ("fn_list", "[^a]: - x\n - y"), - ("fn_fence", "[^a]: b\n ```\n x\n ```"), - ("fn_ref_case", "[^A]: b\n\nx[^a]"), - ("fn_no_space", "[^a]:b"), - ("fn_interrupts_paragraph", "text\n[^a]: b"), - ("fn_then_dashes", "[^a]: b\n---"), - ("fn_then_equals", "[^a]: b\n==="), - ("fn_ref_basic", "[^a]\n\n[^a]: note"), - ("fn_ref_no_def", "[^b] no def"), - ("fn_ref_inline", "x[^a] and [^a]!\n\n[^a]: note"), - ("fn_ref_in_link_text", "[a[^a]](b)\n\n[^a]: note"), - ("html_div", "<div>\nx\n</div>\n\ny"), - ("html_indented", " <div>\n x\n"), - ("html_oneline", "<div>x</div>"), - ("html_script", "<script>\nx\n\ny\n</script>\nz"), - ("html_pre", "<pre>\na\n\nb</pre>\nc"), - ("html_style_inline_end", "<style>x</style> y\nz"), - ("html_comment", "<!-- c\n\n d -->\nx"), - ("html_instruction", "<?php\nx ?>\ny"), - ("html_declaration", "<!DOCTYPE html>\nx"), - ("html_cdata", "<![CDATA[\n\nx]]>\ny"), - ("html_complete_tag", "<a href=\"x\">\ny"), - ("html_tag_then_text", "<a href=\"x\">y"), - ("html_interrupt_basic", "text\n<div>"), - ("html_no_interrupt_complete", "text\n<a href=\"x\">"), - ("html_custom_element", "<x-y>\nz"), - ("html_closing", "</div>\nz"), - ("html_multiline_open", "<div\nclass=\"a\">\nz"), - ("html_in_quote", "> <div>\n> x"), - ("html_in_list", "- <div>\n x"), - ("html_uppercase", "<DIV>\nx"), - ("html_self_closing", "<div/>\nx"), - ("html_ins", "<ins>\nx"), - ("html_br", "<br>\nx"), - ("html_textarea", "<textarea>\nx\n\ny</textarea>"), - ("html_table_two_blocks", "<table>\n<tr>\n\n<td>"), - ("html_img_self_close", "<img src=\"x\" />\nz"), - ("html_img_then_text", "<img src=\"x\" /> y"), - ("html_multiline_tag_inline", "<a\nb>\nz"), - ("html_indent4", " <div>"), - ("html_comment_then_text", "<!--x-->y\nz"), - ("html_then_code_line", "<div>\n code"), - ("html_comment_short", "<!-->\nx"), - ("html_instruction_short", "<?>\nx"), - ("html_lazy_quote", "> <div>\nx"), - ("html_attr_forms", "<a b c=d e='f' g=\"h\">\nx"), - ("html_bad_attr", "<a b=>\nx"), - ("fm_yaml", "---\na: b\n---\ntext"), - ("fm_yaml_only", "---\na: b\n---"), - ("fm_empty", "---\n---"), - ("fm_unclosed", "---\na\n"), - ("fm_blank_content", "---\n\n---\nx"), - ("fm_toml", "+++\na = 1\n+++\nx"), - ("fm_indented", " ---\na\n---"), - ("fm_close_trailing_space", "---\na\n--- \nx"), - ("fm_close_longer", "---\na\n----\nx"), - ("fm_open_trailing_space", "--- \na\n---"), - ("fm_open_junk", "---x\na\n---"), - ("fm_not_first", "text\n---\na\n---"), - ("fm_blank_before", "\n---\na\n---"), - ("fm_multiline_content", "---\na: 1\n\nb: 2\n---\n# h"), - ("fm_hr_after", "---\na\n---\n\n---"), - ("math_basic", "$$\na\n$$"), - ("math_then_text", "$$\na\n$$\nx"), - ("math_meta", "$$ meta\na\n$$"), - ("math_unclosed", "$$\na"), - ("math_longer_open", "$$$\na\n$$"), - ("math_longer_close", "$$\na\n$$$$"), - ("math_indented", " $$\n a\n b\n $$"), - ("math_empty", "$$\n$$"), - ("math_inline_not_block", "$$a$$"), - ("math_info_no_close", "$$a\n$$"), - ("math_in_quote", "> $$\n> a\n> $$"), - ("math_in_list", "- $$\n a\n $$"), - ("math_close_junk", "$$\na\n$$ x"), - ("math_interrupts_paragraph", "text\n$$\na\n$$"), - ("math_blank_inside", "$$\n\na\n$$"), - ("math_single_dollar_line", "$ $\nx"), - ("crlf_frontmatter", "---\r\na\r\n---\r\nx"), - ("break_end_indented", "a \n b"), - ("text_end_indented_continuation", "*a*\n <x>"), - ("soft_end_before_node", "a\n `b`"), - ("break_in_quote", "> a \n> b"), - ("text_eol_in_footnote", "[^a]: x \n [a]"), - ("math_whitespace_content", " $$\n \n $$"), - ("math_whitespace_content2", " $$\n \n $$"), - ("table_indented_no_pipe", " a|b\n|-|:-:|"), - ("tab_nested_list", "- a\n\t- b\n\t\t- c\n"), - ("tab_nested_ordered", "1. a\n\t1. b\n\t\t1. c\n"), - ("tab_list_code", "- a\n\n\t\tcode\n"), - ("tab_in_text", "a\tb\t*c*\n"), - ("tab_code_block", "\tcode\n\t\tmore\n"), - ("tab_after_marker", "-\ta\n"), - ("atx_leading_sequences", "# # a"), - ("atx_leading_sequences2", "## # a"), - ("atx_only_sequences", "# # #"), - ("atx_no_space_sequence", "# #a"), - ("atx_double_sequence", "# ## a"), - ("atx_tab_sequence", "#\t# a"), - ("atx_closing_and_leading", "# # a #"), - ("atx_inner_hash", "# a # b"), - ("quote_footnote_trailing_space", "- a\n > [^a]: x\n "), - ("indented_code_trailing_blank", " code\n\n\n"), - ("bom_heading", "\u{feff}# a\n"), - ("bom_frontmatter", "\u{feff}---\na: b\n---\n"), - ("bom_text", "\u{feff}a\n"), - ("www_multibyte_after_ww", "wwß x"), - ("www_combining", "ww\u{301}.a.com"), - ("www_cjk", "www.あ.com/パス x"), - ("http_cjk", "http://あ.jp/パス ok"), - ("email_cjk_domain", "a@あ.com"), - ("email_cjk_local", "あ@b.com"), - ("html_inline_continuation", "<img a\n b=\"c\">"), - ("html_inline_continuation_tab", "<img a\n\t\tb=\"c\">"), - ("table_trailing_space", "| a | b | \n|-|-| \n| c | d | \n"), - ("table_no_pipes_trailing_space", "a | b\n--|--\nd | e \n"), - ("table_trailing_tab", "| a | b |\t\n|-|-|\n| c | d | \n"), - ("table_cjk", "| あ | 🎉 |\n|:-|-:|\n| é | ß |\n"), - ("table_single_column_colon", "a\n:-\nb\n"), - ("table_single_column_right", "a\n---:\nb\nc\n"), - ("table_double_colon_not_delimiter", "a\n::-\n"), - ("table_body_empty_marker", "a|b\n|-|-|\n*\n"), - ("table_body_ordered_marker", "a|b\n|-|-|\n2. x\n"), - ("table_body_empty_ordered", "a|b\n|-|-|\n1.\n"), - ("paragraph_after_partial_tab", "- a\n\n\t[P](h) x\n"), - ("fence_after_partial_tab", "- a\n\n\t```ts\n\t\tx\n\t```\n"), - ("math_after_partial_tab", "- a\n\n\t$$\n\tm\n\t$$\n"), - ("quote_after_partial_tab", "- a\n\n\t> q\n"), - ("heading_after_partial_tab", "- a\n\n\t# h\n"), - ( - "control_character_references", - "&#1;&#x7f;&#x85;&#xB;&#xC;&#x9f;&#xa0;&#xd800;&#x110000;&#0;", - ), - ("footnote_escaped_bracket", "[^a\\]: x"), - ("footnote_escaped_backslash", "[^a\\\\]: x"), - ("footnote_escaped_open", "[^a\\[]: x"), - ("footnote_unescaped_open", "[^a[]: x"), - ("footnote_empty_then_blank", "[^a]: \n\n"), - ("footnote_empty_then_spaces", "[^a]: \n \n"), - ("footnote_empty_then_text", "[^a]: \n\nb"), - ("definition_nul_destination", "[a]: \0\n\n[a]\n"), - ("link_nul_destination", "[a](x\0y)"), - ("definition_then_empty_marker", "[a]: b\n-"), - ("definition_then_item", "[a]: b\n- x"), - ("definition_then_ordered", "[a]: b\n1."), - ("emphasis_cjk", "*あ*。*い*"), - ("strong_cjk", "**日本語**です"), - ("underscore_accent", "_é_ _ß_"), - ("fullwidth_markers", "*a* _b_"), - ("emphasis_after_cjk_punctuation", "。*a*、**b**"), - ("combining_mark", "e\u{301}*a*"), - ("emoji_zwj", "👨\u{200d}👩\u{200d}👧 *a*"), - ("fence_cjk_info", "```あ\n日本語\n```\n"), - ("fence_in_quote_emoji", "> ```🎉\n> x\n> ```\n"), - ("fence_in_list_cjk", "- ```あ\n 日本語\n ```\n"), - ("math_cjk", "$$\nあ\n$$\n\n$い$\n"), - ("heading_cjk", "# 見出し\n\n見出し2\n===\n"), - ("link_cjk", "[あ](http://い.jp/う \"え\")"), - ("reference_cjk", "[あ]\n\n[あ]: /u\n"), - ("footnote_cjk", "あ[^い]\n\n[^い]: う\n"), - ("ideographic_space", "a\u{3000}b\n\u{3000}c"), - ("nbsp_around_emphasis", "a\u{a0}*b*\u{a0}c"), - ("entity_cjk_context", "あ&amp;い&#12354;&#x1F389;"), - ("autolink_cjk", "<http://あ.jp/パス>"), - ("code_span_cjk", "`あ`と``い`う``"), - ("hard_break_cjk", "あ \nい\\\nう"), - ("nul_in_text", "a\0b"), - ]; - - /// One line with the nodes, positions written as `@line:column-line:column`. - fn dump(nodes: &[Node]) -> String { - format!("{nodes:?}") - .replace("position: Some(Position { start: Point { line: ", "@") - .replace(", column: ", ":") - .replace(" }, end: Point { line: ", "-") - .replace(" } })", "") - } - - /// What the parser and the HTML renderer make of each case, to compare with `snapshots.txt`. - /// Run the tests with `UPDATE_SNAPSHOTS=1` to write it after a change that is meant to alter it. - fn snapshot() -> String { - let mut out = String::new(); - for (name, input) in CASES { - let nodes = dump(&parse(input).unwrap()); - out.push_str(&format!( - "## {name} {input:?}\nnodes: {nodes}\nhtml: {:?}\n", - to_html(input) - )); - } - for (name, input) in MDX_CASES { - let nodes = match parse_mdx(input) { - Ok(nodes) => dump(&nodes), - Err(error) => format!("error: {error}"), - }; - out.push_str(&format!("## mdx {name} {input:?}\nnodes: {nodes}\n")); - } - out - } - - #[test] - fn snapshots_match() { - let actual = snapshot(); - if std::env::var("UPDATE_SNAPSHOTS").is_ok() { - std::fs::write( - concat!(env!("CARGO_MANIFEST_DIR"), "/src/parser/snapshots.txt"), - &actual, - ) - .unwrap(); - return; - } - let expected = include_str!("parser/snapshots.txt"); - for (actual, expected) in actual.split("## ").zip(expected.split("## ")) { - assert_eq!(actual, expected, "snapshot differs, UPDATE_SNAPSHOTS=1 rewrites it"); - } - assert_eq!(actual.len(), expected.len(), "number of snapshots differs"); - } - fn env_number<T: std::str::FromStr>(name: &str) -> Option<T> { std::env::var(name).ok()?.parse().ok() } /// Lines built from container prefixes and block-level bodies only, so inline syntax (not /// implemented yet) never appears. Constructs whose markdown-rs positions depend on quirks of the - /// following lines are left to the hand-written cases: empty items and quotes, unclosed fences, - /// indented code next to containers, ordered lists that do not start at 1, and lazy continuation - /// inside nested containers, more than one container marker on a line, and runs of setext - /// underlines (`===` is only used by the hand-written cases). + /// following lines are left to the cases of `tests/parser_tree_tests.rs`: empty items and quotes, + /// unclosed fences, indented code next to containers, ordered lists that do not start at 1, and + /// lazy continuation inside nested containers, more than one container marker on a line, and runs + /// of setext underlines (`===` is only used by those cases). fn block_input() -> impl Strategy<Value = String> { // At most one container marker per line; deeper nesting comes from indentation. let indent = prop::sample::select(vec!["", " ", " ", " "]); diff --git a/crates/mq-markdown/src/parser/snapshots.txt b/crates/mq-markdown/src/parser/snapshots.txt deleted file mode 100644 index f53bc3d4a..000000000 --- a/crates/mq-markdown/src/parser/snapshots.txt +++ /dev/null @@ -1,1324 +0,0 @@ -## empty "" -nodes: [] -html: "" -## blank_lines "\n\n \n" -nodes: [] -html: "" -## paragraph "hello\n" -nodes: [Text(Text { value: "hello", @1:1-1:6 })] -html: "<p>hello</p>\n" -## paragraph_no_eol "hello" -nodes: [Text(Text { value: "hello", @1:1-1:6 })] -html: "<p>hello</p>" -## paragraph_multiline "a\nb\nc\n" -nodes: [Text(Text { value: "a\nb\nc", @1:1-3:2 })] -html: "<p>a\nb\nc</p>\n" -## paragraph_indented_continuation "a\n b\n" -nodes: [Text(Text { value: "a\nb", @1:1-2:6 })] -html: "<p>a\nb</p>\n" -## paragraph_trailing_space "あい\n うえ \n" -nodes: [Text(Text { value: "あい\nうえ", @1:1-2:9 })] -html: "<p>あい\nうえ</p>\n" -## two_paragraphs "a\n\nb\n" -nodes: [Text(Text { value: "a", @1:1-1:2 }), Text(Text { value: "b", @3:1-3:2 })] -html: "<p>a</p>\n<p>b</p>\n" -## crlf "a\r\nb\r\n\r\nc\r\n" -nodes: [Text(Text { value: "a\r\nb", @1:1-2:2 }), Text(Text { value: "c", @4:1-4:2 })] -html: "<p>a\r\nb</p>\r\n<p>c</p>\r\n" -## crlf_fence "```\r\na\r\nb\r\n```\r\n" -nodes: [Code(Code { value: "a\r\nb", lang: None, @1:1-4:4, meta: None, fence: true })] -html: "<pre><code>a\r\nb\r\n</code></pre>\r\n" -## cr_only "a\rb\r" -nodes: [Text(Text { value: "a\rb", @1:1-2:2 })] -html: "<p>a\rb</p>\r" -## atx_h1 "# title\n" -nodes: [Heading(Heading { depth: 1, values: [Text(Text { value: "title", @1:3-1:8 })], @1:1-1:8 })] -html: "<h1>title</h1>\n" -## atx_h6 "###### title\n" -nodes: [Heading(Heading { depth: 6, values: [Text(Text { value: "title", @1:8-1:13 })], @1:1-1:13 })] -html: "<h6>title</h6>\n" -## atx_seven "####### title\n" -nodes: [Text(Text { value: "####### title", @1:1-1:14 })] -html: "<p>####### title</p>\n" -## atx_no_space "#title\n" -nodes: [Text(Text { value: "#title", @1:1-1:7 })] -html: "<p>#title</p>\n" -## atx_empty "#\n" -nodes: [Heading(Heading { depth: 1, values: [], @1:1-1:2 })] -html: "<h1></h1>\n" -## atx_closing "## title ##\n" -nodes: [Heading(Heading { depth: 2, values: [Text(Text { value: "title", @1:4-1:9 })], @1:1-1:12 })] -html: "<h2>title</h2>\n" -## atx_closing_no_space "## title##\n" -nodes: [Heading(Heading { depth: 2, values: [Text(Text { value: "title##", @1:4-1:11 })], @1:1-1:11 })] -html: "<h2>title##</h2>\n" -## atx_indent " ## title ## \n" -nodes: [Heading(Heading { depth: 2, values: [Text(Text { value: "title", @1:6-1:11 })], @1:1-1:16 })] -html: "<h2>title</h2>\n" -## atx_multibyte "# あ h #\n" -nodes: [Heading(Heading { depth: 1, values: [Text(Text { value: "あ h", @1:3-1:8 })], @1:1-1:10 })] -html: "<h1>あ h</h1>\n" -## atx_interrupts_paragraph "a\n# b\n" -nodes: [Text(Text { value: "a", @1:1-1:2 }), Heading(Heading { depth: 1, values: [Text(Text { value: "b", @2:3-2:4 })], @2:1-2:4 })] -html: "<p>a</p>\n<h1>b</h1>\n" -## setext_h1 "Title\n===\n" -nodes: [Heading(Heading { depth: 1, values: [Text(Text { value: "Title", @1:1-1:6 })], @1:1-2:4 })] -html: "<h1>Title</h1>\n" -## setext_h2 "a\n ---\n" -nodes: [Heading(Heading { depth: 2, values: [Text(Text { value: "a", @1:1-1:2 })], @1:1-2:6 })] -html: "<h2>a</h2>\n" -## setext_multiline "a\nb\n---\n" -nodes: [Heading(Heading { depth: 2, values: [Text(Text { value: "a\nb", @1:1-2:2 })], @1:1-3:4 })] -html: "<h2>a\nb</h2>\n" -## thematic_star "***\n" -nodes: [HorizontalRule(HorizontalRule { marker: Some(Star), @1:1-1:4 })] -html: "<hr />\n" -## thematic_dash_spaced "- - -\n" -nodes: [HorizontalRule(HorizontalRule { marker: Some(Dash), @1:1-1:6 })] -html: "<hr />\n" -## thematic_underscore " ___ \n" -nodes: [HorizontalRule(HorizontalRule { marker: Some(Underscore), @1:1-1:8 })] -html: "<hr />\n" -## thematic_two "**\n" -nodes: [Text(Text { value: "**", @1:1-1:3 })] -html: "<p>**</p>\n" -## fence_lang_meta "```rust title\nlet a;\n```\n" -nodes: [Code(Code { value: "let a;", lang: Some("rust"), @1:1-3:4, meta: Some("title"), fence: true })] -html: "<pre><code class=\"language-rust\">let a;\n</code></pre>\n" -## fence_indented " ```\n a\n b\n ```\n" -nodes: [Code(Code { value: "a\n b", lang: None, @1:3-4:6, meta: None, fence: true })] -html: "<pre><code>a\n b\n</code></pre>\n" -## fence_tilde "~~~\na\n~~~\n" -nodes: [Code(Code { value: "a", lang: None, @1:1-3:4, meta: None, fence: true })] -html: "<pre><code>a\n</code></pre>\n" -## fence_unclosed "```\na\n" -nodes: [Code(Code { value: "a", lang: None, @1:1-3:1, meta: None, fence: true })] -html: "<pre><code>a\n</code></pre>\n" -## fence_empty "para\n\n\n```\n```\n" -nodes: [Text(Text { value: "para", @1:1-1:5 }), Code(Code { value: "", lang: None, @4:1-5:4, meta: None, fence: true })] -html: "<p>para</p>\n<pre><code></code></pre>\n" -## fence_longer_close "```\na\n`````\n" -nodes: [Code(Code { value: "a", lang: None, @1:1-3:6, meta: None, fence: true })] -html: "<pre><code>a\n</code></pre>\n" -## fence_shorter_close "````\na\n```\nb\n````\n" -nodes: [Code(Code { value: "a\n```\nb", lang: None, @1:1-5:5, meta: None, fence: true })] -html: "<pre><code>a\n```\nb\n</code></pre>\n" -## fence_blank_inside "```\na\n\nb\n```\n" -nodes: [Code(Code { value: "a\n\nb", lang: None, @1:1-5:4, meta: None, fence: true })] -html: "<pre><code>a\n\nb\n</code></pre>\n" -## fence_backtick_info "``` a`b\nc\n" -nodes: [Text(Text { value: "``` a`b\nc", @1:1-2:2 })] -html: "<p>``` a`b\nc</p>\n" -## fence_interrupts_paragraph "a\n```\nb\n```\n" -nodes: [Text(Text { value: "a", @1:1-1:2 }), Code(Code { value: "b", lang: None, @2:1-4:4, meta: None, fence: true })] -html: "<p>a</p>\n<pre><code>b\n</code></pre>\n" -## indented_code " code\n\n more\n\nx\n" -nodes: [Code(Code { value: "code\n\n more", lang: None, @1:1-3:11, meta: None, fence: false }), Text(Text { value: "x", @5:1-5:2 })] -html: "<pre><code>code\n\n more\n</code></pre>\n<p>x</p>\n" -## quote "> a\n> b\n" -nodes: [Blockquote(Blockquote { values: [Text(Text { value: "a\nb", @1:3-2:4 })], @1:1-2:4 })] -html: "<blockquote>\n<p>a\nb</p>\n</blockquote>\n" -## quote_indented_lazy " > a\nb\n" -nodes: [Blockquote(Blockquote { values: [Text(Text { value: "a\nb", @1:5-2:2 })], @1:1-2:2 })] -html: "<blockquote>\n<p>a\nb</p>\n</blockquote>\n" -## quote_heading "> # h\n>\n> c\n" -nodes: [Blockquote(Blockquote { values: [Heading(Heading { depth: 1, values: [Text(Text { value: "h", @1:5-1:6 })], @1:3-1:6 }), Text(Text { value: "c", @3:3-3:4 })], @1:1-3:4 })] -html: "<blockquote>\n<h1>h</h1>\n<p>c</p>\n</blockquote>\n" -## quote_no_space ">a\n" -nodes: [Blockquote(Blockquote { values: [Text(Text { value: "a", @1:2-1:3 })], @1:1-1:3 })] -html: "<blockquote>\n<p>a</p>\n</blockquote>\n" -## quote_empty ">\n" -nodes: [Blockquote(Blockquote { values: [], @1:1-1:2 })] -html: "<blockquote>\n</blockquote>\n" -## quote_nested "> > a\n> b\n" -nodes: [Blockquote(Blockquote { values: [Blockquote(Blockquote { values: [Text(Text { value: "a\nb", @1:5-2:4 })], @1:3-2:4 })], @1:1-2:4 })] -html: "<blockquote>\n<blockquote>\n<p>a\nb</p>\n</blockquote>\n</blockquote>\n" -## quote_lazy_nested "> > a\nb\n" -nodes: [Blockquote(Blockquote { values: [Blockquote(Blockquote { values: [Text(Text { value: "a\nb", @1:5-2:2 })], @1:3-2:2 })], @1:1-2:2 })] -html: "<blockquote>\n<blockquote>\n<p>a\nb</p>\n</blockquote>\n</blockquote>\n" -## quote_blank_ends "> a\n\n> b\n" -nodes: [Blockquote(Blockquote { values: [Text(Text { value: "a", @1:3-1:4 })], @1:1-1:4 }), Blockquote(Blockquote { values: [Text(Text { value: "b", @3:3-3:4 })], @3:1-3:4 })] -html: "<blockquote>\n<p>a</p>\n</blockquote>\n<blockquote>\n<p>b</p>\n</blockquote>\n" -## quote_fence "> ```\n> a\n> ```\n" -nodes: [Blockquote(Blockquote { values: [Code(Code { value: "a", lang: None, @1:3-3:6, meta: None, fence: true })], @1:1-3:6 })] -html: "<blockquote>\n<pre><code>a\n</code></pre>\n</blockquote>\n" -## quote_no_lazy_after_fence "> ```\n> a\nb\n" -nodes: [Blockquote(Blockquote { values: [Code(Code { value: "a", lang: None, @1:3-2:4, meta: None, fence: true })], @1:1-2:4 }), Text(Text { value: "b", @3:1-3:2 })] -html: "<blockquote>\n<pre><code>a\n</code></pre>\n</blockquote>\n<p>b</p>\n" -## quote_interrupts_paragraph "a\n> b\n" -nodes: [Text(Text { value: "a", @1:1-1:2 }), Blockquote(Blockquote { values: [Text(Text { value: "b", @2:3-2:4 })], @2:1-2:4 })] -html: "<p>a</p>\n<blockquote>\n<p>b</p>\n</blockquote>\n" -## quote_thematic_not_lazy "> a\n---\n" -nodes: [Blockquote(Blockquote { values: [Text(Text { value: "a", @1:3-1:4 })], @1:1-1:4 }), HorizontalRule(HorizontalRule { marker: Some(Dash), @2:1-2:4 })] -html: "<blockquote>\n<p>a</p>\n</blockquote>\n<hr />\n" -## quote_multibyte "> あ\n> い\n" -nodes: [Blockquote(Blockquote { values: [Text(Text { value: "あ\nい", @1:3-2:6 })], @1:1-2:6 })] -html: "<blockquote>\n<p>あ\nい</p>\n</blockquote>\n" -## bullet "- a\n- b\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-1:4 }), List(List { values: [Text(Text { value: "b", @2:3-2:4 })], index: 1, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @2:3-2:4 })] -html: "<ul>\n<li>a</li>\n<li>b</li>\n</ul>\n" -## bullet_loose "- a\n\n- b\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 })], index: 0, level: 0, ordered: false, checked: None, spread: true, start: None, marker: Some(Dash), @1:3-1:4 }), List(List { values: [Text(Text { value: "b", @3:3-3:4 })], index: 1, level: 0, ordered: false, checked: None, spread: true, start: None, marker: Some(Dash), @3:3-3:4 })] -html: "<ul>\n<li>\n<p>a</p>\n</li>\n<li>\n<p>b</p>\n</li>\n</ul>\n" -## bullet_indent " - a\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:4-1:5 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:4-1:5 })] -html: "<ul>\n<li>a</li>\n</ul>\n" -## bullet_mixed_markers "* a\n+ b\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Star), @1:3-1:4 }), List(List { values: [Text(Text { value: "b", @2:3-2:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Plus), @2:3-2:4 })] -html: "<ul>\n<li>a</li>\n</ul>\n<ul>\n<li>b</li>\n</ul>\n" -## ordered "1. a\n2. b\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:4-1:5 })], index: 0, level: 0, ordered: true, checked: None, spread: false, start: Some(1), marker: Some(Period), @1:4-1:5 }), List(List { values: [Text(Text { value: "b", @2:4-2:5 })], index: 1, level: 0, ordered: true, checked: None, spread: false, start: Some(1), marker: Some(Period), @2:4-2:5 })] -html: "<ol>\n<li>a</li>\n<li>b</li>\n</ol>\n" -## ordered_paren "5) a\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:4-1:5 })], index: 0, level: 0, ordered: true, checked: None, spread: false, start: Some(5), marker: Some(Paren), @1:4-1:5 })] -html: "<ol start=\"5\">\n<li>a</li>\n</ol>\n" -## ordered_zero "0. a\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:4-1:5 })], index: 0, level: 0, ordered: true, checked: None, spread: false, start: Some(0), marker: Some(Period), @1:4-1:5 })] -html: "<ol start=\"0\">\n<li>a</li>\n</ol>\n" -## ordered_too_long "1234567890. a\n" -nodes: [Text(Text { value: "1234567890. a", @1:1-1:14 })] -html: "<p>1234567890. a</p>\n" -## ordered_mixed_delimiters "1. a\n2) b\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:4-1:5 })], index: 0, level: 0, ordered: true, checked: None, spread: false, start: Some(1), marker: Some(Period), @1:4-1:5 }), List(List { values: [Text(Text { value: "b", @2:4-2:5 })], index: 0, level: 0, ordered: true, checked: None, spread: false, start: Some(2), marker: Some(Paren), @2:4-2:5 })] -html: "<ol>\n<li>a</li>\n</ol>\n<ol start=\"2\">\n<li>b</li>\n</ol>\n" -## task "- [ ] x\n- [x] y\n- [X] z\n" -nodes: [List(List { values: [Text(Text { value: "x", @1:7-1:8 })], index: 0, level: 0, ordered: false, checked: Some(false), spread: false, start: None, marker: Some(Dash), @1:7-1:8 }), List(List { values: [Text(Text { value: "y", @2:7-2:8 })], index: 1, level: 0, ordered: false, checked: Some(true), spread: false, start: None, marker: Some(Dash), @2:7-2:8 }), List(List { values: [Text(Text { value: "z", @3:7-3:8 })], index: 2, level: 0, ordered: false, checked: Some(true), spread: false, start: None, marker: Some(Dash), @3:7-3:8 })] -html: "<ul>\n<li><input type=\"checkbox\" disabled=\"\" /> x</li>\n<li><input type=\"checkbox\" checked=\"\" disabled=\"\" /> y</li>\n<li><input type=\"checkbox\" checked=\"\" disabled=\"\" /> z</li>\n</ul>\n" -## task_empty "- [ ]\n" -nodes: [List(List { values: [Text(Text { value: "[ ]", @1:3-1:6 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-1:6 })] -html: "<ul>\n<li>[ ]</li>\n</ul>\n" -## task_no_space "- [ ]x\n" -nodes: [List(List { values: [Text(Text { value: "[ ]x", @1:3-1:7 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-1:7 })] -html: "<ul>\n<li>[ ]x</li>\n</ul>\n" -## nested "- a\n - b\n c\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-1:4 }), List(List { values: [Text(Text { value: "b\nc", @2:5-3:6 })], index: 0, level: 1, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @2:5-3:6 })] -html: "<ul>\n<li>a\n<ul>\n<li>b\nc</li>\n</ul>\n</li>\n</ul>\n" -## nested_ordered "1. a\n 1. b\n 2. c\n2. d\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:4-1:5 })], index: 0, level: 0, ordered: true, checked: None, spread: false, start: Some(1), marker: Some(Period), @1:4-1:5 }), List(List { values: [Text(Text { value: "b", @2:7-2:8 })], index: 0, level: 1, ordered: true, checked: None, spread: false, start: Some(1), marker: Some(Period), @2:7-2:8 }), List(List { values: [Text(Text { value: "c", @3:7-3:8 })], index: 1, level: 1, ordered: true, checked: None, spread: false, start: Some(1), marker: Some(Period), @3:7-3:8 }), List(List { values: [Text(Text { value: "d", @4:4-4:5 })], index: 1, level: 0, ordered: true, checked: None, spread: false, start: Some(1), marker: Some(Period), @4:4-4:5 })] -html: "<ol>\n<li>a\n<ol>\n<li>b</li>\n<li>c</li>\n</ol>\n</li>\n<li>d</li>\n</ol>\n" -## nested_dedent "- a\n - b\n- c\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-1:4 }), List(List { values: [Text(Text { value: "b", @2:5-2:6 })], index: 0, level: 1, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @2:5-2:6 }), List(List { values: [Text(Text { value: "c", @3:3-3:4 })], index: 1, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @3:3-3:4 })] -html: "<ul>\n<li>a\n<ul>\n<li>b</li>\n</ul>\n</li>\n<li>c</li>\n</ul>\n" -## list_item_two_paragraphs "- a\n\n b\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 }), Text(Text { value: "b", @3:3-3:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-3:4 })] -html: "<ul>\n<li>\n<p>a</p>\n<p>b</p>\n</li>\n</ul>\n" -## two_paragraphs_then_item "- a\n\n b\n- c\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 }), Text(Text { value: "b", @3:3-3:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-3:4 }), List(List { values: [Text(Text { value: "c", @4:3-4:4 })], index: 1, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @4:3-4:4 })] -html: "<ul>\n<li>\n<p>a</p>\n<p>b</p>\n</li>\n<li>\n<p>c</p>\n</li>\n</ul>\n" -## empty_item "-\n" -nodes: [List(List { values: [], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:1-1:2 })] -html: "<ul>\n<li></li>\n</ul>\n" -## empty_item_content "-\n foo\n" -nodes: [List(List { values: [Text(Text { value: "foo", @2:3-2:6 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @2:3-2:6 })] -html: "<ul>\n<li>foo</li>\n</ul>\n" -## empty_item_blank_then_text "-\n\n foo\n" -nodes: [List(List { values: [], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:1-1:2 }), Text(Text { value: "foo", @3:3-3:6 })] -html: "<ul>\n<li></li>\n</ul>\n<p>foo</p>\n" -## empty_items "-\n-\n" -nodes: [List(List { values: [], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:1-1:2 }), List(List { values: [], index: 1, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @2:1-2:2 })] -html: "<ul>\n<li></li>\n<li></li>\n</ul>\n" -## wide_marker_gap "- a\n b\n" -nodes: [List(List { values: [Text(Text { value: "a\nb", @1:5-2:6 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:5-2:6 })] -html: "<ul>\n<li>a\nb</li>\n</ul>\n" -## code_in_item "- a\n" -nodes: [List(List { values: [Code(Code { value: "a", lang: None, @1:3-1:8, meta: None, fence: false })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-1:8 })] -html: "<ul>\n<li>\n<pre><code>a\n</code></pre>\n</li>\n</ul>\n" -## lazy "- a\nb\n" -nodes: [List(List { values: [Text(Text { value: "a\nb", @1:3-2:2 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-2:2 })] -html: "<ul>\n<li>a\nb</li>\n</ul>\n" -## lazy_after_blank "- a\n\nb\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-1:4 }), Text(Text { value: "b", @3:1-3:2 })] -html: "<ul>\n<li>a</li>\n</ul>\n<p>b</p>\n" -## list_interrupts_paragraph "a\n- b\n" -nodes: [Text(Text { value: "a", @1:1-1:2 }), List(List { values: [Text(Text { value: "b", @2:3-2:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @2:3-2:4 })] -html: "<p>a</p>\n<ul>\n<li>b</li>\n</ul>\n" -## ordered_two_no_interrupt "a\n2. b\n" -nodes: [Text(Text { value: "a\n2. b", @1:1-2:5 })] -html: "<p>a\n2. b</p>\n" -## ordered_one_interrupts "a\n1. b\n" -nodes: [Text(Text { value: "a", @1:1-1:2 }), List(List { values: [Text(Text { value: "b", @2:4-2:5 })], index: 0, level: 0, ordered: true, checked: None, spread: false, start: Some(1), marker: Some(Period), @2:4-2:5 })] -html: "<p>a</p>\n<ol>\n<li>b</li>\n</ol>\n" -## empty_item_no_interrupt "a\n-\n" -nodes: [Heading(Heading { depth: 2, values: [Text(Text { value: "a", @1:1-1:2 })], @1:1-2:2 })] -html: "<h2>a</h2>\n" -## list_in_quote "> - a\n> - b\n" -nodes: [Blockquote(Blockquote { values: [List(List { values: [Text(Text { value: "a", @1:5-1:6 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:5-1:6 }), List(List { values: [Text(Text { value: "b", @2:5-2:6 })], index: 1, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @2:5-2:6 })], @1:1-2:6 })] -html: "<blockquote>\n<ul>\n<li>a</li>\n<li>b</li>\n</ul>\n</blockquote>\n" -## quote_in_list "- > a\n > b\n" -nodes: [List(List { values: [Blockquote(Blockquote { values: [Text(Text { value: "a\nb", @1:5-2:6 })], @1:3-2:6 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-2:6 })] -html: "<ul>\n<li>\n<blockquote>\n<p>a\nb</p>\n</blockquote>\n</li>\n</ul>\n" -## quote_in_list_level "- a\n > - b\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 }), Blockquote(Blockquote { values: [List(List { values: [Text(Text { value: "b", @2:7-2:8 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @2:7-2:8 })], @2:3-2:8 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-2:8 })] -html: "<ul>\n<li>a\n<blockquote>\n<ul>\n<li>b</li>\n</ul>\n</blockquote>\n</li>\n</ul>\n" -## heading_in_item "- # h\n text\n" -nodes: [List(List { values: [Heading(Heading { depth: 1, values: [Text(Text { value: "h", @1:5-1:6 })], @1:3-1:6 }), Text(Text { value: "text", @2:3-2:7 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-2:7 })] -html: "<ul>\n<li>\n<h1>h</h1>\ntext</li>\n</ul>\n" -## fence_in_item "- ```\n a\n ```\n" -nodes: [List(List { values: [Code(Code { value: "a", lang: None, @1:3-3:6, meta: None, fence: true })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-3:6 })] -html: "<ul>\n<li>\n<pre><code>a\n</code></pre>\n</li>\n</ul>\n" -## multibyte_item "- あ\n- い\n" -nodes: [List(List { values: [Text(Text { value: "あ", @1:3-1:6 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-1:6 }), List(List { values: [Text(Text { value: "い", @2:3-2:6 })], index: 1, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @2:3-2:6 })] -html: "<ul>\n<li>あ</li>\n<li>い</li>\n</ul>\n" -## list_after_paragraph_blank "a\n\n- b\n" -nodes: [Text(Text { value: "a", @1:1-1:2 }), List(List { values: [Text(Text { value: "b", @3:3-3:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @3:3-3:4 })] -html: "<p>a</p>\n<ul>\n<li>b</li>\n</ul>\n" -## two_lists_split "- a\n\n\n- b\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 })], index: 0, level: 0, ordered: false, checked: None, spread: true, start: None, marker: Some(Dash), @1:3-1:4 }), List(List { values: [Text(Text { value: "b", @4:3-4:4 })], index: 1, level: 0, ordered: false, checked: None, spread: true, start: None, marker: Some(Dash), @4:3-4:4 })] -html: "<ul>\n<li>\n<p>a</p>\n</li>\n<li>\n<p>b</p>\n</li>\n</ul>\n" -## deep_quote ">>>>>> a\n" -nodes: [Blockquote(Blockquote { values: [Blockquote(Blockquote { values: [Blockquote(Blockquote { values: [Blockquote(Blockquote { values: [Blockquote(Blockquote { values: [Blockquote(Blockquote { values: [Text(Text { value: "a", @1:8-1:9 })], @1:6-1:9 })], @1:5-1:9 })], @1:4-1:9 })], @1:3-1:9 })], @1:2-1:9 })], @1:1-1:9 })] -html: "<blockquote>\n<blockquote>\n<blockquote>\n<blockquote>\n<blockquote>\n<blockquote>\n<p>a</p>\n</blockquote>\n</blockquote>\n</blockquote>\n</blockquote>\n</blockquote>\n</blockquote>\n" -## fence_unclosed_no_eol "```\na" -nodes: [Code(Code { value: "a", lang: None, @1:1-2:2, meta: None, fence: true })] -html: "<pre><code>a\n</code></pre>\n" -## fence_unclosed_in_item "- ```\n a\n" -nodes: [List(List { values: [Code(Code { value: "a", lang: None, @1:3-3:1, meta: None, fence: true })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-3:1 })] -html: "<ul>\n<li>\n<pre><code>a\n</code></pre>\n</li>\n</ul>" -## fence_unclosed_in_item_no_eol "- ```\n a" -nodes: [List(List { values: [Code(Code { value: "a", lang: None, @1:3-2:4, meta: None, fence: true })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-2:4 })] -html: "<ul>\n<li>\n<pre><code>a\n</code></pre>\n</li>\n</ul>" -## fence_unclosed_in_quote_no_eol "> ```\n> a" -nodes: [Blockquote(Blockquote { values: [Code(Code { value: "a", lang: None, @1:3-2:4, meta: None, fence: true })], @1:1-2:4 })] -html: "<blockquote>\n<pre><code>a\n</code></pre>\n</blockquote>" -## fence_unclosed_in_quote_eof "> ```\n> a\n" -nodes: [Blockquote(Blockquote { values: [Code(Code { value: "a", lang: None, @1:3-3:1, meta: None, fence: true })], @1:1-3:1 })] -html: "<blockquote>\n<pre><code>a\n</code></pre>\n</blockquote>" -## fence_unclosed_trailing_blank "```\na\n\n" -nodes: [Code(Code { value: "a\n", lang: None, @1:1-4:1, meta: None, fence: true })] -html: "<pre><code>a\n\n</code></pre>\n" -## fence_unclosed_item_then_para "- ```\n a\nb\n" -nodes: [List(List { values: [Code(Code { value: "a", lang: None, @1:3-2:4, meta: None, fence: true })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-2:4 }), Text(Text { value: "b", @3:1-3:2 })] -html: "<ul>\n<li>\n<pre><code>a\n</code></pre>\n</li>\n</ul>\n<p>b</p>\n" -## item_code_then_unindented_text "- ===\n\n a\n===" -nodes: [List(List { values: [Text(Text { value: "===", @1:3-1:6 }), Code(Code { value: "a", lang: None, @3:3-3:8, meta: None, fence: false })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-3:8 }), Text(Text { value: "===", @4:1-4:4 })] -html: "<ul>\n<li>\n<p>===</p>\n<pre><code>a\n</code></pre>\n</li>\n</ul>\n<p>===</p>" -## quote_setext_dashes "> a\n> ---\n> ---\n> a" -nodes: [Blockquote(Blockquote { values: [Heading(Heading { depth: 2, values: [Text(Text { value: "a", @1:3-1:4 })], @1:3-2:6 }), HorizontalRule(HorizontalRule { marker: Some(Dash), @3:3-3:6 }), Text(Text { value: "a", @4:3-4:4 })], @1:1-4:4 })] -html: "<blockquote>\n<h2>a</h2>\n<hr />\n<p>a</p>\n</blockquote>" -## table_basic "| a | b |\n|---|:-:|\n| 1 | 2 |\n" -nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:3-1:4 })]:0, row: 0, @1:1-1:5 }), TableCell(TableCell { values: [Text(Text { value: "b", @1:7-1:8 })]:1, row: 0, @1:5-1:10 }), TableAlign(TableAlign { align: [None, Center], @2:1-2:1 }), TableCell(TableCell { values: [Text(Text { value: "1", @3:3-3:4 })]:0, row: 1, @3:1-3:5 }), TableCell(TableCell { values: [Text(Text { value: "2", @3:7-3:8 })]:1, row: 1, @3:5-3:10 })] -html: "<table>\n<thead>\n<tr>\n<th>a</th>\n<th align=\"center\">b</th>\n</tr>\n</thead>\n<tbody>\n<tr>\n<td>1</td>\n<td align=\"center\">2</td>\n</tr>\n</tbody>\n</table>\n" -## table_no_edge_pipes "a|b\n-|-\n1|2|3\n" -nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:1-1:2 })]:0, row: 0, @1:1-1:2 }), TableCell(TableCell { values: [Text(Text { value: "b", @1:3-1:4 })]:1, row: 0, @1:2-1:4 }), TableAlign(TableAlign { align: [None, None], @2:1-2:1 }), TableCell(TableCell { values: [Text(Text { value: "1", @3:1-3:2 })]:0, row: 1, @3:1-3:2 }), TableCell(TableCell { values: [Text(Text { value: "2", @3:3-3:4 })]:1, row: 1, @3:2-3:4 }), TableCell(TableCell { values: [Text(Text { value: "3", @3:5-3:6 })]:2, row: 1, @3:4-3:6 })] -html: "<table>\n<thead>\n<tr>\n<th>a</th>\n<th>b</th>\n</tr>\n</thead>\n<tbody>\n<tr>\n<td>1</td>\n<td>2</td>\n</tr>\n</tbody>\n</table>\n" -## table_short_row "|a|b|\n|-|-|\n|1|\n" -nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:2-1:3 })]:0, row: 0, @1:1-1:3 }), TableCell(TableCell { values: [Text(Text { value: "b", @1:4-1:5 })]:1, row: 0, @1:3-1:6 }), TableAlign(TableAlign { align: [None, None], @2:1-2:1 }), TableCell(TableCell { values: [Text(Text { value: "1", @3:2-3:3 })]:0, row: 1, @3:1-3:4 })] -html: "<table>\n<thead>\n<tr>\n<th>a</th>\n<th>b</th>\n</tr>\n</thead>\n<tbody>\n<tr>\n<td>1</td>\n<td></td>\n</tr>\n</tbody>\n</table>\n" -## table_escaped_pipe "| a \\| b | c |\n|--|--|\n" -nodes: [TableCell(TableCell { values: [Text(Text { value: "a | b", @1:3-1:9 })]:0, row: 0, @1:1-1:10 }), TableCell(TableCell { values: [Text(Text { value: "c", @1:12-1:13 })]:1, row: 0, @1:10-1:15 }), TableAlign(TableAlign { align: [None, None], @2:1-2:1 })] -html: "<table>\n<thead>\n<tr>\n<th>a | b</th>\n<th>c</th>\n</tr>\n</thead>\n</table>\n" -## table_indented " | a |\n |---|\n | b |\n" -nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:5-1:6 })]:0, row: 0, @1:1-1:8 }), TableAlign(TableAlign { align: [None], @2:1-2:1 }), TableCell(TableCell { values: [Text(Text { value: "b", @3:5-3:6 })]:0, row: 1, @3:1-3:8 })] -html: "<table>\n<thead>\n<tr>\n<th>a</th>\n</tr>\n</thead>\n<tbody>\n<tr>\n<td>b</td>\n</tr>\n</tbody>\n</table>\n" -## table_text_after "| a |\n|---|\ntext after\n" -nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:3-1:4 })]:0, row: 0, @1:1-1:6 }), TableAlign(TableAlign { align: [None], @2:1-2:1 }), TableCell(TableCell { values: [Text(Text { value: "text after", @3:1-3:11 })]:0, row: 1, @3:1-3:11 })] -html: "<table>\n<thead>\n<tr>\n<th>a</th>\n</tr>\n</thead>\n<tbody>\n<tr>\n<td>text after</td>\n</tr>\n</tbody>\n</table>\n" -## table_in_quote "> | a |\n> |---|\n> | b |\n" -nodes: [Blockquote(Blockquote { values: [TableCell(TableCell { values: [Text(Text { value: "a", @1:5-1:6 })]:0, row: 0, @1:3-1:8 }), TableAlign(TableAlign { align: [None], @2:1-2:1 }), TableCell(TableCell { values: [Text(Text { value: "b", @3:5-3:6 })]:0, row: 1, @3:3-3:8 })], @1:1-3:8 })] -html: "<blockquote>\n<table>\n<thead>\n<tr>\n<th>a</th>\n</tr>\n</thead>\n<tbody>\n<tr>\n<td>b</td>\n</tr>\n</tbody>\n</table>\n</blockquote>\n" -## table_after_paragraph "x\n| a |\n|---|\n" -nodes: [Text(Text { value: "x", @1:1-1:2 }), TableCell(TableCell { values: [Text(Text { value: "a", @2:3-2:4 })]:0, row: 0, @2:1-2:6 }), TableAlign(TableAlign { align: [None], @3:1-3:1 })] -html: "<p>x</p>\n<table>\n<thead>\n<tr>\n<th>a</th>\n</tr>\n</thead>\n</table>\n" -## table_column_mismatch "| a | b |\n|---|\n" -nodes: [Text(Text { value: "| a | b |\n|---|", @1:1-2:6 })] -html: "<p>| a | b |\n|---|</p>\n" -## table_bad_delimiter "|a|\n|:|\n" -nodes: [Text(Text { value: "|a|\n|:|", @1:1-2:4 })] -html: "<p>|a|\n|:|</p>\n" -## table_then_blank_para "|a|\n|-|\n\nnext\n" -nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:2-1:3 })]:0, row: 0, @1:1-1:4 }), TableAlign(TableAlign { align: [None], @2:1-2:1 }), Text(Text { value: "next", @4:1-4:5 })] -html: "<table>\n<thead>\n<tr>\n<th>a</th>\n</tr>\n</thead>\n</table>\n<p>next</p>\n" -## table_then_quote "|a|\n|-|\n> q\n" -nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:2-1:3 })]:0, row: 0, @1:1-1:4 }), TableAlign(TableAlign { align: [None], @2:1-2:1 }), Blockquote(Blockquote { values: [Text(Text { value: "q", @3:3-3:4 })], @3:1-3:4 })] -html: "<table>\n<thead>\n<tr>\n<th>a</th>\n</tr>\n</thead>\n</table>\n<blockquote>\n<p>q</p>\n</blockquote>\n" -## table_then_heading "|a|\n|-|\n# h\n" -nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:2-1:3 })]:0, row: 0, @1:1-1:4 }), TableAlign(TableAlign { align: [None], @2:1-2:1 }), Heading(Heading { depth: 1, values: [Text(Text { value: "h", @3:3-3:4 })], @3:1-3:4 })] -html: "<table>\n<thead>\n<tr>\n<th>a</th>\n</tr>\n</thead>\n</table>\n<h1>h</h1>\n" -## table_then_fence "|a|\n|-|\n```\nx\n```\n" -nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:2-1:3 })]:0, row: 0, @1:1-1:4 }), TableAlign(TableAlign { align: [None], @2:1-2:1 }), Code(Code { value: "x", lang: None, @3:1-5:4, meta: None, fence: true })] -html: "<table>\n<thead>\n<tr>\n<th>a</th>\n</tr>\n</thead>\n</table>\n<pre><code>x\n</code></pre>\n" -## table_then_list "|a|\n|-|\n- x\n" -nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:2-1:3 })]:0, row: 0, @1:1-1:4 }), TableAlign(TableAlign { align: [None], @2:1-2:1 }), List(List { values: [Text(Text { value: "x", @3:3-3:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @3:3-3:4 })] -html: "<table>\n<thead>\n<tr>\n<th>a</th>\n</tr>\n</thead>\n</table>\n<ul>\n<li>x</li>\n</ul>\n" -## table_two "|a|\n|-|\n\n|b|\n|-|\n" -nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:2-1:3 })]:0, row: 0, @1:1-1:4 }), TableAlign(TableAlign { align: [None], @2:1-2:1 }), TableCell(TableCell { values: [Text(Text { value: "b", @4:2-4:3 })]:0, row: 0, @4:1-4:4 }), TableAlign(TableAlign { align: [None], @5:1-5:1 })] -html: "<table>\n<thead>\n<tr>\n<th>a</th>\n</tr>\n</thead>\n</table>\n<table>\n<thead>\n<tr>\n<th>b</th>\n</tr>\n</thead>\n</table>\n" -## table_align_spaces "|a|b|\n|:-|-:|\n| x | y|\n" -nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:2-1:3 })]:0, row: 0, @1:1-1:3 }), TableCell(TableCell { values: [Text(Text { value: "b", @1:4-1:5 })]:1, row: 0, @1:3-1:6 }), TableAlign(TableAlign { align: [Left, Right], @2:1-2:1 }), TableCell(TableCell { values: [Text(Text { value: "x", @3:4-3:5 })]:0, row: 1, @3:1-3:7 }), TableCell(TableCell { values: [Text(Text { value: "y", @3:10-3:11 })]:1, row: 1, @3:7-3:12 })] -html: "<table>\n<thead>\n<tr>\n<th align=\"left\">a</th>\n<th align=\"right\">b</th>\n</tr>\n</thead>\n<tbody>\n<tr>\n<td align=\"left\">x</td>\n<td align=\"right\">y</td>\n</tr>\n</tbody>\n</table>\n" -## table_indented_code_row "|a|\n|-|\n|b|\n |c|\n" -nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:2-1:3 })]:0, row: 0, @1:1-1:4 }), TableAlign(TableAlign { align: [None], @2:1-2:1 }), TableCell(TableCell { values: [Text(Text { value: "b", @3:2-3:3 })]:0, row: 1, @3:1-3:4 }), Code(Code { value: "|c|", lang: None, @4:1-4:8, meta: None, fence: false })] -html: "<table>\n<thead>\n<tr>\n<th>a</th>\n</tr>\n</thead>\n<tbody>\n<tr>\n<td>b</td>\n</tr>\n</tbody>\n</table>\n<pre><code>|c|\n</code></pre>\n" -## table_no_body "| a |\n| - |\n" -nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:3-1:4 })]:0, row: 0, @1:1-1:6 }), TableAlign(TableAlign { align: [None], @2:1-2:1 })] -html: "<table>\n<thead>\n<tr>\n<th>a</th>\n</tr>\n</thead>\n</table>\n" -## table_empty_cells "||\n|-|\n" -nodes: [TableCell(TableCell { values: []:0, row: 0, @1:1-1:3 }), TableAlign(TableAlign { align: [None], @2:1-2:1 })] -html: "<table>\n<thead>\n<tr>\n<th></th>\n</tr>\n</thead>\n</table>\n" -## table_lone_pipe_row "|a|\n|-|\n|\n" -nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:2-1:3 })]:0, row: 0, @1:1-1:4 }), TableAlign(TableAlign { align: [None], @2:1-2:1 }), TableCell(TableCell { values: []:0, row: 1, @3:1-3:2 })] -html: "<table>\n<thead>\n<tr>\n<th>a</th>\n</tr>\n</thead>\n<tbody>\n<tr>\n<td></td>\n</tr>\n</tbody>\n</table>\n" -## table_no_trailing_pipe "| a | b\n|-|-\n| c | d\n" -nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:3-1:4 })]:0, row: 0, @1:1-1:5 }), TableCell(TableCell { values: [Text(Text { value: "b", @1:7-1:8 })]:1, row: 0, @1:5-1:8 }), TableAlign(TableAlign { align: [None, None], @2:1-2:1 }), TableCell(TableCell { values: [Text(Text { value: "c", @3:3-3:4 })]:0, row: 1, @3:1-3:5 }), TableCell(TableCell { values: [Text(Text { value: "d", @3:7-3:8 })]:1, row: 1, @3:5-3:8 })] -html: "<table>\n<thead>\n<tr>\n<th>a</th>\n<th>b</th>\n</tr>\n</thead>\n<tbody>\n<tr>\n<td>c</td>\n<td>d</td>\n</tr>\n</tbody>\n</table>\n" -## table_multibyte "| あ | い |\n|---|---|\n| う | え |\n" -nodes: [TableCell(TableCell { values: [Text(Text { value: "あ", @1:3-1:6 })]:0, row: 0, @1:1-1:7 }), TableCell(TableCell { values: [Text(Text { value: "い", @1:9-1:12 })]:1, row: 0, @1:7-1:14 }), TableAlign(TableAlign { align: [None, None], @2:1-2:1 }), TableCell(TableCell { values: [Text(Text { value: "う", @3:3-3:6 })]:0, row: 1, @3:1-3:7 }), TableCell(TableCell { values: [Text(Text { value: "え", @3:9-3:12 })]:1, row: 1, @3:7-3:14 })] -html: "<table>\n<thead>\n<tr>\n<th>あ</th>\n<th>い</th>\n</tr>\n</thead>\n<tbody>\n<tr>\n<td>う</td>\n<td>え</td>\n</tr>\n</tbody>\n</table>\n" -## not_table_setext "Title\n---\n" -nodes: [Heading(Heading { depth: 2, values: [Text(Text { value: "Title", @1:1-1:6 })], @1:1-2:4 })] -html: "<h2>Title</h2>\n" -## not_table_no_pipe_delim "a|b\n---\n" -nodes: [Heading(Heading { depth: 2, values: [Text(Text { value: "a|b", @1:1-1:4 })], @1:1-2:4 })] -html: "<h2>a|b</h2>\n" -## table_header_no_pipe "a\n|-|\n" -nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:1-1:2 })]:0, row: 0, @1:1-1:2 }), TableAlign(TableAlign { align: [None], @2:1-2:1 })] -html: "<table>\n<thead>\n<tr>\n<th>a</th>\n</tr>\n</thead>\n</table>\n" -## table_one_col_dash "|a|\n-\n" -nodes: [Heading(Heading { depth: 2, values: [Text(Text { value: "|a|", @1:1-1:4 })], @1:1-2:2 })] -html: "<h2>|a|</h2>\n" -## inline_escape "a\\*b" -nodes: [Text(Text { value: "a*b", @1:1-1:5 })] -html: "<p>a*b</p>" -## inline_escape_nonpunct "a\\qb" -nodes: [Text(Text { value: "a\\qb", @1:1-1:5 })] -html: "<p>a\\qb</p>" -## inline_entities "a&amp;b &copy; &#35; &#x41; &unknown; &" -nodes: [Text(Text { value: "a&b © # A &unknown; &", @1:1-1:40 })] -html: "<p>a&amp;b © # A &amp;unknown; &amp;</p>" -## inline_emphasis_all "a *b* **c** _d_ __e__ ***f***" -nodes: [Text(Text { value: "a ", @1:1-1:3 }), Emphasis(Emphasis { values: [Text(Text { value: "b", @1:4-1:5 })], @1:3-1:6 }), Text(Text { value: " ", @1:6-1:7 }), Strong(Strong { values: [Text(Text { value: "c", @1:9-1:10 })], @1:7-1:12 }), Text(Text { value: " ", @1:12-1:13 }), Emphasis(Emphasis { values: [Text(Text { value: "d", @1:14-1:15 })], @1:13-1:16 }), Text(Text { value: " ", @1:16-1:17 }), Strong(Strong { values: [Text(Text { value: "e", @1:19-1:20 })], @1:17-1:22 }), Text(Text { value: " ", @1:22-1:23 }), Emphasis(Emphasis { values: [Strong(Strong { values: [Text(Text { value: "f", @1:26-1:27 })], @1:24-1:29 })], @1:23-1:30 })] -html: "<p>a <em>b</em> <strong>c</strong> <em>d</em> <strong>e</strong> <em><strong>f</strong></em></p>" -## inline_code "`code` `` a`b `` ` a `" -nodes: [CodeInline(CodeInline { value: "code", @1:1-1:7 }), Text(Text { value: " ", @1:7-1:8 }), CodeInline(CodeInline { value: "a`b", @1:8-1:17 }), Text(Text { value: " ", @1:17-1:18 }), CodeInline(CodeInline { value: "a", @1:18-1:23 })] -html: "<p><code>code</code> <code>a`b</code> <code>a</code></p>" -## inline_code_unclosed "`a ``b`" -nodes: [CodeInline(CodeInline { value: "a ``b", @1:1-1:8 })] -html: "<p><code>a ``b</code></p>" -## inline_hard_break_spaces "a \nb" -nodes: [Text(Text { value: "a", @1:1-1:2 }), Break(Break { @1:2-2:1 }), Text(Text { value: "b", @2:1-2:2 })] -html: "<p>a<br />\nb</p>" -## inline_hard_break_backslash "a\\\nb" -nodes: [Text(Text { value: "a", @1:1-1:2 }), Break(Break { @1:2-2:1 }), Text(Text { value: "b", @2:1-2:2 })] -html: "<p>a<br />\nb</p>" -## inline_soft_break "a\nb" -nodes: [Text(Text { value: "a\nb", @1:1-2:2 })] -html: "<p>a\nb</p>" -## inline_soft_break_trailing_space "a \nb" -nodes: [Text(Text { value: "a\nb", @1:1-2:2 })] -html: "<p>a\nb</p>" -## inline_emphasis_multiline "*a\nb*" -nodes: [Emphasis(Emphasis { values: [Text(Text { value: "a\nb", @1:2-2:2 })], @1:1-2:3 })] -html: "<p><em>a\nb</em></p>" -## inline_strike "a ~b~ ~~c~~ ~~~d~~~" -nodes: [Text(Text { value: "a ", @1:1-1:3 }), Delete(Delete { values: [Text(Text { value: "b", @1:4-1:5 })], @1:3-1:6 }), Text(Text { value: " ", @1:6-1:7 }), Delete(Delete { values: [Text(Text { value: "c", @1:9-1:10 })], @1:7-1:12 }), Text(Text { value: " ~~~d~~~", @1:12-1:20 })] -html: "<p>a <del>b</del> <del>c</del> ~~~d~~~</p>" -## inline_math "$x$ $$y$$ $ a" -nodes: [MathInline(MathInline { value: "x", @1:1-1:4 }), Text(Text { value: " ", @1:4-1:5 }), MathInline(MathInline { value: "y", @1:5-1:10 }), Text(Text { value: " $ a", @1:10-1:14 })] -html: "<p><code class=\"language-math math-inline\">x</code> <code class=\"language-math math-inline\">y</code> $ a</p>" -## inline_autolinks "<http://a.b> <a@b.c> <span> <!-- c --> <br/>" -nodes: [Link(Link { url: Url("http://a.b"), title: None, values: [Text(Text { value: "http://a.b", @1:2-1:12 })], @1:1-1:13 }), Text(Text { value: " ", @1:13-1:14 }), Link(Link { url: Url("mailto:a@b.c"), title: None, values: [Text(Text { value: "a@b.c", @1:15-1:20 })], @1:14-1:21 }), Text(Text { value: " ", @1:21-1:22 }), Html(Html { value: "<span>", @1:22-1:28 }), Text(Text { value: " ", @1:28-1:29 }), Html(Html { value: "<!-- c -->", @1:29-1:39 }), Text(Text { value: " ", @1:39-1:40 }), Html(Html { value: "<br/>", @1:40-1:45 })] -html: "<p><a href=\"http://a.b\">http://a.b</a> <a href=\"mailto:a@b.c\">a@b.c</a> <span> <!-- c --> <br/></p>" -## inline_links "[a](http://x \"t\") [b][c] [d][] [e] ![f](g) ![h][i]" -nodes: [Link(Link { url: Url("http://x"), title: Some(Title("t")), values: [Text(Text { value: "a", @1:2-1:3 })], @1:1-1:18 }), Text(Text { value: " [b][c] [d][] [e] ", @1:18-1:36 }), Image(Image { alt: "f", url: "g", title: None, @1:36-1:43 }), Text(Text { value: " ![h][i]", @1:43-1:51 })] -html: "<p><a href=\"http://x\" title=\"t\">a</a> [b][c] [d][] [e] <img src=\"g\" alt=\"f\" /> ![h][i]</p>" -## inline_link_forms "[a](<b c> 'x') [d]( e ) [f](g (h)) [i](j \"k\\\"l\") [m](n&amp;o)" -nodes: [Link(Link { url: Url("b c"), title: Some(Title("x")), values: [Text(Text { value: "a", @1:2-1:3 })], @1:1-1:15 }), Text(Text { value: " ", @1:15-1:16 }), Link(Link { url: Url("e"), title: None, values: [Text(Text { value: "d", @1:17-1:18 })], @1:16-1:24 }), Text(Text { value: " ", @1:24-1:25 }), Link(Link { url: Url("g"), title: Some(Title("h")), values: [Text(Text { value: "f", @1:26-1:27 })], @1:25-1:35 }), Text(Text { value: " ", @1:35-1:36 }), Link(Link { url: Url("j"), title: Some(Title("k\"l")), values: [Text(Text { value: "i", @1:37-1:38 })], @1:36-1:49 }), Text(Text { value: " ", @1:49-1:50 }), Link(Link { url: Url("n&o"), title: None, values: [Text(Text { value: "m", @1:51-1:52 })], @1:50-1:62 })] -html: "<p><a href=\"b%20c\" title=\"x\">a</a> <a href=\"e\">d</a> <a href=\"g\" title=\"h\">f</a> <a href=\"j\" title=\"k&quot;l\">i</a> <a href=\"n&amp;o\">m</a></p>" -## inline_link_multiline "[a](b\n\"t\") [c](d" -nodes: [Link(Link { url: Url("b"), title: Some(Title("t")), values: [Text(Text { value: "a", @1:2-1:3 })], @1:1-2:5 }), Text(Text { value: " [c](d", @2:5-2:11 })] -html: "<p><a href=\"b\" title=\"t\">a</a> [c](d</p>" -## inline_image_alt "![a *b* `c`](d) ![e ![f](g)](h)" -nodes: [Image(Image { alt: "a b c", url: "d", title: None, @1:1-1:16 }), Text(Text { value: " ", @1:16-1:17 }), Image(Image { alt: "e f", url: "h", title: None, @1:17-1:32 })] -html: "<p><img src=\"d\" alt=\"a b c\" /> <img src=\"h\" alt=\"e f\" /></p>" -## inline_link_in_link "[a [b](c) d](e)" -nodes: [Text(Text { value: "[a ", @1:1-1:4 }), Link(Link { url: Url("c"), title: None, values: [Text(Text { value: "b", @1:5-1:6 })], @1:4-1:10 }), Text(Text { value: " d](e)", @1:10-1:16 })] -html: "<p>[a <a href=\"c\">b</a> d](e)</p>" -## inline_emphasis_link "*[a](b)* **[c][a]**" -nodes: [Emphasis(Emphasis { values: [Link(Link { url: Url("b"), title: None, values: [Text(Text { value: "a", @1:3-1:4 })], @1:2-1:8 })], @1:1-1:9 }), Text(Text { value: " ", @1:9-1:10 }), Strong(Strong { values: [Text(Text { value: "[c][a]", @1:12-1:18 })], @1:10-1:20 })] -html: "<p><em><a href=\"b\">a</a></em> <strong>[c][a]</strong></p>" -## inline_html "<a href=\"x\">t</a> <a\nhref=x>" -nodes: [Html(Html { value: "<a href=\"x\">", @1:1-1:13 }), Text(Text { value: "t", @1:13-1:14 }), Html(Html { value: "</a>", @1:14-1:18 }), Text(Text { value: " ", @1:18-1:19 }), Html(Html { value: "<a\nhref=x>", @1:19-2:8 })] -html: "<p><a href=\"x\">t</a> <a\nhref=x></p>" -## inline_html_misc "<http://a b> <a+b@c> <ab> </a> <?x?> <![CDATA[x]]> <!X y>" -nodes: [Text(Text { value: "<", @1:1-1:2 }), Link(Link { url: Url("http://a"), title: None, values: [Text(Text { value: "http://a", @1:2-1:10 })], @1:2-1:10 }), Text(Text { value: " b> ", @1:10-1:14 }), Link(Link { url: Url("mailto:a+b@c"), title: None, values: [Text(Text { value: "a+b@c", @1:15-1:20 })], @1:14-1:21 }), Text(Text { value: " ", @1:21-1:22 }), Html(Html { value: "<ab>", @1:22-1:26 }), Text(Text { value: " ", @1:26-1:27 }), Html(Html { value: "</a>", @1:27-1:31 }), Text(Text { value: " ", @1:31-1:32 }), Html(Html { value: "<?x?>", @1:32-1:37 }), Text(Text { value: " ", @1:37-1:38 }), Html(Html { value: "<![CDATA[x]]>", @1:38-1:51 }), Text(Text { value: " ", @1:51-1:52 }), Html(Html { value: "<!X y>", @1:52-1:58 })] -html: "<p>&lt;<a href=\"http://a\">http://a</a> b&gt; <a href=\"mailto:a+b@c\">a+b@c</a> <ab> </a> <?x?> <![CDATA[x]]> <!X y></p>" -## inline_nested_emphasis "*a **b** c* **a *b* c**" -nodes: [Emphasis(Emphasis { values: [Text(Text { value: "a ", @1:2-1:4 }), Strong(Strong { values: [Text(Text { value: "b", @1:6-1:7 })], @1:4-1:9 }), Text(Text { value: " c", @1:9-1:11 })], @1:1-1:12 }), Text(Text { value: " ", @1:12-1:13 }), Strong(Strong { values: [Text(Text { value: "a ", @1:15-1:17 }), Emphasis(Emphasis { values: [Text(Text { value: "b", @1:18-1:19 })], @1:17-1:20 }), Text(Text { value: " c", @1:20-1:22 })], @1:13-1:24 })] -html: "<p><em>a <strong>b</strong> c</em> <strong>a <em>b</em> c</strong></p>" -## inline_intraword "a_b_c a*b*c _a_b" -nodes: [Text(Text { value: "a_b_c a", @1:1-1:8 }), Emphasis(Emphasis { values: [Text(Text { value: "b", @1:9-1:10 })], @1:8-1:11 }), Text(Text { value: "c _a_b", @1:11-1:17 })] -html: "<p>a_b_c a<em>b</em>c _a_b</p>" -## inline_unmatched "*a **b _c ~d [e ![f" -nodes: [Text(Text { value: "*a **b _c ~d [e ![f", @1:1-1:20 })] -html: "<p>*a **b _c ~d [e ![f</p>" -## inline_rule_of_three "*foo**bar**baz* ***a** b*" -nodes: [Emphasis(Emphasis { values: [Text(Text { value: "foo", @1:2-1:5 }), Strong(Strong { values: [Text(Text { value: "bar", @1:7-1:10 })], @1:5-1:12 }), Text(Text { value: "baz", @1:12-1:15 })], @1:1-1:16 }), Text(Text { value: " ", @1:16-1:17 }), Emphasis(Emphasis { values: [Strong(Strong { values: [Text(Text { value: "a", @1:20-1:21 })], @1:18-1:23 }), Text(Text { value: " b", @1:23-1:25 })], @1:17-1:26 })] -html: "<p><em>foo<strong>bar</strong>baz</em> <em><strong>a</strong> b</em></p>" -## inline_heading "# a *b* `c`\n" -nodes: [Heading(Heading { depth: 1, values: [Text(Text { value: "a ", @1:3-1:5 }), Emphasis(Emphasis { values: [Text(Text { value: "b", @1:6-1:7 })], @1:5-1:8 }), Text(Text { value: " ", @1:8-1:9 }), CodeInline(CodeInline { value: "c", @1:9-1:12 })], @1:1-1:12 })] -html: "<h1>a <em>b</em> <code>c</code></h1>\n" -## inline_setext_heading "a *b*\nc\n===\n" -nodes: [Heading(Heading { depth: 1, values: [Text(Text { value: "a ", @1:1-1:3 }), Emphasis(Emphasis { values: [Text(Text { value: "b", @1:4-1:5 })], @1:3-1:6 }), Text(Text { value: "\nc", @1:6-2:2 })], @1:1-3:4 })] -html: "<h1>a <em>b</em>\nc</h1>\n" -## inline_list_item "- a *b*\n c **d**\n" -nodes: [List(List { values: [Text(Text { value: "a ", @1:3-1:5 }), Emphasis(Emphasis { values: [Text(Text { value: "b", @1:6-1:7 })], @1:5-1:8 }), Text(Text { value: "\nc ", @1:8-2:5 }), Strong(Strong { values: [Text(Text { value: "d", @2:7-2:8 })], @2:5-2:10 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-2:10 })] -html: "<ul>\n<li>a <em>b</em>\nc <strong>d</strong></li>\n</ul>\n" -## inline_table_cell "| a *b* | `c\\|d` |\n|-|-|\n" -nodes: [TableCell(TableCell { values: [Text(Text { value: "a ", @1:3-1:5 }), Emphasis(Emphasis { values: [Text(Text { value: "b", @1:6-1:7 })], @1:5-1:8 })]:0, row: 0, @1:1-1:9 }), TableCell(TableCell { values: [CodeInline(CodeInline { value: "c|d", @1:11-1:17 })]:1, row: 0, @1:9-1:19 }), TableAlign(TableAlign { align: [None, None], @2:1-2:1 })] -html: "<table>\n<thead>\n<tr>\n<th>a <em>b</em></th>\n<th><code>c|d</code></th>\n</tr>\n</thead>\n</table>\n" -## inline_multibyte "あ*い*う `え` [お](か)" -nodes: [Text(Text { value: "あ", @1:1-1:4 }), Emphasis(Emphasis { values: [Text(Text { value: "い", @1:5-1:8 })], @1:4-1:9 }), Text(Text { value: "う ", @1:9-1:13 }), CodeInline(CodeInline { value: "え", @1:13-1:18 }), Text(Text { value: " ", @1:18-1:19 }), Link(Link { url: Url("か"), title: None, values: [Text(Text { value: "お", @1:20-1:23 })], @1:19-1:29 })] -html: "<p>あ<em>い</em>う <code>え</code> <a href=\"%E3%81%8B\">お</a></p>" -## inline_crlf_break "a \r\nb\r\n" -nodes: [Text(Text { value: "a", @1:1-1:2 }), Break(Break { @1:2-2:1 }), Text(Text { value: "b", @2:1-2:2 })] -html: "<p>a<br />\r\nb</p>\r\n" -## math_multiline "x$\nx$" -nodes: [Text(Text { value: "x", @1:1-1:2 }), MathInline(MathInline { value: "\nx", @1:2-2:3 })] -html: "<p>x<code class=\"language-math math-inline\"> x</code></p>" -## decl_email "x<!a@b.co>" -nodes: [Text(Text { value: "x", @1:1-1:2 }), Html(Html { value: "<!a@b.co>", @1:2-1:11 })] -html: "<p>x<!a@b.co></p>" -## code_multiline "x`\nx`" -nodes: [Text(Text { value: "x", @1:1-1:2 }), CodeInline(CodeInline { value: "\nx", @1:2-2:3 })] -html: "<p>x<code> x</code></p>" -## def_basic "[a]: b" -nodes: [Definition(Definition { @1:1-1:7, url: Url("b"), title: None, ident: "a", label: Some("a") })] -html: "" -## def_title "[a]: <b c> 'T'" -nodes: [Definition(Definition { @1:1-1:15, url: Url("b c"), title: Some(Title("T")), ident: "a", label: Some("a") })] -html: "" -## def_next_lines "[a]:\nb\n\"t\"" -nodes: [Definition(Definition { @1:1-3:4, url: Url("b"), title: Some(Title("t")), ident: "a", label: Some("a") })] -html: "" -## def_title_junk_next_line "[a]: b\n\"t\" x" -nodes: [Definition(Definition { @1:1-1:7, url: Url("b"), title: None, ident: "a", label: Some("a") }), Text(Text { value: "\"t\" x", @2:1-2:6 })] -html: "<p>&quot;t&quot; x</p>" -## def_title_junk_same_line "[a]: b \"t\" x" -nodes: [Text(Text { value: "[a]: b \"t\" x", @1:1-1:13 })] -html: "<p>[a]: b &quot;t&quot; x</p>" -## def_then_paragraph "[a]: b\nc" -nodes: [Definition(Definition { @1:1-1:7, url: Url("b"), title: None, ident: "a", label: Some("a") }), Text(Text { value: "c", @2:1-2:2 })] -html: "<p>c</p>" -## def_two_then_text "[a]: b\n[c]: d\ntext" -nodes: [Definition(Definition { @1:1-1:7, url: Url("b"), title: None, ident: "a", label: Some("a") }), Definition(Definition { @2:1-2:7, url: Url("d"), title: None, ident: "c", label: Some("c") }), Text(Text { value: "text", @3:1-3:5 })] -html: "<p>text</p>" -## def_not_at_start "text\n[a]: b" -nodes: [Text(Text { value: "text\n[a]: b", @1:1-2:7 })] -html: "<p>text\n[a]: b</p>" -## def_then_setext_equals "[a]: b\n=== " -nodes: [Definition(Definition { @1:1-1:7, url: Url("b"), title: None, ident: "a", label: Some("a") }), Text(Text { value: "===", @2:1-2:4 })] -html: "<p>===</p>" -## def_then_setext_dashes "[a]: b\n---" -nodes: [Definition(Definition { @1:1-1:7, url: Url("b"), title: None, ident: "a", label: Some("a") }), HorizontalRule(HorizontalRule { marker: Some(Dash), @2:1-2:4 })] -html: "<hr />" -## def_indent1 " [a]: b" -nodes: [Definition(Definition { @1:1-1:8, url: Url("b"), title: None, ident: "a", label: Some("a") })] -html: "" -## def_indent3 " [a]: b" -nodes: [Definition(Definition { @1:1-1:10, url: Url("b"), title: None, ident: "a", label: Some("a") })] -html: "" -## def_indent4 " [a]: b" -nodes: [Code(Code { value: "[a]: b", lang: None, @1:1-1:11, meta: None, fence: false })] -html: "<pre><code>[a]: b\n</code></pre>" -## def_in_quote "> [a]: b" -nodes: [Blockquote(Blockquote { values: [Definition(Definition { @1:3-1:9, url: Url("b"), title: None, ident: "a", label: Some("a") })], @1:1-1:9 })] -html: "<blockquote>\n</blockquote>" -## def_in_list "- [a]: b" -nodes: [List(List { values: [Definition(Definition { @1:3-1:9, url: Url("b"), title: None, ident: "a", label: Some("a") })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-1:9 })] -html: "<ul>\n<li></li>\n</ul>" -## def_no_dest "[a]:" -nodes: [Text(Text { value: "[a]:", @1:1-1:5 })] -html: "<p>[a]:</p>" -## def_empty_label "[]: b" -nodes: [Text(Text { value: "[]: b", @1:1-1:6 })] -html: "<p>[]: b</p>" -## def_blank_label "[ ]: b" -nodes: [Text(Text { value: "[ ]: b", @1:1-1:7 })] -html: "<p>[ ]: b</p>" -## def_escaped_label "[a\\]b]: c" -nodes: [Definition(Definition { @1:1-1:10, url: Url("c"), title: None, ident: "a\\]b", label: Some("a]b") })] -html: "" -## def_multiline_title "[a]: b \"t\nt\"" -nodes: [Definition(Definition { @1:1-2:3, url: Url("b"), title: Some(Title("t\nt")), ident: "a", label: Some("a") })] -html: "" -## def_title_indented "[a]: b\n \"t\"" -nodes: [Definition(Definition { @1:1-2:6, url: Url("b"), title: Some(Title("t")), ident: "a", label: Some("a") })] -html: "" -## def_no_space "[a]:b" -nodes: [Definition(Definition { @1:1-1:6, url: Url("b"), title: None, ident: "a", label: Some("a") })] -html: "" -## def_dest_junk "[a]: b c" -nodes: [Text(Text { value: "[a]: b c", @1:1-1:9 })] -html: "<p>[a]: b c</p>" -## def_duplicate_labels "[A b]: c\n[a b]: d" -nodes: [Definition(Definition { @1:1-1:10, url: Url("c"), title: None, ident: "a b", label: Some("A b") }), Definition(Definition { @2:1-2:9, url: Url("d"), title: None, ident: "a b", label: Some("a b") })] -html: "" -## def_empty_angle "[a]: <>" -nodes: [Definition(Definition { @1:1-1:8, url: Url(""), title: None, ident: "a", label: Some("a") })] -html: "" -## def_quote_in_dest "[a]: 'x" -nodes: [Definition(Definition { @1:1-1:8, url: Url("'x"), title: None, ident: "a", label: Some("a") })] -html: "" -## def_use_shortcut "[a]: /u\n\n[a] [a][] [b][a] [a][b] [A b]" -nodes: [Definition(Definition { @1:1-1:8, url: Url("/u"), title: None, ident: "a", label: Some("a") }), LinkRef(LinkRef { ident: "a", label: Some("a"), values: [Text(Text { value: "a", @3:2-3:3 })], @3:1-3:4 }), Text(Text { value: " ", @3:4-3:5 }), LinkRef(LinkRef { ident: "a", label: Some("a"), values: [Text(Text { value: "a", @3:6-3:7 })], @3:5-3:10 }), Text(Text { value: " ", @3:10-3:11 }), LinkRef(LinkRef { ident: "a", label: Some("a"), values: [Text(Text { value: "b", @3:12-3:13 })], @3:11-3:17 }), Text(Text { value: " [a][b] [A b]", @3:17-3:30 })] -html: "<p><a href=\"/u\">a</a> <a href=\"/u\">a</a> <a href=\"/u\">b</a> [a][b] [A b]</p>" -## def_use_normalized "[ab cd]: /u\n\n[AB CD] [ab\ncd]" -nodes: [Definition(Definition { @1:1-1:12, url: Url("/u"), title: None, ident: "ab cd", label: Some("ab cd") }), LinkRef(LinkRef { ident: "ab cd", label: Some("AB CD"), values: [Text(Text { value: "AB CD", @3:2-3:9 })], @3:1-3:10 }), Text(Text { value: " ", @3:10-3:11 }), LinkRef(LinkRef { ident: "ab cd", label: Some("ab\ncd"), values: [Text(Text { value: "ab\ncd", @3:12-4:3 })], @3:11-4:4 })] -html: "<p><a href=\"/u\">AB CD</a> <a href=\"/u\">ab\ncd</a></p>" -## def_use_before "[a] and [b][a]\n\n[a]: /u \"T\"" -nodes: [LinkRef(LinkRef { ident: "a", label: Some("a"), values: [Text(Text { value: "a", @1:2-1:3 })], @1:1-1:4 }), Text(Text { value: " and ", @1:4-1:9 }), LinkRef(LinkRef { ident: "a", label: Some("a"), values: [Text(Text { value: "b", @1:10-1:11 })], @1:9-1:15 }), Definition(Definition { @3:1-3:12, url: Url("/u"), title: Some(Title("T")), ident: "a", label: Some("a") })] -html: "<p><a href=\"/u\" title=\"T\">a</a> and <a href=\"/u\" title=\"T\">b</a></p>\n" -## def_use_in_link_text "[a]: b\n\n[a](x) [a]" -nodes: [Definition(Definition { @1:1-1:7, url: Url("b"), title: None, ident: "a", label: Some("a") }), Link(Link { url: Url("x"), title: None, values: [Text(Text { value: "a", @3:2-3:3 })], @3:1-3:7 }), Text(Text { value: " ", @3:7-3:8 }), LinkRef(LinkRef { ident: "a", label: Some("a"), values: [Text(Text { value: "a", @3:9-3:10 })], @3:8-3:11 })] -html: "<p><a href=\"x\">a</a> <a href=\"b\">a</a></p>" -## def_use_image "[x]: /u\n\n![x] ![y][x] ![x][]" -nodes: [Definition(Definition { @1:1-1:8, url: Url("/u"), title: None, ident: "x", label: Some("x") }), ImageRef(ImageRef { alt: "x", ident: "x", label: Some("x"), @3:1-3:5 }), Text(Text { value: " ", @3:5-3:6 }), ImageRef(ImageRef { alt: "y", ident: "x", label: Some("x"), @3:6-3:13 }), Text(Text { value: " ", @3:13-3:14 }), ImageRef(ImageRef { alt: "x", ident: "x", label: Some("x"), @3:14-3:20 })] -html: "<p><img src=\"/u\" alt=\"x\" /> <img src=\"/u\" alt=\"y\" /> <img src=\"/u\" alt=\"x\" /></p>" -## def_link_in_emphasis "[a]: b\n\n*[a]* **[c][a]**" -nodes: [Definition(Definition { @1:1-1:7, url: Url("b"), title: None, ident: "a", label: Some("a") }), Emphasis(Emphasis { values: [LinkRef(LinkRef { ident: "a", label: Some("a"), values: [Text(Text { value: "a", @3:3-3:4 })], @3:2-3:5 })], @3:1-3:6 }), Text(Text { value: " ", @3:6-3:7 }), Strong(Strong { values: [LinkRef(LinkRef { ident: "a", label: Some("a"), values: [Text(Text { value: "c", @3:10-3:11 })], @3:9-3:15 })], @3:7-3:17 })] -html: "<p><em><a href=\"b\">a</a></em> <strong><a href=\"b\">c</a></strong></p>" -## fn_basic "[^a]: b" -nodes: [Footnote(Footnote { ident: "a", values: [Text(Text { value: "b", @1:7-1:8 })], @1:1-1:8 })] -html: "" -## fn_lazy "[^a]: b\nc" -nodes: [Footnote(Footnote { ident: "a", values: [Text(Text { value: "b\nc", @1:7-2:2 })], @1:1-2:2 })] -html: "" -## fn_indented_continuation "[^a]: b\n c" -nodes: [Footnote(Footnote { ident: "a", values: [Text(Text { value: "b\nc", @1:7-2:6 })], @1:1-2:6 })] -html: "" -## fn_two_paragraphs "[^a]: b\n\n c" -nodes: [Footnote(Footnote { ident: "a", values: [Text(Text { value: "b", @1:7-1:8 }), Text(Text { value: "c", @3:5-3:6 })], @1:1-3:6 })] -html: "" -## fn_two_paragraphs_then_text "[^a]: b\n\n c\n\nd" -nodes: [Footnote(Footnote { ident: "a", values: [Text(Text { value: "b", @1:7-1:8 }), Text(Text { value: "c", @3:5-3:6 })], @1:1-4:1 }), Text(Text { value: "d", @5:1-5:2 })] -html: "<p>d</p>" -## fn_two_space_continuation "[^a]: b\n c" -nodes: [Footnote(Footnote { ident: "a", values: [Text(Text { value: "b\nc", @1:7-2:4 })], @1:1-2:4 })] -html: "" -## fn_empty "[^a]:" -nodes: [Footnote(Footnote { ident: "a", values: [], @1:1-1:6 })] -html: "" -## fn_empty_space "[^a]: " -nodes: [Footnote(Footnote { ident: "a", values: [], @1:1-1:7 })] -html: "" -## fn_content_next_line "[^a]:\n b" -nodes: [Footnote(Footnote { ident: "a", values: [Text(Text { value: "b", @2:5-2:6 })], @1:1-2:6 })] -html: "" -## fn_two_adjacent "[^a]: b\n[^c]: d" -nodes: [Footnote(Footnote { ident: "a", values: [Text(Text { value: "b", @1:7-1:8 })], @1:1-1:8 }), Footnote(Footnote { ident: "c", values: [Text(Text { value: "d", @2:7-2:8 })], @2:1-2:8 })] -html: "" -## fn_two_separated "[^a]: b\n\n[^c]: d" -nodes: [Footnote(Footnote { ident: "a", values: [Text(Text { value: "b", @1:7-1:8 })], @1:1-2:1 }), Footnote(Footnote { ident: "c", values: [Text(Text { value: "d", @3:7-3:8 })], @3:1-3:8 })] -html: "" -## fn_label_space "[^ a]: b" -nodes: [Definition(Definition { @1:1-1:9, url: Url("b"), title: None, ident: "^ a", label: Some("^ a") })] -html: "" -## fn_label_inner_space "[^a b]: b" -nodes: [Definition(Definition { @1:1-1:10, url: Url("b"), title: None, ident: "^a b", label: Some("^a b") })] -html: "" -## fn_label_empty "[^]: b" -nodes: [Definition(Definition { @1:1-1:7, url: Url("b"), title: None, ident: "^", label: Some("^") })] -html: "" -## fn_indent1 " [^a]: b" -nodes: [Footnote(Footnote { ident: "a", values: [Text(Text { value: "b", @1:8-1:9 })], @1:1-1:9 })] -html: "" -## fn_indent4 " [^a]: b" -nodes: [Code(Code { value: "[^a]: b", lang: None, @1:1-1:12, meta: None, fence: false })] -html: "<pre><code>[^a]: b\n</code></pre>" -## fn_in_quote "> [^a]: b" -nodes: [Blockquote(Blockquote { values: [Footnote(Footnote { ident: "a", values: [Text(Text { value: "b", @1:9-1:10 })], @1:3-1:10 })], @1:1-1:10 })] -html: "<blockquote>\n</blockquote>" -## fn_in_list "- [^a]: b" -nodes: [List(List { values: [Footnote(Footnote { ident: "a", values: [Text(Text { value: "b", @1:9-1:10 })], @1:3-1:10 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-1:10 })] -html: "<ul>\n<li></li>\n</ul>" -## fn_then_quote "[^a]: b\n> q" -nodes: [Footnote(Footnote { ident: "a", values: [Text(Text { value: "b", @1:7-1:8 })], @1:1-1:8 }), Blockquote(Blockquote { values: [Text(Text { value: "q", @2:3-2:4 })], @2:1-2:4 })] -html: "<blockquote>\n<p>q</p>\n</blockquote>" -## fn_then_list "[^a]: b\n- x" -nodes: [Footnote(Footnote { ident: "a", values: [Text(Text { value: "b", @1:7-1:8 })], @1:1-1:8 }), List(List { values: [Text(Text { value: "x", @2:3-2:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @2:3-2:4 })] -html: "<ul>\n<li>x</li>\n</ul>" -## fn_heading "[^a]: # h" -nodes: [Footnote(Footnote { ident: "a", values: [Heading(Heading { depth: 1, values: [Text(Text { value: "h", @1:9-1:10 })], @1:7-1:10 })], @1:1-1:10 })] -html: "" -## fn_list "[^a]: - x\n - y" -nodes: [Footnote(Footnote { ident: "a", values: [List(List { values: [Text(Text { value: "x", @1:9-1:10 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:9-1:10 }), List(List { values: [Text(Text { value: "y", @2:7-2:8 })], index: 1, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @2:7-2:8 })], @1:1-2:8 })] -html: "" -## fn_fence "[^a]: b\n ```\n x\n ```" -nodes: [Footnote(Footnote { ident: "a", values: [Text(Text { value: "b", @1:7-1:8 }), Code(Code { value: "x", lang: None, @2:5-4:8, meta: None, fence: true })], @1:1-4:8 })] -html: "" -## fn_ref_case "[^A]: b\n\nx[^a]" -nodes: [Footnote(Footnote { ident: "a", values: [Text(Text { value: "b", @1:7-1:8 })], @1:1-2:1 }), Text(Text { value: "x", @3:1-3:2 }), FootnoteRef(FootnoteRef { ident: "a", label: Some("a"), @3:2-3:6 })] -html: "<p>x<sup><a href=\"#user-content-fn-a\" id=\"user-content-fnref-a\" data-footnote-ref=\"\" aria-describedby=\"footnote-label\">1</a></sup></p>\n<section data-footnotes=\"\" class=\"footnotes\"><h2 id=\"footnote-label\" class=\"sr-only\">Footnotes</h2>\n<ol>\n<li id=\"user-content-fn-a\">\n<p>b <a href=\"#user-content-fnref-a\" data-footnote-backref=\"\" aria-label=\"Back to content\" class=\"data-footnote-backref\">↩</a></p>\n</li>\n</ol>\n</section>\n" -## fn_no_space "[^a]:b" -nodes: [Footnote(Footnote { ident: "a", values: [Text(Text { value: "b", @1:6-1:7 })], @1:1-1:7 })] -html: "" -## fn_interrupts_paragraph "text\n[^a]: b" -nodes: [Text(Text { value: "text", @1:1-1:5 }), Footnote(Footnote { ident: "a", values: [Text(Text { value: "b", @2:7-2:8 })], @2:1-2:8 })] -html: "<p>text</p>\n" -## fn_then_dashes "[^a]: b\n---" -nodes: [Footnote(Footnote { ident: "a", values: [Text(Text { value: "b", @1:7-1:8 })], @1:1-1:8 }), HorizontalRule(HorizontalRule { marker: Some(Dash), @2:1-2:4 })] -html: "<hr />" -## fn_then_equals "[^a]: b\n===" -nodes: [Footnote(Footnote { ident: "a", values: [Text(Text { value: "b\n===", @1:7-2:4 })], @1:1-2:4 })] -html: "" -## fn_ref_basic "[^a]\n\n[^a]: note" -nodes: [FootnoteRef(FootnoteRef { ident: "a", label: Some("a"), @1:1-1:5 }), Footnote(Footnote { ident: "a", values: [Text(Text { value: "note", @3:7-3:11 })], @3:1-3:11 })] -html: "<p><sup><a href=\"#user-content-fn-a\" id=\"user-content-fnref-a\" data-footnote-ref=\"\" aria-describedby=\"footnote-label\">1</a></sup></p>\n<section data-footnotes=\"\" class=\"footnotes\"><h2 id=\"footnote-label\" class=\"sr-only\">Footnotes</h2>\n<ol>\n<li id=\"user-content-fn-a\">\n<p>note <a href=\"#user-content-fnref-a\" data-footnote-backref=\"\" aria-label=\"Back to content\" class=\"data-footnote-backref\">↩</a></p>\n</li>\n</ol>\n</section>\n" -## fn_ref_no_def "[^b] no def" -nodes: [Text(Text { value: "[^b] no def", @1:1-1:12 })] -html: "<p>[^b] no def</p>" -## fn_ref_inline "x[^a] and [^a]!\n\n[^a]: note" -nodes: [Text(Text { value: "x", @1:1-1:2 }), FootnoteRef(FootnoteRef { ident: "a", label: Some("a"), @1:2-1:6 }), Text(Text { value: " and ", @1:6-1:11 }), FootnoteRef(FootnoteRef { ident: "a", label: Some("a"), @1:11-1:15 }), Text(Text { value: "!", @1:15-1:16 }), Footnote(Footnote { ident: "a", values: [Text(Text { value: "note", @3:7-3:11 })], @3:1-3:11 })] -html: "<p>x<sup><a href=\"#user-content-fn-a\" id=\"user-content-fnref-a\" data-footnote-ref=\"\" aria-describedby=\"footnote-label\">1</a></sup> and <sup><a href=\"#user-content-fn-a\" id=\"user-content-fnref-a-2\" data-footnote-ref=\"\" aria-describedby=\"footnote-label\">1</a></sup>!</p>\n<section data-footnotes=\"\" class=\"footnotes\"><h2 id=\"footnote-label\" class=\"sr-only\">Footnotes</h2>\n<ol>\n<li id=\"user-content-fn-a\">\n<p>note <a href=\"#user-content-fnref-a\" data-footnote-backref=\"\" aria-label=\"Back to content\" class=\"data-footnote-backref\">↩</a> <a href=\"#user-content-fnref-a-2\" data-footnote-backref=\"\" aria-label=\"Back to content\" class=\"data-footnote-backref\">↩<sup>2</sup></a></p>\n</li>\n</ol>\n</section>\n" -## fn_ref_in_link_text "[a[^a]](b)\n\n[^a]: note" -nodes: [Text(Text { value: "[a", @1:1-1:3 }), FootnoteRef(FootnoteRef { ident: "a", label: Some("a"), @1:3-1:7 }), Text(Text { value: "](b)", @1:7-1:11 }), Footnote(Footnote { ident: "a", values: [Text(Text { value: "note", @3:7-3:11 })], @3:1-3:11 })] -html: "<p>[a<sup><a href=\"#user-content-fn-a\" id=\"user-content-fnref-a\" data-footnote-ref=\"\" aria-describedby=\"footnote-label\">1</a></sup>](b)</p>\n<section data-footnotes=\"\" class=\"footnotes\"><h2 id=\"footnote-label\" class=\"sr-only\">Footnotes</h2>\n<ol>\n<li id=\"user-content-fn-a\">\n<p>note <a href=\"#user-content-fnref-a\" data-footnote-backref=\"\" aria-label=\"Back to content\" class=\"data-footnote-backref\">↩</a></p>\n</li>\n</ol>\n</section>\n" -## html_div "<div>\nx\n</div>\n\ny" -nodes: [Html(Html { value: "<div>\nx\n</div>", @1:1-3:7 }), Text(Text { value: "y", @5:1-5:2 })] -html: "<div>\nx\n</div>\n<p>y</p>" -## html_indented " <div>\n x\n" -nodes: [Html(Html { value: " <div>\n x", @1:1-2:4 })] -html: " <div>\n x\n" -## html_oneline "<div>x</div>" -nodes: [Html(Html { value: "<div>x</div>", @1:1-1:13 })] -html: "<div>x</div>" -## html_script "<script>\nx\n\ny\n</script>\nz" -nodes: [Html(Html { value: "<script>\nx\n\ny\n</script>", @1:1-5:10 }), Text(Text { value: "z", @6:1-6:2 })] -html: "&lt;script>\nx\n\ny\n&lt;/script>\n<p>z</p>" -## html_pre "<pre>\na\n\nb</pre>\nc" -nodes: [Html(Html { value: "<pre>\na\n\nb</pre>", @1:1-4:8 }), Text(Text { value: "c", @5:1-5:2 })] -html: "<pre>\na\n\nb</pre>\n<p>c</p>" -## html_style_inline_end "<style>x</style> y\nz" -nodes: [Html(Html { value: "<style>x</style> y", @1:1-1:19 }), Text(Text { value: "z", @2:1-2:2 })] -html: "&lt;style>x&lt;/style> y\n<p>z</p>" -## html_comment "<!-- c\n\n d -->\nx" -nodes: [Html(Html { value: "<!-- c\n\n d -->", @1:1-3:7 }), Text(Text { value: "x", @4:1-4:2 })] -html: "<!-- c\n\n d -->\n<p>x</p>" -## html_instruction "<?php\nx ?>\ny" -nodes: [Html(Html { value: "<?php\nx ?>", @1:1-2:5 }), Text(Text { value: "y", @3:1-3:2 })] -html: "<?php\nx ?>\n<p>y</p>" -## html_declaration "<!DOCTYPE html>\nx" -nodes: [Html(Html { value: "<!DOCTYPE html>", @1:1-1:16 }), Text(Text { value: "x", @2:1-2:2 })] -html: "<!DOCTYPE html>\n<p>x</p>" -## html_cdata "<![CDATA[\n\nx]]>\ny" -nodes: [Html(Html { value: "<![CDATA[\n\nx]]>", @1:1-3:5 }), Text(Text { value: "y", @4:1-4:2 })] -html: "<![CDATA[\n\nx]]>\n<p>y</p>" -## html_complete_tag "<a href=\"x\">\ny" -nodes: [Html(Html { value: "<a href=\"x\">\ny", @1:1-2:2 })] -html: "<a href=\"x\">\ny" -## html_tag_then_text "<a href=\"x\">y" -nodes: [Html(Html { value: "<a href=\"x\">", @1:1-1:13 }), Text(Text { value: "y", @1:13-1:14 })] -html: "<p><a href=\"x\">y</p>" -## html_interrupt_basic "text\n<div>" -nodes: [Text(Text { value: "text", @1:1-1:5 }), Html(Html { value: "<div>", @2:1-2:6 })] -html: "<p>text</p>\n<div>" -## html_no_interrupt_complete "text\n<a href=\"x\">" -nodes: [Text(Text { value: "text\n", @1:1-2:1 }), Html(Html { value: "<a href=\"x\">", @2:1-2:13 })] -html: "<p>text\n<a href=\"x\"></p>" -## html_custom_element "<x-y>\nz" -nodes: [Html(Html { value: "<x-y>\nz", @1:1-2:2 })] -html: "<x-y>\nz" -## html_closing "</div>\nz" -nodes: [Html(Html { value: "</div>\nz", @1:1-2:2 })] -html: "</div>\nz" -## html_multiline_open "<div\nclass=\"a\">\nz" -nodes: [Html(Html { value: "<div\nclass=\"a\">\nz", @1:1-3:2 })] -html: "<div\nclass=\"a\">\nz" -## html_in_quote "> <div>\n> x" -nodes: [Blockquote(Blockquote { values: [Html(Html { value: "<div>\nx", @1:3-2:4 })], @1:1-2:4 })] -html: "<blockquote>\n<div>\nx\n</blockquote>" -## html_in_list "- <div>\n x" -nodes: [List(List { values: [Html(Html { value: "<div>\nx", @1:3-2:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-2:4 })] -html: "<ul>\n<li>\n<div>\nx\n</li>\n</ul>" -## html_uppercase "<DIV>\nx" -nodes: [Html(Html { value: "<DIV>\nx", @1:1-2:2 })] -html: "<DIV>\nx" -## html_self_closing "<div/>\nx" -nodes: [Html(Html { value: "<div/>\nx", @1:1-2:2 })] -html: "<div/>\nx" -## html_ins "<ins>\nx" -nodes: [Html(Html { value: "<ins>\nx", @1:1-2:2 })] -html: "<ins>\nx" -## html_br "<br>\nx" -nodes: [Html(Html { value: "<br>\nx", @1:1-2:2 })] -html: "<br>\nx" -## html_textarea "<textarea>\nx\n\ny</textarea>" -nodes: [Html(Html { value: "<textarea>\nx\n\ny</textarea>", @1:1-4:13 })] -html: "&lt;textarea>\nx\n\ny&lt;/textarea>" -## html_table_two_blocks "<table>\n<tr>\n\n<td>" -nodes: [Html(Html { value: "<table>\n<tr>", @1:1-2:5 }), Html(Html { value: "<td>", @4:1-4:5 })] -html: "<table>\n<tr>\n<td>" -## html_img_self_close "<img src=\"x\" />\nz" -nodes: [Html(Html { value: "<img src=\"x\" />\nz", @1:1-2:2 })] -html: "<img src=\"x\" />\nz" -## html_img_then_text "<img src=\"x\" /> y" -nodes: [Html(Html { value: "<img src=\"x\" />", @1:1-1:16 }), Text(Text { value: " y", @1:16-1:18 })] -html: "<p><img src=\"x\" /> y</p>" -## html_multiline_tag_inline "<a\nb>\nz" -nodes: [Html(Html { value: "<a\nb>", @1:1-2:3 }), Text(Text { value: "\nz", @2:3-3:2 })] -html: "<p><a\nb>\nz</p>" -## html_indent4 " <div>" -nodes: [Code(Code { value: "<div>", lang: None, @1:1-1:10, meta: None, fence: false })] -html: "<pre><code>&lt;div&gt;\n</code></pre>" -## html_comment_then_text "<!--x-->y\nz" -nodes: [Html(Html { value: "<!--x-->y", @1:1-1:10 }), Text(Text { value: "z", @2:1-2:2 })] -html: "<!--x-->y\n<p>z</p>" -## html_then_code_line "<div>\n code" -nodes: [Html(Html { value: "<div>\n code", @1:1-2:9 })] -html: "<div>\n code" -## html_comment_short "<!-->\nx" -nodes: [Html(Html { value: "<!-->", @1:1-1:6 }), Text(Text { value: "x", @2:1-2:2 })] -html: "<!-->\n<p>x</p>" -## html_instruction_short "<?>\nx" -nodes: [Html(Html { value: "<?>", @1:1-1:4 }), Text(Text { value: "x", @2:1-2:2 })] -html: "<?>\n<p>x</p>" -## html_lazy_quote "> <div>\nx" -nodes: [Blockquote(Blockquote { values: [Html(Html { value: "<div>", @1:3-1:8 })], @1:1-1:8 }), Text(Text { value: "x", @2:1-2:2 })] -html: "<blockquote>\n<div>\n</blockquote>\n<p>x</p>" -## html_attr_forms "<a b c=d e='f' g=\"h\">\nx" -nodes: [Html(Html { value: "<a b c=d e='f' g=\"h\">\nx", @1:1-2:2 })] -html: "<a b c=d e='f' g=\"h\">\nx" -## html_bad_attr "<a b=>\nx" -nodes: [Text(Text { value: "<a b=>\nx", @1:1-2:2 })] -html: "<p>&lt;a b=&gt;\nx</p>" -## fm_yaml "---\na: b\n---\ntext" -nodes: [Yaml(Yaml { value: "a: b", @1:1-3:4 }), Text(Text { value: "text", @4:1-4:5 })] -html: "<p>text</p>" -## fm_yaml_only "---\na: b\n---" -nodes: [Yaml(Yaml { value: "a: b", @1:1-3:4 })] -html: "" -## fm_empty "---\n---" -nodes: [Yaml(Yaml { value: "", @1:1-2:4 })] -html: "" -## fm_unclosed "---\na\n" -nodes: [HorizontalRule(HorizontalRule { marker: Some(Dash), @1:1-1:4 }), Text(Text { value: "a", @2:1-2:2 })] -html: "<hr />\n<p>a</p>\n" -## fm_blank_content "---\n\n---\nx" -nodes: [Yaml(Yaml { value: "", @1:1-3:4 }), Text(Text { value: "x", @4:1-4:2 })] -html: "<p>x</p>" -## fm_toml "+++\na = 1\n+++\nx" -nodes: [Toml(Toml { value: "a = 1", @1:1-3:4 }), Text(Text { value: "x", @4:1-4:2 })] -html: "<p>x</p>" -## fm_indented " ---\na\n---" -nodes: [HorizontalRule(HorizontalRule { marker: Some(Dash), @1:1-1:5 }), Heading(Heading { depth: 2, values: [Text(Text { value: "a", @2:1-2:2 })], @2:1-3:4 })] -html: "<hr />\n<h2>a</h2>" -## fm_close_trailing_space "---\na\n--- \nx" -nodes: [Yaml(Yaml { value: "a", @1:1-3:5 }), Text(Text { value: "x", @4:1-4:2 })] -html: "<p>x</p>" -## fm_close_longer "---\na\n----\nx" -nodes: [HorizontalRule(HorizontalRule { marker: Some(Dash), @1:1-1:4 }), Heading(Heading { depth: 2, values: [Text(Text { value: "a", @2:1-2:2 })], @2:1-3:5 }), Text(Text { value: "x", @4:1-4:2 })] -html: "<hr />\n<h2>a</h2>\n<p>x</p>" -## fm_open_trailing_space "--- \na\n---" -nodes: [Yaml(Yaml { value: "a", @1:1-3:4 })] -html: "" -## fm_open_junk "---x\na\n---" -nodes: [Heading(Heading { depth: 2, values: [Text(Text { value: "---x\na", @1:1-2:2 })], @1:1-3:4 })] -html: "<h2>---x\na</h2>" -## fm_not_first "text\n---\na\n---" -nodes: [Heading(Heading { depth: 2, values: [Text(Text { value: "text", @1:1-1:5 })], @1:1-2:4 }), Heading(Heading { depth: 2, values: [Text(Text { value: "a", @3:1-3:2 })], @3:1-4:4 })] -html: "<h2>text</h2>\n<h2>a</h2>" -## fm_blank_before "\n---\na\n---" -nodes: [HorizontalRule(HorizontalRule { marker: Some(Dash), @2:1-2:4 }), Heading(Heading { depth: 2, values: [Text(Text { value: "a", @3:1-3:2 })], @3:1-4:4 })] -html: "<hr />\n<h2>a</h2>" -## fm_multiline_content "---\na: 1\n\nb: 2\n---\n# h" -nodes: [Yaml(Yaml { value: "a: 1\n\nb: 2", @1:1-5:4 }), Heading(Heading { depth: 1, values: [Text(Text { value: "h", @6:3-6:4 })], @6:1-6:4 })] -html: "<h1>h</h1>" -## fm_hr_after "---\na\n---\n\n---" -nodes: [Yaml(Yaml { value: "a", @1:1-3:4 }), HorizontalRule(HorizontalRule { marker: Some(Dash), @5:1-5:4 })] -html: "<hr />" -## math_basic "$$\na\n$$" -nodes: [Math(Math { value: "a", @1:1-3:3 })] -html: "<pre><code class=\"language-math math-display\">a\n</code></pre>" -## math_then_text "$$\na\n$$\nx" -nodes: [Math(Math { value: "a", @1:1-3:3 }), Text(Text { value: "x", @4:1-4:2 })] -html: "<pre><code class=\"language-math math-display\">a\n</code></pre>\n<p>x</p>" -## math_meta "$$ meta\na\n$$" -nodes: [Math(Math { value: "a", @1:1-3:3 })] -html: "<pre><code class=\"language-math math-display\">a\n</code></pre>" -## math_unclosed "$$\na" -nodes: [Math(Math { value: "a", @1:1-2:2 })] -html: "<pre><code class=\"language-math math-display\">a\n</code></pre>\n" -## math_longer_open "$$$\na\n$$" -nodes: [Math(Math { value: "a\n$$", @1:1-3:3 })] -html: "<pre><code class=\"language-math math-display\">a\n$$\n</code></pre>\n" -## math_longer_close "$$\na\n$$$$" -nodes: [Math(Math { value: "a", @1:1-3:5 })] -html: "<pre><code class=\"language-math math-display\">a\n</code></pre>" -## math_indented " $$\n a\n b\n $$" -nodes: [Math(Math { value: "a\n b", @1:3-4:5 })] -html: "<pre><code class=\"language-math math-display\">a\n b\n</code></pre>" -## math_empty "$$\n$$" -nodes: [Math(Math { value: "", @1:1-2:3 })] -html: "<pre><code class=\"language-math math-display\"></code></pre>" -## math_inline_not_block "$$a$$" -nodes: [MathInline(MathInline { value: "a", @1:1-1:6 })] -html: "<p><code class=\"language-math math-inline\">a</code></p>" -## math_info_no_close "$$a\n$$" -nodes: [Math(Math { value: "", @1:1-2:3 })] -html: "<pre><code class=\"language-math math-display\"></code></pre>" -## math_in_quote "> $$\n> a\n> $$" -nodes: [Blockquote(Blockquote { values: [Math(Math { value: "a", @1:3-3:5 })], @1:1-3:5 })] -html: "<blockquote>\n<pre><code class=\"language-math math-display\">a\n</code></pre>\n</blockquote>" -## math_in_list "- $$\n a\n $$" -nodes: [List(List { values: [Math(Math { value: "a", @1:3-3:5 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-3:5 })] -html: "<ul>\n<li>\n<pre><code class=\"language-math math-display\">a\n</code></pre>\n</li>\n</ul>" -## math_close_junk "$$\na\n$$ x" -nodes: [Math(Math { value: "a\n$$ x", @1:1-3:5 })] -html: "<pre><code class=\"language-math math-display\">a\n$$ x\n</code></pre>\n" -## math_interrupts_paragraph "text\n$$\na\n$$" -nodes: [Text(Text { value: "text", @1:1-1:5 }), Math(Math { value: "a", @2:1-4:3 })] -html: "<p>text</p>\n<pre><code class=\"language-math math-display\">a\n</code></pre>" -## math_blank_inside "$$\n\na\n$$" -nodes: [Math(Math { value: "\na", @1:1-4:3 })] -html: "<pre><code class=\"language-math math-display\">\na\n</code></pre>" -## math_single_dollar_line "$ $\nx" -nodes: [MathInline(MathInline { value: " ", @1:1-1:4 }), Text(Text { value: "\nx", @1:4-2:2 })] -html: "<p><code class=\"language-math math-inline\"> </code>\nx</p>" -## crlf_frontmatter "---\r\na\r\n---\r\nx" -nodes: [Yaml(Yaml { value: "a", @1:1-3:4 }), Text(Text { value: "x", @4:1-4:2 })] -html: "<p>x</p>" -## break_end_indented "a \n b" -nodes: [Text(Text { value: "a", @1:1-1:2 }), Break(Break { @1:2-2:1 }), Text(Text { value: "b", @2:3-2:4 })] -html: "<p>a<br />\nb</p>" -## text_end_indented_continuation "*a*\n <x>" -nodes: [Emphasis(Emphasis { values: [Text(Text { value: "a", @1:2-1:3 })], @1:1-1:4 }), Text(Text { value: "\n", @1:4-2:1 }), Html(Html { value: "<x>", @2:3-2:6 })] -html: "<p><em>a</em>\n<x></p>" -## soft_end_before_node "a\n `b`" -nodes: [Text(Text { value: "a\n", @1:1-2:1 }), CodeInline(CodeInline { value: "b", @2:3-2:6 })] -html: "<p>a\n<code>b</code></p>" -## break_in_quote "> a \n> b" -nodes: [Blockquote(Blockquote { values: [Text(Text { value: "a", @1:3-1:4 }), Break(Break { @1:4-2:1 }), Text(Text { value: "b", @2:3-2:4 })], @1:1-2:4 })] -html: "<blockquote>\n<p>a<br />\nb</p>\n</blockquote>" -## text_eol_in_footnote "[^a]: x \n [a]" -nodes: [Footnote(Footnote { ident: "a", values: [Text(Text { value: "x\n[a]", @1:7-2:6 })], @1:1-2:6 })] -html: "" -## math_whitespace_content " $$\n \n $$" -nodes: [Math(Math { value: "", @1:3-3:5 })] -html: "<pre><code class=\"language-math math-display\">\n</code></pre>" -## math_whitespace_content2 " $$\n \n $$" -nodes: [Math(Math { value: "", @1:3-3:5 })] -html: "<pre><code class=\"language-math math-display\">\n</code></pre>" -## table_indented_no_pipe " a|b\n|-|:-:|" -nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:3-1:4 })]:0, row: 0, @1:1-1:4 }), TableCell(TableCell { values: [Text(Text { value: "b", @1:5-1:6 })]:1, row: 0, @1:4-1:6 }), TableAlign(TableAlign { align: [None, Center], @2:1-2:1 })] -html: "<table>\n<thead>\n<tr>\n<th>a</th>\n<th align=\"center\">b</th>\n</tr>\n</thead>\n</table>" -## tab_nested_list "- a\n\t- b\n\t\t- c\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-1:4 }), List(List { values: [Text(Text { value: "b", @2:7-2:8 })], index: 0, level: 1, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @2:7-2:8 }), List(List { values: [Text(Text { value: "c", @3:11-3:12 })], index: 0, level: 2, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @3:11-3:12 })] -html: "<ul>\n<li>a\n<ul>\n<li>b\n<ul>\n<li>c</li>\n</ul>\n</li>\n</ul>\n</li>\n</ul>\n" -## tab_nested_ordered "1. a\n\t1. b\n\t\t1. c\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:4-1:5 })], index: 0, level: 0, ordered: true, checked: None, spread: false, start: Some(1), marker: Some(Period), @1:4-1:5 }), List(List { values: [Text(Text { value: "b", @2:8-2:9 })], index: 0, level: 1, ordered: true, checked: None, spread: false, start: Some(1), marker: Some(Period), @2:8-2:9 }), List(List { values: [Text(Text { value: "c", @3:12-3:13 })], index: 0, level: 2, ordered: true, checked: None, spread: false, start: Some(1), marker: Some(Period), @3:12-3:13 })] -html: "<ol>\n<li>a\n<ol>\n<li>b\n<ol>\n<li>c</li>\n</ol>\n</li>\n</ol>\n</li>\n</ol>\n" -## tab_list_code "- a\n\n\t\tcode\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 }), Code(Code { value: " code", lang: None, @3:3-3:13, meta: None, fence: false })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-3:13 })] -html: "<ul>\n<li>\n<p>a</p>\n<pre><code> code\n</code></pre>\n</li>\n</ul>\n" -## tab_in_text "a\tb\t*c*\n" -nodes: [Text(Text { value: "a\tb\t", @1:1-1:9 }), Emphasis(Emphasis { values: [Text(Text { value: "c", @1:10-1:11 })], @1:9-1:12 })] -html: "<p>a\tb\t<em>c</em></p>\n" -## tab_code_block "\tcode\n\t\tmore\n" -nodes: [Code(Code { value: "code\n\tmore", lang: None, @1:1-2:13, meta: None, fence: false })] -html: "<pre><code>code\n\tmore\n</code></pre>\n" -## tab_after_marker "-\ta\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:5-1:6 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:5-1:6 })] -html: "<ul>\n<li>a</li>\n</ul>\n" -## atx_leading_sequences "# # a" -nodes: [Heading(Heading { depth: 1, values: [Text(Text { value: "# a", @1:3-1:6 })], @1:1-1:6 })] -html: "<h1># a</h1>" -## atx_leading_sequences2 "## # a" -nodes: [Heading(Heading { depth: 2, values: [Text(Text { value: "# a", @1:4-1:7 })], @1:1-1:7 })] -html: "<h2># a</h2>" -## atx_only_sequences "# # #" -nodes: [Heading(Heading { depth: 1, values: [Text(Text { value: "#", @1:3-1:4 })], @1:1-1:6 })] -html: "<h1>#</h1>" -## atx_no_space_sequence "# #a" -nodes: [Heading(Heading { depth: 1, values: [Text(Text { value: "#a", @1:3-1:5 })], @1:1-1:5 })] -html: "<h1>#a</h1>" -## atx_double_sequence "# ## a" -nodes: [Heading(Heading { depth: 1, values: [Text(Text { value: "## a", @1:3-1:7 })], @1:1-1:7 })] -html: "<h1>## a</h1>" -## atx_tab_sequence "#\t# a" -nodes: [Heading(Heading { depth: 1, values: [Text(Text { value: "# a", @1:5-1:8 })], @1:1-1:8 })] -html: "<h1># a</h1>" -## atx_closing_and_leading "# # a #" -nodes: [Heading(Heading { depth: 1, values: [Text(Text { value: "# a", @1:3-1:6 })], @1:1-1:8 })] -html: "<h1># a</h1>" -## atx_inner_hash "# a # b" -nodes: [Heading(Heading { depth: 1, values: [Text(Text { value: "a # b", @1:3-1:8 })], @1:1-1:8 })] -html: "<h1>a # b</h1>" -## quote_footnote_trailing_space "- a\n > [^a]: x\n " -nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 }), Blockquote(Blockquote { values: [Footnote(Footnote { ident: "a", values: [Text(Text { value: "x", @2:11-2:12 })], @2:5-3:3 })], @2:3-3:3 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-3:3 })] -html: "<ul>\n<li>a\n<blockquote>\n</blockquote>\n</li>\n</ul>" -## indented_code_trailing_blank " code\n\n\n" -nodes: [Code(Code { value: "code", lang: None, @1:1-1:9, meta: None, fence: false })] -html: "<pre><code>code\n</code></pre>\n" -## bom_heading "\u{feff}# a\n" -nodes: [Heading(Heading { depth: 1, values: [Text(Text { value: "a", @1:6-1:7 })], @1:4-1:7 })] -html: "<h1>a</h1>\n" -## bom_frontmatter "\u{feff}---\na: b\n---\n" -nodes: [Yaml(Yaml { value: "a: b", @1:4-3:4 })] -html: "" -## bom_text "\u{feff}a\n" -nodes: [Text(Text { value: "a", @1:4-1:5 })] -html: "<p>a</p>\n" -## www_multibyte_after_ww "wwß x" -nodes: [Text(Text { value: "wwß x", @1:1-1:7 })] -html: "<p>wwß x</p>" -## www_combining "ww\u{301}.a.com" -nodes: [Text(Text { value: "ww\u{301}.a.com", @1:1-1:11 })] -html: "<p>ww\u{301}.a.com</p>" -## www_cjk "www.あ.com/パス x" -nodes: [Link(Link { url: Url("http://www.あ.com/パス"), title: None, values: [Text(Text { value: "www.あ.com/パス", @1:1-1:19 })], @1:1-1:19 }), Text(Text { value: " x", @1:19-1:21 })] -html: "<p><a href=\"http://www.%E3%81%82.com/%E3%83%91%E3%82%B9\">www.あ.com/パス</a> x</p>" -## http_cjk "http://あ.jp/パス ok" -nodes: [Link(Link { url: Url("http://あ.jp/パス"), title: None, values: [Text(Text { value: "http://あ.jp/パス", @1:1-1:21 })], @1:1-1:21 }), Text(Text { value: " ok", @1:21-1:24 })] -html: "<p><a href=\"http://%E3%81%82.jp/%E3%83%91%E3%82%B9\">http://あ.jp/パス</a> ok</p>" -## email_cjk_domain "a@あ.com" -nodes: [Text(Text { value: "a@あ.com", @1:1-1:10 })] -html: "<p>a@あ.com</p>" -## email_cjk_local "あ@b.com" -nodes: [Text(Text { value: "あ@b.com", @1:1-1:10 })] -html: "<p>あ@b.com</p>" -## html_inline_continuation "<img a\n b=\"c\">" -nodes: [Html(Html { value: "<img a\nb=\"c\">", @1:1-2:12 })] -html: "<p><img a\nb=\"c\"></p>" -## html_inline_continuation_tab "<img a\n\t\tb=\"c\">" -nodes: [Html(Html { value: "<img a\nb=\"c\">", @1:1-2:15 })] -html: "<p><img a\nb=\"c\"></p>" -## table_trailing_space "| a | b | \n|-|-| \n| c | d | \n" -nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:3-1:4 })]:0, row: 0, @1:1-1:5 }), TableCell(TableCell { values: [Text(Text { value: "b", @1:7-1:8 })]:1, row: 0, @1:5-1:12 }), TableAlign(TableAlign { align: [None, None], @2:1-2:1 }), TableCell(TableCell { values: [Text(Text { value: "c", @3:3-3:4 })]:0, row: 1, @3:1-3:5 }), TableCell(TableCell { values: [Text(Text { value: "d", @3:7-3:8 })]:1, row: 1, @3:5-3:12 })] -html: "<table>\n<thead>\n<tr>\n<th>a</th>\n<th>b</th>\n</tr>\n</thead>\n<tbody>\n<tr>\n<td>c</td>\n<td>d</td>\n</tr>\n</tbody>\n</table>\n" -## table_no_pipes_trailing_space "a | b\n--|--\nd | e \n" -nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:1-1:2 })]:0, row: 0, @1:1-1:3 }), TableCell(TableCell { values: [Text(Text { value: "b", @1:5-1:6 })]:1, row: 0, @1:3-1:6 }), TableAlign(TableAlign { align: [None, None], @2:1-2:1 }), TableCell(TableCell { values: [Text(Text { value: "d", @3:1-3:2 })]:0, row: 1, @3:1-3:3 }), TableCell(TableCell { values: [Text(Text { value: "e", @3:5-3:6 })]:1, row: 1, @3:3-3:7 })] -html: "<table>\n<thead>\n<tr>\n<th>a</th>\n<th>b</th>\n</tr>\n</thead>\n<tbody>\n<tr>\n<td>d</td>\n<td>e</td>\n</tr>\n</tbody>\n</table>\n" -## table_trailing_tab "| a | b |\t\n|-|-|\n| c | d | \n" -nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:3-1:4 })]:0, row: 0, @1:1-1:5 }), TableCell(TableCell { values: [Text(Text { value: "b", @1:7-1:8 })]:1, row: 0, @1:5-1:13 }), TableAlign(TableAlign { align: [None, None], @2:1-2:1 }), TableCell(TableCell { values: [Text(Text { value: "c", @3:3-3:4 })]:0, row: 1, @3:1-3:5 }), TableCell(TableCell { values: [Text(Text { value: "d", @3:7-3:8 })]:1, row: 1, @3:5-3:13 })] -html: "<table>\n<thead>\n<tr>\n<th>a</th>\n<th>b</th>\n</tr>\n</thead>\n<tbody>\n<tr>\n<td>c</td>\n<td>d</td>\n</tr>\n</tbody>\n</table>\n" -## table_cjk "| あ | 🎉 |\n|:-|-:|\n| é | ß |\n" -nodes: [TableCell(TableCell { values: [Text(Text { value: "あ", @1:3-1:6 })]:0, row: 0, @1:1-1:7 }), TableCell(TableCell { values: [Text(Text { value: "🎉", @1:9-1:13 })]:1, row: 0, @1:7-1:15 }), TableAlign(TableAlign { align: [Left, Right], @2:1-2:1 }), TableCell(TableCell { values: [Text(Text { value: "é", @3:3-3:5 })]:0, row: 1, @3:1-3:6 }), TableCell(TableCell { values: [Text(Text { value: "ß", @3:8-3:10 })]:1, row: 1, @3:6-3:12 })] -html: "<table>\n<thead>\n<tr>\n<th align=\"left\">あ</th>\n<th align=\"right\">🎉</th>\n</tr>\n</thead>\n<tbody>\n<tr>\n<td align=\"left\">é</td>\n<td align=\"right\">ß</td>\n</tr>\n</tbody>\n</table>\n" -## table_single_column_colon "a\n:-\nb\n" -nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:1-1:2 })]:0, row: 0, @1:1-1:2 }), TableAlign(TableAlign { align: [Left], @2:1-2:1 }), TableCell(TableCell { values: [Text(Text { value: "b", @3:1-3:2 })]:0, row: 1, @3:1-3:2 })] -html: "<table>\n<thead>\n<tr>\n<th align=\"left\">a</th>\n</tr>\n</thead>\n<tbody>\n<tr>\n<td align=\"left\">b</td>\n</tr>\n</tbody>\n</table>\n" -## table_single_column_right "a\n---:\nb\nc\n" -nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:1-1:2 })]:0, row: 0, @1:1-1:2 }), TableAlign(TableAlign { align: [Right], @2:1-2:1 }), TableCell(TableCell { values: [Text(Text { value: "b", @3:1-3:2 })]:0, row: 1, @3:1-3:2 }), TableCell(TableCell { values: [Text(Text { value: "c", @4:1-4:2 })]:0, row: 2, @4:1-4:2 })] -html: "<table>\n<thead>\n<tr>\n<th align=\"right\">a</th>\n</tr>\n</thead>\n<tbody>\n<tr>\n<td align=\"right\">b</td>\n</tr>\n<tr>\n<td align=\"right\">c</td>\n</tr>\n</tbody>\n</table>\n" -## table_double_colon_not_delimiter "a\n::-\n" -nodes: [Text(Text { value: "a\n::-", @1:1-2:4 })] -html: "<p>a\n::-</p>\n" -## table_body_empty_marker "a|b\n|-|-|\n*\n" -nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:1-1:2 })]:0, row: 0, @1:1-1:2 }), TableCell(TableCell { values: [Text(Text { value: "b", @1:3-1:4 })]:1, row: 0, @1:2-1:4 }), TableAlign(TableAlign { align: [None, None], @2:1-2:1 }), List(List { values: [], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Star), @3:1-3:2 })] -html: "<table>\n<thead>\n<tr>\n<th>a</th>\n<th>b</th>\n</tr>\n</thead>\n</table>\n<ul>\n<li></li>\n</ul>\n" -## table_body_ordered_marker "a|b\n|-|-|\n2. x\n" -nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:1-1:2 })]:0, row: 0, @1:1-1:2 }), TableCell(TableCell { values: [Text(Text { value: "b", @1:3-1:4 })]:1, row: 0, @1:2-1:4 }), TableAlign(TableAlign { align: [None, None], @2:1-2:1 }), List(List { values: [Text(Text { value: "x", @3:4-3:5 })], index: 0, level: 0, ordered: true, checked: None, spread: false, start: Some(2), marker: Some(Period), @3:4-3:5 })] -html: "<table>\n<thead>\n<tr>\n<th>a</th>\n<th>b</th>\n</tr>\n</thead>\n</table>\n<ol start=\"2\">\n<li>x</li>\n</ol>\n" -## table_body_empty_ordered "a|b\n|-|-|\n1.\n" -nodes: [TableCell(TableCell { values: [Text(Text { value: "a", @1:1-1:2 })]:0, row: 0, @1:1-1:2 }), TableCell(TableCell { values: [Text(Text { value: "b", @1:3-1:4 })]:1, row: 0, @1:2-1:4 }), TableAlign(TableAlign { align: [None, None], @2:1-2:1 }), List(List { values: [], index: 0, level: 0, ordered: true, checked: None, spread: false, start: Some(1), marker: Some(Period), @3:1-3:3 })] -html: "<table>\n<thead>\n<tr>\n<th>a</th>\n<th>b</th>\n</tr>\n</thead>\n</table>\n<ol>\n<li></li>\n</ol>\n" -## paragraph_after_partial_tab "- a\n\n\t[P](h) x\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 }), Link(Link { url: Url("h"), title: None, values: [Text(Text { value: "P", @3:6-3:7 })], @3:5-3:11 }), Text(Text { value: " x", @3:11-3:13 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-3:13 })] -html: "<ul>\n<li>\n<p>a</p>\n<p><a href=\"h\">P</a> x</p>\n</li>\n</ul>\n" -## fence_after_partial_tab "- a\n\n\t```ts\n\t\tx\n\t```\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 }), Code(Code { value: "\tx", lang: Some("ts"), @3:5-5:8, meta: None, fence: true })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-5:8 })] -html: "<ul>\n<li>\n<p>a</p>\n<pre><code class=\"language-ts\">\tx\n</code></pre>\n</li>\n</ul>\n" -## math_after_partial_tab "- a\n\n\t$$\n\tm\n\t$$\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 }), Math(Math { value: "m", @3:5-5:7 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-5:7 })] -html: "<ul>\n<li>\n<p>a</p>\n<pre><code class=\"language-math math-display\">m\n</code></pre>\n</li>\n</ul>\n" -## quote_after_partial_tab "- a\n\n\t> q\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 }), Blockquote(Blockquote { values: [Text(Text { value: "q", @3:7-3:8 })], @3:3-3:8 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-3:8 })] -html: "<ul>\n<li>\n<p>a</p>\n<blockquote>\n<p>q</p>\n</blockquote>\n</li>\n</ul>\n" -## heading_after_partial_tab "- a\n\n\t# h\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:3-1:4 }), Heading(Heading { depth: 1, values: [Text(Text { value: "h", @3:7-3:8 })], @3:3-3:8 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-3:8 })] -html: "<ul>\n<li>\n<p>a</p>\n<h1>h</h1>\n</li>\n</ul>\n" -## control_character_references "&#1;&#x7f;&#x85;&#xB;&#xC;&#x9f;&#xa0;&#xd800;&#x110000;&#0;" -nodes: [Text(Text { value: "����\u{c}�\u{a0}���", @1:1-1:61 })] -html: "<p>����\u{c}�\u{a0}���</p>" -## footnote_escaped_bracket "[^a\\]: x" -nodes: [Text(Text { value: "[^a]: x", @1:1-1:9 })] -html: "<p>[^a]: x</p>" -## footnote_escaped_backslash "[^a\\\\]: x" -nodes: [Footnote(Footnote { ident: "a\\\\", values: [Text(Text { value: "x", @1:9-1:10 })], @1:1-1:10 })] -html: "" -## footnote_escaped_open "[^a\\[]: x" -nodes: [Footnote(Footnote { ident: "a\\[", values: [Text(Text { value: "x", @1:9-1:10 })], @1:1-1:10 })] -html: "" -## footnote_unescaped_open "[^a[]: x" -nodes: [Text(Text { value: "[^a[]: x", @1:1-1:9 })] -html: "<p>[^a[]: x</p>" -## footnote_empty_then_blank "[^a]: \n\n" -nodes: [Footnote(Footnote { ident: "a", values: [], @1:1-2:1 })] -html: "" -## footnote_empty_then_spaces "[^a]: \n \n" -nodes: [Footnote(Footnote { ident: "a", values: [], @1:1-2:5 })] -html: "" -## footnote_empty_then_text "[^a]: \n\nb" -nodes: [Footnote(Footnote { ident: "a", values: [], @1:1-2:1 }), Text(Text { value: "b", @3:1-3:2 })] -html: "<p>b</p>" -## definition_nul_destination "[a]: \0\n\n[a]\n" -nodes: [Definition(Definition { @1:1-1:7, url: Url("\0"), title: None, ident: "a", label: Some("a") }), LinkRef(LinkRef { ident: "a", label: Some("a"), values: [Text(Text { value: "a", @3:2-3:3 })], @3:1-3:4 })] -html: "<p><a href=\"%EF%BF%BD\">a</a></p>\n" -## link_nul_destination "[a](x\0y)" -nodes: [Link(Link { url: Url("x\0y"), title: None, values: [Text(Text { value: "a", @1:2-1:3 })], @1:1-1:9 })] -html: "<p><a href=\"x%EF%BF%BDy\">a</a></p>" -## definition_then_empty_marker "[a]: b\n-" -nodes: [Definition(Definition { @1:1-1:7, url: Url("b"), title: None, ident: "a", label: Some("a") }), Text(Text { value: "-", @2:1-2:2 })] -html: "<p>-</p>" -## definition_then_item "[a]: b\n- x" -nodes: [Definition(Definition { @1:1-1:7, url: Url("b"), title: None, ident: "a", label: Some("a") }), List(List { values: [Text(Text { value: "x", @2:3-2:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @2:3-2:4 })] -html: "<ul>\n<li>x</li>\n</ul>" -## definition_then_ordered "[a]: b\n1." -nodes: [Definition(Definition { @1:1-1:7, url: Url("b"), title: None, ident: "a", label: Some("a") }), Text(Text { value: "1.", @2:1-2:3 })] -html: "<p>1.</p>" -## emphasis_cjk "*あ*。*い*" -nodes: [Emphasis(Emphasis { values: [Text(Text { value: "あ", @1:2-1:5 })], @1:1-1:6 }), Text(Text { value: "。", @1:6-1:9 }), Emphasis(Emphasis { values: [Text(Text { value: "い", @1:10-1:13 })], @1:9-1:14 })] -html: "<p><em>あ</em>。<em>い</em></p>" -## strong_cjk "**日本語**です" -nodes: [Strong(Strong { values: [Text(Text { value: "日本語", @1:3-1:12 })], @1:1-1:14 }), Text(Text { value: "です", @1:14-1:20 })] -html: "<p><strong>日本語</strong>です</p>" -## underscore_accent "_é_ _ß_" -nodes: [Emphasis(Emphasis { values: [Text(Text { value: "é", @1:2-1:4 })], @1:1-1:5 }), Text(Text { value: " ", @1:5-1:6 }), Emphasis(Emphasis { values: [Text(Text { value: "ß", @1:7-1:9 })], @1:6-1:10 })] -html: "<p><em>é</em> <em>ß</em></p>" -## fullwidth_markers "*a* _b_" -nodes: [Text(Text { value: "*a* _b_", @1:1-1:16 })] -html: "<p>*a* _b_</p>" -## emphasis_after_cjk_punctuation "。*a*、**b**" -nodes: [Text(Text { value: "。", @1:1-1:4 }), Emphasis(Emphasis { values: [Text(Text { value: "a", @1:5-1:6 })], @1:4-1:7 }), Text(Text { value: "、", @1:7-1:10 }), Strong(Strong { values: [Text(Text { value: "b", @1:12-1:13 })], @1:10-1:15 })] -html: "<p>。<em>a</em>、<strong>b</strong></p>" -## combining_mark "e\u{301}*a*" -nodes: [Text(Text { value: "e\u{301}", @1:1-1:4 }), Emphasis(Emphasis { values: [Text(Text { value: "a", @1:5-1:6 })], @1:4-1:7 })] -html: "<p>e\u{301}<em>a</em></p>" -## emoji_zwj "👨\u{200d}👩\u{200d}👧 *a*" -nodes: [Text(Text { value: "👨\u{200d}👩\u{200d}👧 ", @1:1-1:20 }), Emphasis(Emphasis { values: [Text(Text { value: "a", @1:21-1:22 })], @1:20-1:23 })] -html: "<p>👨\u{200d}👩\u{200d}👧 <em>a</em></p>" -## fence_cjk_info "```あ\n日本語\n```\n" -nodes: [Code(Code { value: "日本語", lang: Some("あ"), @1:1-3:4, meta: None, fence: true })] -html: "<pre><code class=\"language-あ\">日本語\n</code></pre>\n" -## fence_in_quote_emoji "> ```🎉\n> x\n> ```\n" -nodes: [Blockquote(Blockquote { values: [Code(Code { value: "x", lang: Some("🎉"), @1:3-3:6, meta: None, fence: true })], @1:1-3:6 })] -html: "<blockquote>\n<pre><code class=\"language-🎉\">x\n</code></pre>\n</blockquote>\n" -## fence_in_list_cjk "- ```あ\n 日本語\n ```\n" -nodes: [List(List { values: [Code(Code { value: "日本語", lang: Some("あ"), @1:3-3:6, meta: None, fence: true })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-3:6 })] -html: "<ul>\n<li>\n<pre><code class=\"language-あ\">日本語\n</code></pre>\n</li>\n</ul>\n" -## math_cjk "$$\nあ\n$$\n\n$い$\n" -nodes: [Math(Math { value: "あ", @1:1-3:3 }), MathInline(MathInline { value: "い", @5:1-5:6 })] -html: "<pre><code class=\"language-math math-display\">あ\n</code></pre>\n<p><code class=\"language-math math-inline\">い</code></p>\n" -## heading_cjk "# 見出し\n\n見出し2\n===\n" -nodes: [Heading(Heading { depth: 1, values: [Text(Text { value: "見出し", @1:3-1:12 })], @1:1-1:12 }), Heading(Heading { depth: 1, values: [Text(Text { value: "見出し2", @3:1-3:11 })], @3:1-4:4 })] -html: "<h1>見出し</h1>\n<h1>見出し2</h1>\n" -## link_cjk "[あ](http://い.jp/う \"え\")" -nodes: [Link(Link { url: Url("http://い.jp/う"), title: Some(Title("え")), values: [Text(Text { value: "あ", @1:2-1:5 })], @1:1-1:31 })] -html: "<p><a href=\"http://%E3%81%84.jp/%E3%81%86\" title=\"え\">あ</a></p>" -## reference_cjk "[あ]\n\n[あ]: /u\n" -nodes: [LinkRef(LinkRef { ident: "あ", label: Some("あ"), values: [Text(Text { value: "あ", @1:2-1:5 })], @1:1-1:6 }), Definition(Definition { @3:1-3:10, url: Url("/u"), title: None, ident: "あ", label: Some("あ") })] -html: "<p><a href=\"/u\">あ</a></p>\n" -## footnote_cjk "あ[^い]\n\n[^い]: う\n" -nodes: [Text(Text { value: "あ", @1:1-1:4 }), FootnoteRef(FootnoteRef { ident: "い", label: Some("い"), @1:4-1:10 }), Footnote(Footnote { ident: "い", values: [Text(Text { value: "う", @3:9-3:12 })], @3:1-3:12 })] -html: "<p>あ<sup><a href=\"#user-content-fn-%E3%81%84\" id=\"user-content-fnref-%E3%81%84\" data-footnote-ref=\"\" aria-describedby=\"footnote-label\">1</a></sup></p>\n<section data-footnotes=\"\" class=\"footnotes\"><h2 id=\"footnote-label\" class=\"sr-only\">Footnotes</h2>\n<ol>\n<li id=\"user-content-fn-%E3%81%84\">\n<p>う <a href=\"#user-content-fnref-%E3%81%84\" data-footnote-backref=\"\" aria-label=\"Back to content\" class=\"data-footnote-backref\">↩</a></p>\n</li>\n</ol>\n</section>\n" -## ideographic_space "a\u{3000}b\n\u{3000}c" -nodes: [Text(Text { value: "a\u{3000}b\n\u{3000}c", @1:1-2:5 })] -html: "<p>a\u{3000}b\n\u{3000}c</p>" -## nbsp_around_emphasis "a\u{a0}*b*\u{a0}c" -nodes: [Text(Text { value: "a\u{a0}", @1:1-1:4 }), Emphasis(Emphasis { values: [Text(Text { value: "b", @1:5-1:6 })], @1:4-1:7 }), Text(Text { value: "\u{a0}c", @1:7-1:10 })] -html: "<p>a\u{a0}<em>b</em>\u{a0}c</p>" -## entity_cjk_context "あ&amp;い&#12354;&#x1F389;" -nodes: [Text(Text { value: "あ&いあ🎉", @1:1-1:29 })] -html: "<p>あ&amp;いあ🎉</p>" -## autolink_cjk "<http://あ.jp/パス>" -nodes: [Link(Link { url: Url("http://あ.jp/パス"), title: None, values: [Text(Text { value: "http://あ.jp/パス", @1:2-1:22 })], @1:1-1:23 })] -html: "<p><a href=\"http://%E3%81%82.jp/%E3%83%91%E3%82%B9\">http://あ.jp/パス</a></p>" -## code_span_cjk "`あ`と``い`う``" -nodes: [CodeInline(CodeInline { value: "あ", @1:1-1:6 }), Text(Text { value: "と", @1:6-1:9 }), CodeInline(CodeInline { value: "い`う", @1:9-1:20 })] -html: "<p><code>あ</code>と<code>い`う</code></p>" -## hard_break_cjk "あ \nい\\\nう" -nodes: [Text(Text { value: "あ", @1:1-1:4 }), Break(Break { @1:4-2:1 }), Text(Text { value: "い", @2:1-2:4 }), Break(Break { @2:4-3:1 }), Text(Text { value: "う", @3:1-3:4 })] -html: "<p>あ<br />\nい<br />\nう</p>" -## nul_in_text "a\0b" -nodes: [Text(Text { value: "a\0b", @1:1-1:4 })] -html: "<p>a�b</p>" -## mdx jsx_self "<a />" -nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [], @1:1-1:6, name: Some("a"), attributes: [] })] -## mdx jsx_attrs "<a b=\"c&amp;d\" e='f' g={h} {...i} j />" -nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [], @1:1-1:39, name: Some("a"), attributes: [Property(MdxJsxAttribute { name: "b", value: Some(Literal("c&d")) }), Property(MdxJsxAttribute { name: "e", value: Some(Literal("f")) }), Property(MdxJsxAttribute { name: "g", value: Some(Expression("h")) }), Expression("...i"), Property(MdxJsxAttribute { name: "j", value: None })] })] -## mdx jsx_member "<a.b.c />" -nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [], @1:1-1:10, name: Some("a.b.c"), attributes: [] })] -## mdx jsx_namespace "<a:b c:d=\"e\" />" -nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [], @1:1-1:16, name: Some("a:b"), attributes: [Property(MdxJsxAttribute { name: "c:d", value: Some(Literal("e")) })] })] -## mdx jsx_fragment "<></>" -nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [], @1:1-1:6, name: None, attributes: [] })] -## mdx jsx_text_pair "<a>x</a>" -nodes: [MdxJsxTextElement(MdxJsxTextElement { children: [Text(Text { value: "x", @1:4-1:5 })], @1:1-1:9, name: Some("a"), attributes: [] })] -## mdx jsx_flow_pair "<a>\n\nx\n\n</a>" -nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [Text(Text { value: "x", @3:1-3:2 })], @1:1-5:5, name: Some("a"), attributes: [] })] -## mdx jsx_flow_indented_child "<a>\n x\n</a>" -nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [Text(Text { value: "x", @2:3-2:4 })], @1:1-3:5, name: Some("a"), attributes: [] })] -## mdx jsx_inline "a <b>c</b> d" -nodes: [Text(Text { value: "a ", @1:1-1:3 }), MdxJsxTextElement(MdxJsxTextElement { children: [Text(Text { value: "c", @1:6-1:7 })], @1:3-1:11, name: Some("b"), attributes: [] }), Text(Text { value: " d", @1:11-1:13 })] -## mdx jsx_inline_self "a <b/> d" -nodes: [Text(Text { value: "a ", @1:1-1:3 }), MdxJsxTextElement(MdxJsxTextElement { children: [], @1:3-1:7, name: Some("b"), attributes: [] }), Text(Text { value: " d", @1:7-1:9 })] -## mdx expr_flow "{a}" -nodes: [MdxFlowExpression(MdxFlowExpression { value: "a", @1:1-1:4 })] -## mdx expr_nested "{a {b} c}" -nodes: [MdxFlowExpression(MdxFlowExpression { value: "a {b} c", @1:1-1:10 })] -## mdx expr_multiline "{a\nb}" -nodes: [MdxFlowExpression(MdxFlowExpression { value: "a\nb", @1:1-2:3 })] -## mdx expr_multiline_indented "{a\n b\n c}" -nodes: [MdxFlowExpression(MdxFlowExpression { value: "a\nb\n c", @1:1-3:6 })] -## mdx expr_text "x {a} y" -nodes: [Text(Text { value: "x ", @1:1-1:3 }), MdxTextExpression(MdxTextExpression { value: "a", @1:3-1:6 }), Text(Text { value: " y", @1:6-1:8 })] -## mdx expr_then_tag "{a} <b/>" -nodes: [MdxFlowExpression(MdxFlowExpression { value: "a", @1:1-1:4 }), MdxJsxFlowElement(MdxJsxFlowElement { children: [], @1:5-1:9, name: Some("b"), attributes: [] })] -## mdx tag_then_expr "<b/> {a}" -nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [], @1:1-1:5, name: Some("b"), attributes: [] }), MdxFlowExpression(MdxFlowExpression { value: "a", @1:6-1:9 })] -## mdx tag_then_text "<b/> x" -nodes: [MdxJsxTextElement(MdxJsxTextElement { children: [], @1:1-1:5, name: Some("b"), attributes: [] }), Text(Text { value: " x", @1:5-1:7 })] -## mdx expr_then_text "{a}x" -nodes: [MdxTextExpression(MdxTextExpression { value: "a", @1:1-1:4 }), Text(Text { value: "x", @1:4-1:5 })] -## mdx jsx_in_expr_child "<a>{b}</a>" -nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [MdxFlowExpression(MdxFlowExpression { value: "b", @1:4-1:7 })], @1:1-1:11, name: Some("a"), attributes: [] })] -## mdx jsx_unclosed_text "<a>b" -nodes: error: Expected a closing tag for `<a>` before the end of the content -## mdx jsx_mismatch "<a></b>" -nodes: error: Unexpected closing tag `</b>`, expected corresponding closing tag for `<a>` (1:1) -## mdx lt_space "a < b" -nodes: [Text(Text { value: "a < b", @1:1-1:6 })] -## mdx lt_digit "a <3" -nodes: error: Unexpected character `3` (U+0033) before name -## mdx attr_no_value "<a b=>" -nodes: error: Unexpected character `>` (U+003E) before attribute value -## mdx attr_space_after_eq "<b e= \"f\"/>" -nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [], @1:1-1:12, name: Some("b"), attributes: [Property(MdxJsxAttribute { name: "e", value: Some(Literal("f")) })] })] -## mdx attr_spaces_around_eq "a <b e = \"f\"/>." -nodes: [Text(Text { value: "a ", @1:1-1:3 }), MdxJsxTextElement(MdxJsxTextElement { children: [], @1:3-1:15, name: Some("b"), attributes: [Property(MdxJsxAttribute { name: "e", value: Some(Literal("f")) })] }), Text(Text { value: ".", @1:15-1:16 })] -## mdx attr_line_before_eq "<b c\n= \"x\">c</b>" -nodes: [MdxJsxTextElement(MdxJsxTextElement { children: [Text(Text { value: "c", @2:7-2:8 })], @1:1-2:12, name: Some("b"), attributes: [Property(MdxJsxAttribute { name: "c", value: Some(Literal("x")) })] })] -## mdx attr_namespace_spaces "a <b xml :\tlang\n= \"de-CH\" foo:bar>c</b>." -nodes: [Text(Text { value: "a ", @1:1-1:3 }), MdxJsxTextElement(MdxJsxTextElement { children: [Text(Text { value: "c", @2:19-2:20 })], @1:3-2:24, name: Some("b"), attributes: [Property(MdxJsxAttribute { name: "xml:lang", value: Some(Literal("de-CH")) }), Property(MdxJsxAttribute { name: "foo:bar", value: None })] }), Text(Text { value: ".", @2:24-2:25 })] -## mdx attr_spaced_names "a <b a b : c d : e = \"f\" g/>." -nodes: [Text(Text { value: "a ", @1:1-1:3 }), MdxJsxTextElement(MdxJsxTextElement { children: [], @1:3-1:29, name: Some("b"), attributes: [Property(MdxJsxAttribute { name: "a", value: None }), Property(MdxJsxAttribute { name: "b:c", value: None }), Property(MdxJsxAttribute { name: "d:e", value: Some(Literal("f")) }), Property(MdxJsxAttribute { name: "g", value: None })] }), Text(Text { value: ".", @1:29-1:30 })] -## mdx attr_expr_space_after_eq "<b e= {f}/>" -nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [], @1:1-1:12, name: Some("b"), attributes: [Property(MdxJsxAttribute { name: "e", value: Some(Expression("f")) })] })] -## mdx jsx_nested_flow "<a>\n<b>\n</b>\n</a>" -nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [MdxJsxFlowElement(MdxJsxFlowElement { children: [], @2:1-3:5, name: Some("b"), attributes: [] })], @1:1-4:5, name: Some("a"), attributes: [] })] -## mdx jsx_in_quote "> <a>\n> x\n> </a>" -nodes: [Blockquote(Blockquote { values: [MdxJsxFlowElement(MdxJsxFlowElement { children: [Text(Text { value: "x", @2:3-2:4 })], @1:3-3:7, name: Some("a"), attributes: [] })], @1:1-3:7 })] -## mdx jsx_in_list "- <a>\n x\n </a>" -nodes: [List(List { values: [MdxJsxFlowElement(MdxJsxFlowElement { children: [Text(Text { value: "x", @2:3-2:4 })], @1:3-3:7, name: Some("a"), attributes: [] })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:3-3:7 })] -## mdx attr_quote_inside "<a b='c\"d' />" -nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [], @1:1-1:14, name: Some("a"), attributes: [Property(MdxJsxAttribute { name: "b", value: Some(Literal("c\"d")) })] })] -## mdx two_tags "<a/><b/>" -nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [], @1:1-1:5, name: Some("a"), attributes: [] }), MdxJsxFlowElement(MdxJsxFlowElement { children: [], @1:5-1:9, name: Some("b"), attributes: [] })] -## mdx two_text_elements "<a>x</a><b>y</b>" -nodes: [MdxJsxTextElement(MdxJsxTextElement { children: [Text(Text { value: "x", @1:4-1:5 })], @1:1-1:9, name: Some("a"), attributes: [] }), MdxJsxTextElement(MdxJsxTextElement { children: [Text(Text { value: "y", @1:12-1:13 })], @1:9-1:17, name: Some("b"), attributes: [] })] -## mdx interrupt_paragraph_tag "a\n<b/>" -nodes: [Text(Text { value: "a", @1:1-1:2 }), MdxJsxFlowElement(MdxJsxFlowElement { children: [], @2:1-2:5, name: Some("b"), attributes: [] })] -## mdx interrupt_paragraph_expr "a\n{b}" -nodes: [Text(Text { value: "a", @1:1-1:2 }), MdxFlowExpression(MdxFlowExpression { value: "b", @2:1-2:4 })] -## mdx two_expressions "{a}\n{b}" -nodes: [MdxFlowExpression(MdxFlowExpression { value: "a", @1:1-1:4 }), MdxFlowExpression(MdxFlowExpression { value: "b", @2:1-2:4 })] -## mdx expr_in_flow_element "<a>\n\n{b}\n\n</a>" -nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [MdxFlowExpression(MdxFlowExpression { value: "b", @3:1-3:4 })], @1:1-5:5, name: Some("a"), attributes: [] })] -## mdx indented_tag " <a/>" -nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [], @1:4-1:8, name: Some("a"), attributes: [] })] -## mdx very_indented_tag " <a/>" -nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [], @1:5-1:9, name: Some("a"), attributes: [] })] -## mdx heading_jsx "# h <a>b</a>" -nodes: [Heading(Heading { depth: 1, values: [Text(Text { value: "h ", @1:3-1:5 }), MdxJsxTextElement(MdxJsxTextElement { children: [Text(Text { value: "b", @1:8-1:9 })], @1:5-1:13, name: Some("a"), attributes: [] })], @1:1-1:13 })] -## mdx emphasis_crossing "*<a>x</a>*" -nodes: [Emphasis(Emphasis { values: [MdxJsxTextElement(MdxJsxTextElement { children: [Text(Text { value: "x", @1:5-1:6 })], @1:2-1:10, name: Some("a"), attributes: [] })], @1:1-1:11 })] -## mdx emphasis_crossing_bad "*<a>x*</a>" -nodes: error: Expected a closing tag for `<a>` before the end of the content -## mdx link_jsx "[<a>x</a>](y)" -nodes: [Link(Link { url: Url("y"), title: None, values: [MdxJsxTextElement(MdxJsxTextElement { children: [Text(Text { value: "x", @1:5-1:6 })], @1:2-1:10, name: Some("a"), attributes: [] })], @1:1-1:14 })] -## mdx code_not_jsx "`<a/>` {b}" -nodes: [CodeInline(CodeInline { value: "<a/>", @1:1-1:7 }), Text(Text { value: " ", @1:7-1:8 }), MdxTextExpression(MdxTextExpression { value: "b", @1:8-1:11 })] -## mdx escaped "\\<a/> \\{b}" -nodes: [Text(Text { value: "<a/> {b}", @1:2-1:11 })] -## mdx attr_entities "<a b=\"&lt;&#x41;\" />" -nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [], @1:1-1:21, name: Some("a"), attributes: [Property(MdxJsxAttribute { name: "b", value: Some(Literal("<A")) })] })] -## mdx attr_multiline_literal "<a b='x\ny' />" -nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [], @1:1-2:6, name: Some("a"), attributes: [Property(MdxJsxAttribute { name: "b", value: Some(Literal("x\ny")) })] })] -## mdx closing_spaces "<a >x</a >" -nodes: [MdxJsxTextElement(MdxJsxTextElement { children: [Text(Text { value: "x", @1:6-1:7 })], @1:1-1:13, name: Some("a"), attributes: [] })] -## mdx lt_space_name "< a>" -nodes: [Text(Text { value: "< a>", @1:1-1:5 })] -## mdx self_closing_space "<a / >" -nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [], @1:1-1:7, name: Some("a"), attributes: [] })] -## mdx attr_on_next_line "<a\nb />" -nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [], @1:1-2:5, name: Some("a"), attributes: [Property(MdxJsxAttribute { name: "b", value: None })] })] -## mdx dashed_name "<a-b />" -nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [], @1:1-1:8, name: Some("a-b"), attributes: [] })] -## mdx dashed_attr "<a b-c=\"d\" />" -nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [], @1:1-1:14, name: Some("a"), attributes: [Property(MdxJsxAttribute { name: "b-c", value: Some(Literal("d")) })] })] -## mdx bad_name_char "<A_b$.c-d />" -nodes: error: Unexpected character `$` (U+0024) after name -## mdx expr_unclosed "{a" -nodes: error: Unexpected end of file in expression, expected a corresponding closing brace for `{` -## mdx expr_empty "{}" -nodes: [MdxFlowExpression(MdxFlowExpression { value: "", @1:1-1:3 })] -## mdx expr_space "{ }" -nodes: [MdxFlowExpression(MdxFlowExpression { value: " ", @1:1-1:4 })] -## mdx empty_flow_element "<a>\n</a>" -nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [], @1:1-2:5, name: Some("a"), attributes: [] })] -## mdx text_element_unclosed_line "<a>x\n</a>" -nodes: error: Expected a closing tag for `<a>` before the end of the content -## mdx stray_close "</a>" -nodes: error: Unexpected closing slash `/` in tag, expected an open tag first -## mdx stray_close_text "x </a>" -nodes: error: Unexpected closing slash `/` in tag, expected an open tag first -## mdx element_with_emphasis "<a>*b*</a>" -nodes: [MdxJsxTextElement(MdxJsxTextElement { children: [Emphasis(Emphasis { values: [Text(Text { value: "b", @1:5-1:6 })], @1:4-1:7 })], @1:1-1:11, name: Some("a"), attributes: [] })] -## mdx flow_element_with_emphasis "<a>\n*b*\n</a>" -nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [Emphasis(Emphasis { values: [Text(Text { value: "b", @2:2-2:3 })], @2:1-2:4 })], @1:1-3:5, name: Some("a"), attributes: [] })] -## mdx flow_element_with_list "<a>\n- b\n</a>" -nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [List(List { values: [Text(Text { value: "b", @2:3-2:4 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @2:3-2:4 })], @1:1-3:5, name: Some("a"), attributes: [] })] -## mdx list_items_unbalanced "- <a>\n- </a>" -nodes: error: Expected a closing tag for `<a>` (1:3) before the end of its container -## mdx spaced_equals "<a b = \"c\" />" -nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [], @1:1-1:14, name: Some("a"), attributes: [Property(MdxJsxAttribute { name: "b", value: Some(Literal("c")) })] })] -## mdx heading "# h\n\ntext *a* [b](c) `d`\n" -nodes: [Heading(Heading { depth: 1, values: [Text(Text { value: "h", @1:3-1:4 })], @1:1-1:4 }), Text(Text { value: "text ", @3:1-3:6 }), Emphasis(Emphasis { values: [Text(Text { value: "a", @3:7-3:8 })], @3:6-3:9 }), Text(Text { value: " ", @3:9-3:10 }), Link(Link { url: Url("c"), title: None, values: [Text(Text { value: "b", @3:11-3:12 })], @3:10-3:16 }), Text(Text { value: " ", @3:16-3:17 }), CodeInline(CodeInline { value: "d", @3:17-3:20 })] -## mdx indented_is_text " not code\n" -nodes: [Text(Text { value: "not code", @1:5-1:13 })] -## mdx indented_heading " # h\n" -nodes: [Heading(Heading { depth: 1, values: [Text(Text { value: "h", @1:8-1:9 })], @1:1-1:9 })] -## mdx no_autolink "<http://a.b>" -nodes: error: Unexpected character `/` (U+002F) before local name -## mdx no_gfm "| a |\n|-|\n\n~a~ www.a.b [^a]\n\n- [ ] a\n" -nodes: [Text(Text { value: "| a |\n|-|", @1:1-2:4 }), Text(Text { value: "~a~ www.a.b [^a]", @4:1-4:17 }), List(List { values: [Text(Text { value: "[ ] a", @6:3-6:8 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @6:3-6:8 })] -## mdx no_math "$a$\n\n$$\na\n$$\n" -nodes: [Text(Text { value: "$a$", @1:1-1:4 }), Text(Text { value: "$$\na\n$$", @3:1-5:3 })] -## mdx frontmatter "---\na\n---\n" -nodes: [Yaml(Yaml { value: "a", @1:1-3:4 })] -## mdx frontmatter_toml "+++\na = 1\n+++\n\n<A />" -nodes: [Toml(Toml { value: "a = 1", @1:1-3:4 }), MdxJsxFlowElement(MdxJsxFlowElement { children: [], @5:1-5:6, name: Some("A"), attributes: [] })] -## mdx frontmatter_unclosed "---\na\n" -nodes: [HorizontalRule(HorizontalRule { marker: Some(Dash), @1:1-1:4 }), Text(Text { value: "a", @2:1-2:2 })] -## mdx esm "import a from 'b'\n\nexport const c = 1\n" -nodes: [MdxJsEsm(MdxJsEsm { value: "import a from 'b'", @1:1-1:18 }), MdxJsEsm(MdxJsEsm { value: "export const c = 1", @3:1-3:19 })] -## mdx definition "[a]: /u\n\n[a]\n" -nodes: [Definition(Definition { @1:1-1:8, url: Url("/u"), title: None, ident: "a", label: Some("a") }), LinkRef(LinkRef { ident: "a", label: Some("a"), values: [Text(Text { value: "a", @3:2-3:3 })], @3:1-3:4 })] -## mdx list_deep_indent "- a\n" -nodes: [List(List { values: [Text(Text { value: "a", @1:8-1:9 })], index: 0, level: 0, ordered: false, checked: None, spread: false, start: None, marker: Some(Dash), @1:8-1:9 })] -## mdx fence "```rust\ncode\n```\n" -nodes: [Code(Code { value: "code", lang: Some("rust"), @1:1-3:4, meta: None, fence: true })] -## mdx lone_cr_after_lt "<\r/-x" -nodes: [Text(Text { value: "<\r/-x", @1:1-2:4 })] -## mdx crlf_after_lt "<\r\n/-x" -nodes: [Text(Text { value: "<\r\n/-x", @1:1-2:4 })] -## mdx expression_lone_cr "{\r }" -nodes: [MdxFlowExpression(MdxFlowExpression { value: "\r", @1:1-2:3 })] -## mdx expression_cr_indent "{a\r b\r\n c\n d}" -nodes: [MdxFlowExpression(MdxFlowExpression { value: "a\r b\r\nc\nd", @1:1-4:5 })] -## mdx tilde_not_delimiter "*~*a" -nodes: [Text(Text { value: "*~*a", @1:1-1:5 })] -## mdx tilde_strong_not_delimiter "**~**あ" -nodes: [Text(Text { value: "**~**あ", @1:1-1:9 })] -## mdx dollar_after_name "<a$/>" -nodes: error: Unexpected character `$` (U+0024) after name -## mdx dollar_after_space "<a $b/>" -nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [], @1:1-1:8, name: Some("a"), attributes: [Property(MdxJsxAttribute { name: "$b", value: None })] })] -## mdx cjk_tag_name "<あ b=\"c\" />" -nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [], @1:1-1:14, name: Some("あ"), attributes: [Property(MdxJsxAttribute { name: "b", value: Some(Literal("c")) })] })] -## mdx fullwidth_underscore_name "<a_b />" -nodes: [MdxJsxFlowElement(MdxJsxFlowElement { children: [], @1:1-1:10, name: Some("a_b"), attributes: [] })] -## mdx jsx_text_cjk "あ <b>い</b> う" -nodes: [Text(Text { value: "あ ", @1:1-1:5 }), MdxJsxTextElement(MdxJsxTextElement { children: [Text(Text { value: "い", @1:8-1:11 })], @1:5-1:15, name: Some("b"), attributes: [] }), Text(Text { value: " う", @1:15-1:19 })] -## mdx expression_cjk "{あ} い" -nodes: [MdxTextExpression(MdxTextExpression { value: "あ", @1:1-1:6 }), Text(Text { value: " い", @1:6-1:10 })] diff --git a/crates/mq-markdown/tests/parser_tree_tests.rs b/crates/mq-markdown/tests/parser_tree_tests.rs new file mode 100644 index 000000000..d4dd74506 --- /dev/null +++ b/crates/mq-markdown/tests/parser_tree_tests.rs @@ -0,0 +1,3441 @@ +//! What the parser makes of each input: the nodes with their positions and, for Markdown, the HTML. +//! +//! Nodes are written one per line, children indented under their parent, as the kind, the fields that +//! are not empty or default, and `@line:column-line:column`. + +use mq_markdown::{ + HorizontalRuleMarker, ListMarker, Markdown, MdxAttributeContent, MdxAttributeValue, Node, Position, TableAlignKind, +}; +use rstest::rstest; + +fn position(position: &Option<Position>) -> String { + position.as_ref().map_or(String::new(), |position| { + format!( + " @{}:{}-{}:{}", + position.start.line, position.start.column, position.end.line, position.end.column + ) + }) +} + +fn field<T: std::fmt::Debug>(name: &str, value: &Option<T>) -> String { + value + .as_ref() + .map_or(String::new(), |value| format!(" {name}={value:?}")) +} + +fn flag(name: &str, set: bool) -> String { + if set { format!(" {name}") } else { String::new() } +} + +fn attributes(attributes: &[MdxAttributeContent]) -> String { + attributes + .iter() + .map(|attribute| match attribute { + MdxAttributeContent::Expression(value) => format!(" {{{value}}}"), + MdxAttributeContent::Property(property) => match &property.value { + None => format!(" {}", property.name), + Some(MdxAttributeValue::Literal(value)) => format!(" {}={value:?}", property.name), + Some(MdxAttributeValue::Expression(value)) => format!(" {}={{{value}}}", property.name), + }, + }) + .collect() +} + +/// The line of `node` and the children under it. +fn node_line(node: &Node) -> (String, &[Node]) { + match node { + Node::Text(text) => (format!("text {:?}{}", text.value, position(&text.position)), &[]), + Node::Heading(heading) => ( + format!("heading {}{}", heading.depth, position(&heading.position)), + &heading.values, + ), + Node::Blockquote(quote) => (format!("blockquote{}", position(&quote.position)), &quote.values), + Node::List(list) => { + let marker = list.marker.map(|marker| match marker { + ListMarker::Continuation => "continuation".to_string(), + marker => marker.as_char().unwrap_or_default().to_string(), + }); + ( + format!( + "list level={} index={}{}{}{}{}{}{}", + list.level, + list.index, + flag("ordered", list.ordered), + field("start", &list.start), + field("checked", &list.checked), + flag("spread", list.spread), + marker.map_or(String::new(), |marker| format!(" marker={marker}")), + position(&list.position) + ), + &list.values, + ) + } + Node::Code(code) => ( + format!( + "code {:?}{}{}{}{}", + code.value, + field("lang", &code.lang), + field("meta", &code.meta), + flag("fence", code.fence), + position(&code.position) + ), + &[], + ), + Node::Html(html) => (format!("html {:?}{}", html.value, position(&html.position)), &[]), + Node::Yaml(yaml) => (format!("yaml {:?}{}", yaml.value, position(&yaml.position)), &[]), + Node::Toml(toml) => (format!("toml {:?}{}", toml.value, position(&toml.position)), &[]), + Node::Math(math) => (format!("math {:?}{}", math.value, position(&math.position)), &[]), + Node::MathInline(math) => (format!("math_inline {:?}{}", math.value, position(&math.position)), &[]), + Node::CodeInline(code) => (format!("code_inline {:?}{}", code.value, position(&code.position)), &[]), + Node::Break(node) => (format!("break{}", position(&node.position)), &[]), + Node::HorizontalRule(rule) => { + let marker = rule.marker.map(HorizontalRuleMarker::as_char); + ( + format!("hr{}{}", field("marker", &marker), position(&rule.position)), + &[], + ) + } + Node::Emphasis(node) => (format!("emphasis{}", position(&node.position)), &node.values), + Node::Strong(node) => (format!("strong{}", position(&node.position)), &node.values), + Node::Delete(node) => (format!("delete{}", position(&node.position)), &node.values), + Node::Link(link) => ( + format!( + "link {:?}{}{}", + link.url.as_str(), + field("title", &link.title.as_ref().map(|title| title.to_value())), + position(&link.position) + ), + &link.values, + ), + Node::Image(image) => ( + format!( + "image {:?} alt={:?}{}{}", + image.url, + image.alt, + field("title", &image.title), + position(&image.position) + ), + &[], + ), + Node::Definition(definition) => ( + format!( + "definition {:?} {:?}{}{}{}", + definition.ident, + definition.url.as_str(), + field("label", &definition.label), + field("title", &definition.title.as_ref().map(|title| title.to_value())), + position(&definition.position) + ), + &[], + ), + Node::LinkRef(reference) => ( + format!( + "link_ref {:?}{}{}", + reference.ident, + field("label", &reference.label), + position(&reference.position) + ), + &reference.values, + ), + Node::ImageRef(reference) => ( + format!( + "image_ref {:?} alt={:?}{}{}", + reference.ident, + reference.alt, + field("label", &reference.label), + position(&reference.position) + ), + &[], + ), + Node::Footnote(footnote) => ( + format!("footnote {:?}{}", footnote.ident, position(&footnote.position)), + &footnote.values, + ), + Node::FootnoteRef(reference) => ( + format!( + "footnote_ref {:?}{}{}", + reference.ident, + field("label", &reference.label), + position(&reference.position) + ), + &[], + ), + Node::TableRow(row) => (format!("row{}", position(&row.position)), &row.values), + Node::TableCell(cell) => ( + format!("cell {}:{}{}", cell.row, cell.column, position(&cell.position)), + &cell.values, + ), + Node::TableAlign(align) => { + let kinds = align + .align + .iter() + .map(|kind| match kind { + TableAlignKind::Left => "left", + TableAlignKind::Right => "right", + TableAlignKind::Center => "center", + TableAlignKind::None => "none", + }) + .collect::<Vec<_>>(); + (format!("align {}{}", kinds.join(","), position(&align.position)), &[]) + } + Node::MdxJsxFlowElement(element) => ( + format!( + "jsx_flow <{}{}>{}", + element.name.as_deref().unwrap_or_default(), + attributes(&element.attributes), + position(&element.position) + ), + &element.children, + ), + Node::MdxJsxTextElement(element) => ( + format!( + "jsx_text <{}{}>{}", + element.name.as_deref().unwrap_or_default(), + attributes(&element.attributes), + position(&element.position) + ), + &element.children, + ), + Node::MdxFlowExpression(expression) => ( + format!( + "expression_flow {:?}{}", + expression.value, + position(&expression.position) + ), + &[], + ), + Node::MdxTextExpression(expression) => ( + format!( + "expression_text {:?}{}", + expression.value, + position(&expression.position) + ), + &[], + ), + Node::MdxJsEsm(esm) => (format!("esm {:?}{}", esm.value, position(&esm.position)), &[]), + Node::Fragment(fragment) => ("fragment".to_string(), &fragment.values), + Node::Empty => ("empty".to_string(), &[]), + #[allow(unreachable_patterns)] + node => panic!("unexpected node {}", node.name()), + } +} + +fn write_tree(nodes: &[Node], depth: usize, out: &mut Vec<String>) { + for node in nodes { + let (line, children) = node_line(node); + out.push(format!("{}{line}", " ".repeat(depth))); + write_tree(children, depth + 1, out); + } +} + +fn tree(nodes: &[Node]) -> String { + let mut out = Vec::new(); + write_tree(nodes, 0, &mut out); + out.join("\n") +} + +#[rstest] +#[case::empty("", "", "")] +#[case::blank_lines("\n\n \n", "", "")] +#[case::paragraph("hello\n", "<p>hello</p>\n", r#"text "hello" @1:1-1:6"#)] +#[case::paragraph_no_eol("hello", "<p>hello</p>", r#"text "hello" @1:1-1:6"#)] +#[case::paragraph_multiline("a\nb\nc\n", "<p>a\nb\nc</p>\n", r#"text "a\nb\nc" @1:1-3:2"#)] +#[case::paragraph_indented_continuation("a\n b\n", "<p>a\nb</p>\n", r#"text "a\nb" @1:1-2:6"#)] +#[case::paragraph_trailing_space("あい\n うえ \n", "<p>あい\nうえ</p>\n", r#"text "あい\nうえ" @1:1-2:9"#)] +#[case::two_paragraphs( + "a\n\nb\n", + "<p>a</p>\n<p>b</p>\n", + r#" +text "a" @1:1-1:2 +text "b" @3:1-3:2 +"# +)] +#[case::crlf( + "a\r\nb\r\n\r\nc\r\n", + "<p>a\r\nb</p>\r\n<p>c</p>\r\n", + r#" +text "a\r\nb" @1:1-2:2 +text "c" @4:1-4:2 +"# +)] +#[case::crlf_fence( + "```\r\na\r\nb\r\n```\r\n", + "<pre><code>a\r\nb\r\n</code></pre>\r\n", + r#"code "a\r\nb" fence @1:1-4:4"# +)] +#[case::cr_only("a\rb\r", "<p>a\rb</p>\r", r#"text "a\rb" @1:1-2:2"#)] +#[case::atx_h1( + "# title\n", + "<h1>title</h1>\n", + r#" +heading 1 @1:1-1:8 + text "title" @1:3-1:8 +"# +)] +#[case::atx_h6( + "###### title\n", + "<h6>title</h6>\n", + r#" +heading 6 @1:1-1:13 + text "title" @1:8-1:13 +"# +)] +#[case::atx_seven( + "####### title\n", + "<p>####### title</p>\n", + r########"text "####### title" @1:1-1:14"######## +)] +#[case::atx_no_space("#title\n", "<p>#title</p>\n", r##"text "#title" @1:1-1:7"##)] +#[case::atx_empty("#\n", "<h1></h1>\n", "heading 1 @1:1-1:2")] +#[case::atx_closing( + "## title ##\n", + "<h2>title</h2>\n", + r#" +heading 2 @1:1-1:12 + text "title" @1:4-1:9 +"# +)] +#[case::atx_closing_no_space( + "## title##\n", + "<h2>title##</h2>\n", + r#" +heading 2 @1:1-1:11 + text "title##" @1:4-1:11 +"# +)] +#[case::atx_indent( + " ## title ## \n", + "<h2>title</h2>\n", + r#" +heading 2 @1:1-1:16 + text "title" @1:6-1:11 +"# +)] +#[case::atx_multibyte( + "# あ h #\n", + "<h1>あ h</h1>\n", + r#" +heading 1 @1:1-1:10 + text "あ h" @1:3-1:8 +"# +)] +#[case::atx_interrupts_paragraph( + "a\n# b\n", + "<p>a</p>\n<h1>b</h1>\n", + r#" +text "a" @1:1-1:2 +heading 1 @2:1-2:4 + text "b" @2:3-2:4 +"# +)] +#[case::setext_h1( + "Title\n===\n", + "<h1>Title</h1>\n", + r#" +heading 1 @1:1-2:4 + text "Title" @1:1-1:6 +"# +)] +#[case::setext_h2( + "a\n ---\n", + "<h2>a</h2>\n", + r#" +heading 2 @1:1-2:6 + text "a" @1:1-1:2 +"# +)] +#[case::setext_multiline( + "a\nb\n---\n", + "<h2>a\nb</h2>\n", + r#" +heading 2 @1:1-3:4 + text "a\nb" @1:1-2:2 +"# +)] +#[case::thematic_star("***\n", "<hr />\n", "hr marker='*' @1:1-1:4")] +#[case::thematic_dash_spaced("- - -\n", "<hr />\n", "hr marker='-' @1:1-1:6")] +#[case::thematic_underscore(" ___ \n", "<hr />\n", "hr marker='_' @1:1-1:8")] +#[case::thematic_two("**\n", "<p>**</p>\n", r#"text "**" @1:1-1:3"#)] +#[case::fence_lang_meta( + "```rust title\nlet a;\n```\n", + "<pre><code class=\"language-rust\">let a;\n</code></pre>\n", + r#"code "let a;" lang="rust" meta="title" fence @1:1-3:4"# +)] +#[case::fence_indented( + " ```\n a\n b\n ```\n", + "<pre><code>a\n b\n</code></pre>\n", + r#"code "a\n b" fence @1:3-4:6"# +)] +#[case::fence_tilde("~~~\na\n~~~\n", "<pre><code>a\n</code></pre>\n", r#"code "a" fence @1:1-3:4"#)] +#[case::fence_unclosed("```\na\n", "<pre><code>a\n</code></pre>\n", r#"code "a" fence @1:1-3:1"#)] +#[case::fence_empty( + "para\n\n\n```\n```\n", + "<p>para</p>\n<pre><code></code></pre>\n", + r#" +text "para" @1:1-1:5 +code "" fence @4:1-5:4 +"# +)] +#[case::fence_longer_close("```\na\n`````\n", "<pre><code>a\n</code></pre>\n", r#"code "a" fence @1:1-3:6"#)] +#[case::fence_shorter_close( + "````\na\n```\nb\n````\n", + "<pre><code>a\n```\nb\n</code></pre>\n", + r#"code "a\n```\nb" fence @1:1-5:5"# +)] +#[case::fence_blank_inside( + "```\na\n\nb\n```\n", + "<pre><code>a\n\nb\n</code></pre>\n", + r#"code "a\n\nb" fence @1:1-5:4"# +)] +#[case::fence_backtick_info("``` a`b\nc\n", "<p>``` a`b\nc</p>\n", r#"text "``` a`b\nc" @1:1-2:2"#)] +#[case::fence_interrupts_paragraph( + "a\n```\nb\n```\n", + "<p>a</p>\n<pre><code>b\n</code></pre>\n", + r#" +text "a" @1:1-1:2 +code "b" fence @2:1-4:4 +"# +)] +#[case::indented_code( + " code\n\n more\n\nx\n", + "<pre><code>code\n\n more\n</code></pre>\n<p>x</p>\n", + r#" +code "code\n\n more" @1:1-3:11 +text "x" @5:1-5:2 +"# +)] +#[case::quote( + "> a\n> b\n", + "<blockquote>\n<p>a\nb</p>\n</blockquote>\n", + r#" +blockquote @1:1-2:4 + text "a\nb" @1:3-2:4 +"# +)] +#[case::quote_indented_lazy( + " > a\nb\n", + "<blockquote>\n<p>a\nb</p>\n</blockquote>\n", + r#" +blockquote @1:1-2:2 + text "a\nb" @1:5-2:2 +"# +)] +#[case::quote_heading( + "> # h\n>\n> c\n", + "<blockquote>\n<h1>h</h1>\n<p>c</p>\n</blockquote>\n", + r#" +blockquote @1:1-3:4 + heading 1 @1:3-1:6 + text "h" @1:5-1:6 + text "c" @3:3-3:4 +"# +)] +#[case::quote_no_space( + ">a\n", + "<blockquote>\n<p>a</p>\n</blockquote>\n", + r#" +blockquote @1:1-1:3 + text "a" @1:2-1:3 +"# +)] +#[case::quote_empty(">\n", "<blockquote>\n</blockquote>\n", "blockquote @1:1-1:2")] +#[case::quote_nested( + "> > a\n> b\n", + "<blockquote>\n<blockquote>\n<p>a\nb</p>\n</blockquote>\n</blockquote>\n", + r#" +blockquote @1:1-2:4 + blockquote @1:3-2:4 + text "a\nb" @1:5-2:4 +"# +)] +#[case::quote_lazy_nested( + "> > a\nb\n", + "<blockquote>\n<blockquote>\n<p>a\nb</p>\n</blockquote>\n</blockquote>\n", + r#" +blockquote @1:1-2:2 + blockquote @1:3-2:2 + text "a\nb" @1:5-2:2 +"# +)] +#[case::quote_blank_ends( + "> a\n\n> b\n", + "<blockquote>\n<p>a</p>\n</blockquote>\n<blockquote>\n<p>b</p>\n</blockquote>\n", + r#" +blockquote @1:1-1:4 + text "a" @1:3-1:4 +blockquote @3:1-3:4 + text "b" @3:3-3:4 +"# +)] +#[case::quote_fence( + "> ```\n> a\n> ```\n", + "<blockquote>\n<pre><code>a\n</code></pre>\n</blockquote>\n", + r#" +blockquote @1:1-3:6 + code "a" fence @1:3-3:6 +"# +)] +#[case::quote_no_lazy_after_fence( + "> ```\n> a\nb\n", + "<blockquote>\n<pre><code>a\n</code></pre>\n</blockquote>\n<p>b</p>\n", + r#" +blockquote @1:1-2:4 + code "a" fence @1:3-2:4 +text "b" @3:1-3:2 +"# +)] +#[case::quote_interrupts_paragraph( + "a\n> b\n", + "<p>a</p>\n<blockquote>\n<p>b</p>\n</blockquote>\n", + r#" +text "a" @1:1-1:2 +blockquote @2:1-2:4 + text "b" @2:3-2:4 +"# +)] +#[case::quote_thematic_not_lazy( + "> a\n---\n", + "<blockquote>\n<p>a</p>\n</blockquote>\n<hr />\n", + r#" +blockquote @1:1-1:4 + text "a" @1:3-1:4 +hr marker='-' @2:1-2:4 +"# +)] +#[case::quote_multibyte( + "> あ\n> い\n", + "<blockquote>\n<p>あ\nい</p>\n</blockquote>\n", + r#" +blockquote @1:1-2:6 + text "あ\nい" @1:3-2:6 +"# +)] +#[case::bullet( + "- a\n- b\n", + "<ul>\n<li>a</li>\n<li>b</li>\n</ul>\n", + r#" +list level=0 index=0 marker=- @1:3-1:4 + text "a" @1:3-1:4 +list level=0 index=1 marker=- @2:3-2:4 + text "b" @2:3-2:4 +"# +)] +#[case::bullet_loose( + "- a\n\n- b\n", + "<ul>\n<li>\n<p>a</p>\n</li>\n<li>\n<p>b</p>\n</li>\n</ul>\n", + r#" +list level=0 index=0 spread marker=- @1:3-1:4 + text "a" @1:3-1:4 +list level=0 index=1 spread marker=- @3:3-3:4 + text "b" @3:3-3:4 +"# +)] +#[case::bullet_indent( + " - a\n", + "<ul>\n<li>a</li>\n</ul>\n", + r#" +list level=0 index=0 marker=- @1:4-1:5 + text "a" @1:4-1:5 +"# +)] +#[case::bullet_mixed_markers( + "* a\n+ b\n", + "<ul>\n<li>a</li>\n</ul>\n<ul>\n<li>b</li>\n</ul>\n", + r#" +list level=0 index=0 marker=* @1:3-1:4 + text "a" @1:3-1:4 +list level=0 index=0 marker=+ @2:3-2:4 + text "b" @2:3-2:4 +"# +)] +#[case::ordered( + "1. a\n2. b\n", + "<ol>\n<li>a</li>\n<li>b</li>\n</ol>\n", + r#" +list level=0 index=0 ordered start=1 marker=. @1:4-1:5 + text "a" @1:4-1:5 +list level=0 index=1 ordered start=1 marker=. @2:4-2:5 + text "b" @2:4-2:5 +"# +)] +#[case::ordered_paren( + "5) a\n", + "<ol start=\"5\">\n<li>a</li>\n</ol>\n", + r#" +list level=0 index=0 ordered start=5 marker=) @1:4-1:5 + text "a" @1:4-1:5 +"# +)] +#[case::ordered_zero( + "0. a\n", + "<ol start=\"0\">\n<li>a</li>\n</ol>\n", + r#" +list level=0 index=0 ordered start=0 marker=. @1:4-1:5 + text "a" @1:4-1:5 +"# +)] +#[case::ordered_too_long("1234567890. a\n", "<p>1234567890. a</p>\n", r#"text "1234567890. a" @1:1-1:14"#)] +#[case::ordered_mixed_delimiters( + "1. a\n2) b\n", + "<ol>\n<li>a</li>\n</ol>\n<ol start=\"2\">\n<li>b</li>\n</ol>\n", + r#" +list level=0 index=0 ordered start=1 marker=. @1:4-1:5 + text "a" @1:4-1:5 +list level=0 index=0 ordered start=2 marker=) @2:4-2:5 + text "b" @2:4-2:5 +"# +)] +#[case::task( + "- [ ] x\n- [x] y\n- [X] z\n", + "<ul>\n<li><input type=\"checkbox\" disabled=\"\" /> x</li>\n<li><input type=\"checkbox\" checked=\"\" disabled=\"\" /> y</li>\n<li><input type=\"checkbox\" checked=\"\" disabled=\"\" /> z</li>\n</ul>\n", + r#" +list level=0 index=0 checked=false marker=- @1:7-1:8 + text "x" @1:7-1:8 +list level=0 index=1 checked=true marker=- @2:7-2:8 + text "y" @2:7-2:8 +list level=0 index=2 checked=true marker=- @3:7-3:8 + text "z" @3:7-3:8 +"# +)] +#[case::task_empty( + "- [ ]\n", + "<ul>\n<li>[ ]</li>\n</ul>\n", + r#" +list level=0 index=0 marker=- @1:3-1:6 + text "[ ]" @1:3-1:6 +"# +)] +#[case::task_no_space( + "- [ ]x\n", + "<ul>\n<li>[ ]x</li>\n</ul>\n", + r#" +list level=0 index=0 marker=- @1:3-1:7 + text "[ ]x" @1:3-1:7 +"# +)] +#[case::nested( + "- a\n - b\n c\n", + "<ul>\n<li>a\n<ul>\n<li>b\nc</li>\n</ul>\n</li>\n</ul>\n", + r#" +list level=0 index=0 marker=- @1:3-1:4 + text "a" @1:3-1:4 +list level=1 index=0 marker=- @2:5-3:6 + text "b\nc" @2:5-3:6 +"# +)] +#[case::nested_ordered( + "1. a\n 1. b\n 2. c\n2. d\n", + "<ol>\n<li>a\n<ol>\n<li>b</li>\n<li>c</li>\n</ol>\n</li>\n<li>d</li>\n</ol>\n", + r#" +list level=0 index=0 ordered start=1 marker=. @1:4-1:5 + text "a" @1:4-1:5 +list level=1 index=0 ordered start=1 marker=. @2:7-2:8 + text "b" @2:7-2:8 +list level=1 index=1 ordered start=1 marker=. @3:7-3:8 + text "c" @3:7-3:8 +list level=0 index=1 ordered start=1 marker=. @4:4-4:5 + text "d" @4:4-4:5 +"# +)] +#[case::nested_dedent( + "- a\n - b\n- c\n", + "<ul>\n<li>a\n<ul>\n<li>b</li>\n</ul>\n</li>\n<li>c</li>\n</ul>\n", + r#" +list level=0 index=0 marker=- @1:3-1:4 + text "a" @1:3-1:4 +list level=1 index=0 marker=- @2:5-2:6 + text "b" @2:5-2:6 +list level=0 index=1 marker=- @3:3-3:4 + text "c" @3:3-3:4 +"# +)] +#[case::list_item_two_paragraphs( + "- a\n\n b\n", + "<ul>\n<li>\n<p>a</p>\n<p>b</p>\n</li>\n</ul>\n", + r#" +list level=0 index=0 marker=- @1:3-3:4 + text "a" @1:3-1:4 + text "b" @3:3-3:4 +"# +)] +#[case::two_paragraphs_then_item( + "- a\n\n b\n- c\n", + "<ul>\n<li>\n<p>a</p>\n<p>b</p>\n</li>\n<li>\n<p>c</p>\n</li>\n</ul>\n", + r#" +list level=0 index=0 marker=- @1:3-3:4 + text "a" @1:3-1:4 + text "b" @3:3-3:4 +list level=0 index=1 marker=- @4:3-4:4 + text "c" @4:3-4:4 +"# +)] +#[case::empty_item("-\n", "<ul>\n<li></li>\n</ul>\n", "list level=0 index=0 marker=- @1:1-1:2")] +#[case::empty_item_content( + "-\n foo\n", + "<ul>\n<li>foo</li>\n</ul>\n", + r#" +list level=0 index=0 marker=- @2:3-2:6 + text "foo" @2:3-2:6 +"# +)] +#[case::empty_item_blank_then_text( + "-\n\n foo\n", + "<ul>\n<li></li>\n</ul>\n<p>foo</p>\n", + r#" +list level=0 index=0 marker=- @1:1-1:2 +text "foo" @3:3-3:6 +"# +)] +#[case::empty_items( + "-\n-\n", + "<ul>\n<li></li>\n<li></li>\n</ul>\n", + r#" +list level=0 index=0 marker=- @1:1-1:2 +list level=0 index=1 marker=- @2:1-2:2 +"# +)] +#[case::wide_marker_gap( + "- a\n b\n", + "<ul>\n<li>a\nb</li>\n</ul>\n", + r#" +list level=0 index=0 marker=- @1:5-2:6 + text "a\nb" @1:5-2:6 +"# +)] +#[case::code_in_item( + "- a\n", + "<ul>\n<li>\n<pre><code>a\n</code></pre>\n</li>\n</ul>\n", + r#" +list level=0 index=0 marker=- @1:3-1:8 + code "a" @1:3-1:8 +"# +)] +#[case::lazy( + "- a\nb\n", + "<ul>\n<li>a\nb</li>\n</ul>\n", + r#" +list level=0 index=0 marker=- @1:3-2:2 + text "a\nb" @1:3-2:2 +"# +)] +#[case::lazy_after_blank( + "- a\n\nb\n", + "<ul>\n<li>a</li>\n</ul>\n<p>b</p>\n", + r#" +list level=0 index=0 marker=- @1:3-1:4 + text "a" @1:3-1:4 +text "b" @3:1-3:2 +"# +)] +#[case::list_interrupts_paragraph( + "a\n- b\n", + "<p>a</p>\n<ul>\n<li>b</li>\n</ul>\n", + r#" +text "a" @1:1-1:2 +list level=0 index=0 marker=- @2:3-2:4 + text "b" @2:3-2:4 +"# +)] +#[case::ordered_two_no_interrupt("a\n2. b\n", "<p>a\n2. b</p>\n", r#"text "a\n2. b" @1:1-2:5"#)] +#[case::ordered_one_interrupts( + "a\n1. b\n", + "<p>a</p>\n<ol>\n<li>b</li>\n</ol>\n", + r#" +text "a" @1:1-1:2 +list level=0 index=0 ordered start=1 marker=. @2:4-2:5 + text "b" @2:4-2:5 +"# +)] +#[case::empty_item_no_interrupt( + "a\n-\n", + "<h2>a</h2>\n", + r#" +heading 2 @1:1-2:2 + text "a" @1:1-1:2 +"# +)] +#[case::list_in_quote( + "> - a\n> - b\n", + "<blockquote>\n<ul>\n<li>a</li>\n<li>b</li>\n</ul>\n</blockquote>\n", + r#" +blockquote @1:1-2:6 + list level=0 index=0 marker=- @1:5-1:6 + text "a" @1:5-1:6 + list level=0 index=1 marker=- @2:5-2:6 + text "b" @2:5-2:6 +"# +)] +#[case::quote_in_list( + "- > a\n > b\n", + "<ul>\n<li>\n<blockquote>\n<p>a\nb</p>\n</blockquote>\n</li>\n</ul>\n", + r#" +list level=0 index=0 marker=- @1:3-2:6 + blockquote @1:3-2:6 + text "a\nb" @1:5-2:6 +"# +)] +#[case::quote_in_list_level( + "- a\n > - b\n", + "<ul>\n<li>a\n<blockquote>\n<ul>\n<li>b</li>\n</ul>\n</blockquote>\n</li>\n</ul>\n", + r#" +list level=0 index=0 marker=- @1:3-2:8 + text "a" @1:3-1:4 + blockquote @2:3-2:8 + list level=0 index=0 marker=- @2:7-2:8 + text "b" @2:7-2:8 +"# +)] +#[case::heading_in_item( + "- # h\n text\n", + "<ul>\n<li>\n<h1>h</h1>\ntext</li>\n</ul>\n", + r#" +list level=0 index=0 marker=- @1:3-2:7 + heading 1 @1:3-1:6 + text "h" @1:5-1:6 + text "text" @2:3-2:7 +"# +)] +#[case::fence_in_item( + "- ```\n a\n ```\n", + "<ul>\n<li>\n<pre><code>a\n</code></pre>\n</li>\n</ul>\n", + r#" +list level=0 index=0 marker=- @1:3-3:6 + code "a" fence @1:3-3:6 +"# +)] +#[case::multibyte_item( + "- あ\n- い\n", + "<ul>\n<li>あ</li>\n<li>い</li>\n</ul>\n", + r#" +list level=0 index=0 marker=- @1:3-1:6 + text "あ" @1:3-1:6 +list level=0 index=1 marker=- @2:3-2:6 + text "い" @2:3-2:6 +"# +)] +#[case::list_after_paragraph_blank( + "a\n\n- b\n", + "<p>a</p>\n<ul>\n<li>b</li>\n</ul>\n", + r#" +text "a" @1:1-1:2 +list level=0 index=0 marker=- @3:3-3:4 + text "b" @3:3-3:4 +"# +)] +#[case::two_lists_split( + "- a\n\n\n- b\n", + "<ul>\n<li>\n<p>a</p>\n</li>\n<li>\n<p>b</p>\n</li>\n</ul>\n", + r#" +list level=0 index=0 spread marker=- @1:3-1:4 + text "a" @1:3-1:4 +list level=0 index=1 spread marker=- @4:3-4:4 + text "b" @4:3-4:4 +"# +)] +#[case::deep_quote( + ">>>>>> a\n", + "<blockquote>\n<blockquote>\n<blockquote>\n<blockquote>\n<blockquote>\n<blockquote>\n<p>a</p>\n</blockquote>\n</blockquote>\n</blockquote>\n</blockquote>\n</blockquote>\n</blockquote>\n", + r#" +blockquote @1:1-1:9 + blockquote @1:2-1:9 + blockquote @1:3-1:9 + blockquote @1:4-1:9 + blockquote @1:5-1:9 + blockquote @1:6-1:9 + text "a" @1:8-1:9 +"# +)] +#[case::fence_unclosed_no_eol("```\na", "<pre><code>a\n</code></pre>\n", r#"code "a" fence @1:1-2:2"#)] +#[case::fence_unclosed_in_item( + "- ```\n a\n", + "<ul>\n<li>\n<pre><code>a\n</code></pre>\n</li>\n</ul>", + r#" +list level=0 index=0 marker=- @1:3-3:1 + code "a" fence @1:3-3:1 +"# +)] +#[case::fence_unclosed_in_item_no_eol( + "- ```\n a", + "<ul>\n<li>\n<pre><code>a\n</code></pre>\n</li>\n</ul>", + r#" +list level=0 index=0 marker=- @1:3-2:4 + code "a" fence @1:3-2:4 +"# +)] +#[case::fence_unclosed_in_quote_no_eol( + "> ```\n> a", + "<blockquote>\n<pre><code>a\n</code></pre>\n</blockquote>", + r#" +blockquote @1:1-2:4 + code "a" fence @1:3-2:4 +"# +)] +#[case::fence_unclosed_in_quote_eof( + "> ```\n> a\n", + "<blockquote>\n<pre><code>a\n</code></pre>\n</blockquote>", + r#" +blockquote @1:1-3:1 + code "a" fence @1:3-3:1 +"# +)] +#[case::fence_unclosed_trailing_blank("```\na\n\n", "<pre><code>a\n\n</code></pre>\n", r#"code "a\n" fence @1:1-4:1"#)] +#[case::fence_unclosed_item_then_para( + "- ```\n a\nb\n", + "<ul>\n<li>\n<pre><code>a\n</code></pre>\n</li>\n</ul>\n<p>b</p>\n", + r#" +list level=0 index=0 marker=- @1:3-2:4 + code "a" fence @1:3-2:4 +text "b" @3:1-3:2 +"# +)] +#[case::item_code_then_unindented_text( + "- ===\n\n a\n===", + "<ul>\n<li>\n<p>===</p>\n<pre><code>a\n</code></pre>\n</li>\n</ul>\n<p>===</p>", + r#" +list level=0 index=0 marker=- @1:3-3:8 + text "===" @1:3-1:6 + code "a" @3:3-3:8 +text "===" @4:1-4:4 +"# +)] +#[case::quote_setext_dashes( + "> a\n> ---\n> ---\n> a", + "<blockquote>\n<h2>a</h2>\n<hr />\n<p>a</p>\n</blockquote>", + r#" +blockquote @1:1-4:4 + heading 2 @1:3-2:6 + text "a" @1:3-1:4 + hr marker='-' @3:3-3:6 + text "a" @4:3-4:4 +"# +)] +#[case::table_basic( + "| a | b |\n|---|:-:|\n| 1 | 2 |\n", + "<table>\n<thead>\n<tr>\n<th>a</th>\n<th align=\"center\">b</th>\n</tr>\n</thead>\n<tbody>\n<tr>\n<td>1</td>\n<td align=\"center\">2</td>\n</tr>\n</tbody>\n</table>\n", + r#" +cell 0:0 @1:1-1:5 + text "a" @1:3-1:4 +cell 0:1 @1:5-1:10 + text "b" @1:7-1:8 +align none,center @2:1-2:1 +cell 1:0 @3:1-3:5 + text "1" @3:3-3:4 +cell 1:1 @3:5-3:10 + text "2" @3:7-3:8 +"# +)] +#[case::table_no_edge_pipes( + "a|b\n-|-\n1|2|3\n", + "<table>\n<thead>\n<tr>\n<th>a</th>\n<th>b</th>\n</tr>\n</thead>\n<tbody>\n<tr>\n<td>1</td>\n<td>2</td>\n</tr>\n</tbody>\n</table>\n", + r#" +cell 0:0 @1:1-1:2 + text "a" @1:1-1:2 +cell 0:1 @1:2-1:4 + text "b" @1:3-1:4 +align none,none @2:1-2:1 +cell 1:0 @3:1-3:2 + text "1" @3:1-3:2 +cell 1:1 @3:2-3:4 + text "2" @3:3-3:4 +cell 1:2 @3:4-3:6 + text "3" @3:5-3:6 +"# +)] +#[case::table_short_row( + "|a|b|\n|-|-|\n|1|\n", + "<table>\n<thead>\n<tr>\n<th>a</th>\n<th>b</th>\n</tr>\n</thead>\n<tbody>\n<tr>\n<td>1</td>\n<td></td>\n</tr>\n</tbody>\n</table>\n", + r#" +cell 0:0 @1:1-1:3 + text "a" @1:2-1:3 +cell 0:1 @1:3-1:6 + text "b" @1:4-1:5 +align none,none @2:1-2:1 +cell 1:0 @3:1-3:4 + text "1" @3:2-3:3 +"# +)] +#[case::table_escaped_pipe( + "| a \\| b | c |\n|--|--|\n", + "<table>\n<thead>\n<tr>\n<th>a | b</th>\n<th>c</th>\n</tr>\n</thead>\n</table>\n", + r#" +cell 0:0 @1:1-1:10 + text "a | b" @1:3-1:9 +cell 0:1 @1:10-1:15 + text "c" @1:12-1:13 +align none,none @2:1-2:1 +"# +)] +#[case::table_indented( + " | a |\n |---|\n | b |\n", + "<table>\n<thead>\n<tr>\n<th>a</th>\n</tr>\n</thead>\n<tbody>\n<tr>\n<td>b</td>\n</tr>\n</tbody>\n</table>\n", + r#" +cell 0:0 @1:1-1:8 + text "a" @1:5-1:6 +align none @2:1-2:1 +cell 1:0 @3:1-3:8 + text "b" @3:5-3:6 +"# +)] +#[case::table_text_after( + "| a |\n|---|\ntext after\n", + "<table>\n<thead>\n<tr>\n<th>a</th>\n</tr>\n</thead>\n<tbody>\n<tr>\n<td>text after</td>\n</tr>\n</tbody>\n</table>\n", + r#" +cell 0:0 @1:1-1:6 + text "a" @1:3-1:4 +align none @2:1-2:1 +cell 1:0 @3:1-3:11 + text "text after" @3:1-3:11 +"# +)] +#[case::table_in_quote( + "> | a |\n> |---|\n> | b |\n", + "<blockquote>\n<table>\n<thead>\n<tr>\n<th>a</th>\n</tr>\n</thead>\n<tbody>\n<tr>\n<td>b</td>\n</tr>\n</tbody>\n</table>\n</blockquote>\n", + r#" +blockquote @1:1-3:8 + cell 0:0 @1:3-1:8 + text "a" @1:5-1:6 + align none @2:1-2:1 + cell 1:0 @3:3-3:8 + text "b" @3:5-3:6 +"# +)] +#[case::table_after_paragraph( + "x\n| a |\n|---|\n", + "<p>x</p>\n<table>\n<thead>\n<tr>\n<th>a</th>\n</tr>\n</thead>\n</table>\n", + r#" +text "x" @1:1-1:2 +cell 0:0 @2:1-2:6 + text "a" @2:3-2:4 +align none @3:1-3:1 +"# +)] +#[case::table_column_mismatch( + "| a | b |\n|---|\n", + "<p>| a | b |\n|---|</p>\n", + r#"text "| a | b |\n|---|" @1:1-2:6"# +)] +#[case::table_bad_delimiter("|a|\n|:|\n", "<p>|a|\n|:|</p>\n", r#"text "|a|\n|:|" @1:1-2:4"#)] +#[case::table_then_blank_para( + "|a|\n|-|\n\nnext\n", + "<table>\n<thead>\n<tr>\n<th>a</th>\n</tr>\n</thead>\n</table>\n<p>next</p>\n", + r#" +cell 0:0 @1:1-1:4 + text "a" @1:2-1:3 +align none @2:1-2:1 +text "next" @4:1-4:5 +"# +)] +#[case::table_then_quote( + "|a|\n|-|\n> q\n", + "<table>\n<thead>\n<tr>\n<th>a</th>\n</tr>\n</thead>\n</table>\n<blockquote>\n<p>q</p>\n</blockquote>\n", + r#" +cell 0:0 @1:1-1:4 + text "a" @1:2-1:3 +align none @2:1-2:1 +blockquote @3:1-3:4 + text "q" @3:3-3:4 +"# +)] +#[case::table_then_heading( + "|a|\n|-|\n# h\n", + "<table>\n<thead>\n<tr>\n<th>a</th>\n</tr>\n</thead>\n</table>\n<h1>h</h1>\n", + r#" +cell 0:0 @1:1-1:4 + text "a" @1:2-1:3 +align none @2:1-2:1 +heading 1 @3:1-3:4 + text "h" @3:3-3:4 +"# +)] +#[case::table_then_fence( + "|a|\n|-|\n```\nx\n```\n", + "<table>\n<thead>\n<tr>\n<th>a</th>\n</tr>\n</thead>\n</table>\n<pre><code>x\n</code></pre>\n", + r#" +cell 0:0 @1:1-1:4 + text "a" @1:2-1:3 +align none @2:1-2:1 +code "x" fence @3:1-5:4 +"# +)] +#[case::table_then_list( + "|a|\n|-|\n- x\n", + "<table>\n<thead>\n<tr>\n<th>a</th>\n</tr>\n</thead>\n</table>\n<ul>\n<li>x</li>\n</ul>\n", + r#" +cell 0:0 @1:1-1:4 + text "a" @1:2-1:3 +align none @2:1-2:1 +list level=0 index=0 marker=- @3:3-3:4 + text "x" @3:3-3:4 +"# +)] +#[case::table_two( + "|a|\n|-|\n\n|b|\n|-|\n", + "<table>\n<thead>\n<tr>\n<th>a</th>\n</tr>\n</thead>\n</table>\n<table>\n<thead>\n<tr>\n<th>b</th>\n</tr>\n</thead>\n</table>\n", + r#" +cell 0:0 @1:1-1:4 + text "a" @1:2-1:3 +align none @2:1-2:1 +cell 0:0 @4:1-4:4 + text "b" @4:2-4:3 +align none @5:1-5:1 +"# +)] +#[case::table_align_spaces( + "|a|b|\n|:-|-:|\n| x | y|\n", + "<table>\n<thead>\n<tr>\n<th align=\"left\">a</th>\n<th align=\"right\">b</th>\n</tr>\n</thead>\n<tbody>\n<tr>\n<td align=\"left\">x</td>\n<td align=\"right\">y</td>\n</tr>\n</tbody>\n</table>\n", + r#" +cell 0:0 @1:1-1:3 + text "a" @1:2-1:3 +cell 0:1 @1:3-1:6 + text "b" @1:4-1:5 +align left,right @2:1-2:1 +cell 1:0 @3:1-3:7 + text "x" @3:4-3:5 +cell 1:1 @3:7-3:12 + text "y" @3:10-3:11 +"# +)] +#[case::table_indented_code_row( + "|a|\n|-|\n|b|\n |c|\n", + "<table>\n<thead>\n<tr>\n<th>a</th>\n</tr>\n</thead>\n<tbody>\n<tr>\n<td>b</td>\n</tr>\n</tbody>\n</table>\n<pre><code>|c|\n</code></pre>\n", + r#" +cell 0:0 @1:1-1:4 + text "a" @1:2-1:3 +align none @2:1-2:1 +cell 1:0 @3:1-3:4 + text "b" @3:2-3:3 +code "|c|" @4:1-4:8 +"# +)] +#[case::table_no_body( + "| a |\n| - |\n", + "<table>\n<thead>\n<tr>\n<th>a</th>\n</tr>\n</thead>\n</table>\n", + r#" +cell 0:0 @1:1-1:6 + text "a" @1:3-1:4 +align none @2:1-2:1 +"# +)] +#[case::table_empty_cells( + "||\n|-|\n", + "<table>\n<thead>\n<tr>\n<th></th>\n</tr>\n</thead>\n</table>\n", + r#" +cell 0:0 @1:1-1:3 +align none @2:1-2:1 +"# +)] +#[case::table_lone_pipe_row( + "|a|\n|-|\n|\n", + "<table>\n<thead>\n<tr>\n<th>a</th>\n</tr>\n</thead>\n<tbody>\n<tr>\n<td></td>\n</tr>\n</tbody>\n</table>\n", + r#" +cell 0:0 @1:1-1:4 + text "a" @1:2-1:3 +align none @2:1-2:1 +cell 1:0 @3:1-3:2 +"# +)] +#[case::table_no_trailing_pipe( + "| a | b\n|-|-\n| c | d\n", + "<table>\n<thead>\n<tr>\n<th>a</th>\n<th>b</th>\n</tr>\n</thead>\n<tbody>\n<tr>\n<td>c</td>\n<td>d</td>\n</tr>\n</tbody>\n</table>\n", + r#" +cell 0:0 @1:1-1:5 + text "a" @1:3-1:4 +cell 0:1 @1:5-1:8 + text "b" @1:7-1:8 +align none,none @2:1-2:1 +cell 1:0 @3:1-3:5 + text "c" @3:3-3:4 +cell 1:1 @3:5-3:8 + text "d" @3:7-3:8 +"# +)] +#[case::table_multibyte( + "| あ | い |\n|---|---|\n| う | え |\n", + "<table>\n<thead>\n<tr>\n<th>あ</th>\n<th>い</th>\n</tr>\n</thead>\n<tbody>\n<tr>\n<td>う</td>\n<td>え</td>\n</tr>\n</tbody>\n</table>\n", + r#" +cell 0:0 @1:1-1:7 + text "あ" @1:3-1:6 +cell 0:1 @1:7-1:14 + text "い" @1:9-1:12 +align none,none @2:1-2:1 +cell 1:0 @3:1-3:7 + text "う" @3:3-3:6 +cell 1:1 @3:7-3:14 + text "え" @3:9-3:12 +"# +)] +#[case::not_table_setext( + "Title\n---\n", + "<h2>Title</h2>\n", + r#" +heading 2 @1:1-2:4 + text "Title" @1:1-1:6 +"# +)] +#[case::not_table_no_pipe_delim( + "a|b\n---\n", + "<h2>a|b</h2>\n", + r#" +heading 2 @1:1-2:4 + text "a|b" @1:1-1:4 +"# +)] +#[case::table_header_no_pipe( + "a\n|-|\n", + "<table>\n<thead>\n<tr>\n<th>a</th>\n</tr>\n</thead>\n</table>\n", + r#" +cell 0:0 @1:1-1:2 + text "a" @1:1-1:2 +align none @2:1-2:1 +"# +)] +#[case::table_one_col_dash( + "|a|\n-\n", + "<h2>|a|</h2>\n", + r#" +heading 2 @1:1-2:2 + text "|a|" @1:1-1:4 +"# +)] +#[case::inline_escape("a\\*b", "<p>a*b</p>", r#"text "a*b" @1:1-1:5"#)] +#[case::inline_escape_nonpunct("a\\qb", "<p>a\\qb</p>", r#"text "a\\qb" @1:1-1:5"#)] +#[case::inline_entities( + "a&amp;b &copy; &#35; &#x41; &unknown; &", + "<p>a&amp;b © # A &amp;unknown; &amp;</p>", + r#"text "a&b © # A &unknown; &" @1:1-1:40"# +)] +#[case::inline_emphasis_all( + "a *b* **c** _d_ __e__ ***f***", + "<p>a <em>b</em> <strong>c</strong> <em>d</em> <strong>e</strong> <em><strong>f</strong></em></p>", + r#" +text "a " @1:1-1:3 +emphasis @1:3-1:6 + text "b" @1:4-1:5 +text " " @1:6-1:7 +strong @1:7-1:12 + text "c" @1:9-1:10 +text " " @1:12-1:13 +emphasis @1:13-1:16 + text "d" @1:14-1:15 +text " " @1:16-1:17 +strong @1:17-1:22 + text "e" @1:19-1:20 +text " " @1:22-1:23 +emphasis @1:23-1:30 + strong @1:24-1:29 + text "f" @1:26-1:27 +"# +)] +#[case::inline_code( + "`code` `` a`b `` ` a `", + "<p><code>code</code> <code>a`b</code> <code>a</code></p>", + r#" +code_inline "code" @1:1-1:7 +text " " @1:7-1:8 +code_inline "a`b" @1:8-1:17 +text " " @1:17-1:18 +code_inline "a" @1:18-1:23 +"# +)] +#[case::inline_code_unclosed("`a ``b`", "<p><code>a ``b</code></p>", r#"code_inline "a ``b" @1:1-1:8"#)] +#[case::inline_hard_break_spaces( + "a \nb", + "<p>a<br />\nb</p>", + r#" +text "a" @1:1-1:2 +break @1:2-2:1 +text "b" @2:1-2:2 +"# +)] +#[case::inline_hard_break_backslash( + "a\\\nb", + "<p>a<br />\nb</p>", + r#" +text "a" @1:1-1:2 +break @1:2-2:1 +text "b" @2:1-2:2 +"# +)] +#[case::inline_soft_break("a\nb", "<p>a\nb</p>", r#"text "a\nb" @1:1-2:2"#)] +#[case::inline_soft_break_trailing_space("a \nb", "<p>a\nb</p>", r#"text "a\nb" @1:1-2:2"#)] +#[case::inline_emphasis_multiline( + "*a\nb*", + "<p><em>a\nb</em></p>", + r#" +emphasis @1:1-2:3 + text "a\nb" @1:2-2:2 +"# +)] +#[case::inline_strike( + "a ~b~ ~~c~~ ~~~d~~~", + "<p>a <del>b</del> <del>c</del> ~~~d~~~</p>", + r#" +text "a " @1:1-1:3 +delete @1:3-1:6 + text "b" @1:4-1:5 +text " " @1:6-1:7 +delete @1:7-1:12 + text "c" @1:9-1:10 +text " ~~~d~~~" @1:12-1:20 +"# +)] +#[case::inline_math( + "$x$ $$y$$ $ a", + "<p><code class=\"language-math math-inline\">x</code> <code class=\"language-math math-inline\">y</code> $ a</p>", + r#" +math_inline "x" @1:1-1:4 +text " " @1:4-1:5 +math_inline "y" @1:5-1:10 +text " $ a" @1:10-1:14 +"# +)] +#[case::inline_autolinks( + "<http://a.b> <a@b.c> <span> <!-- c --> <br/>", + "<p><a href=\"http://a.b\">http://a.b</a> <a href=\"mailto:a@b.c\">a@b.c</a> <span> <!-- c --> <br/></p>", + r#" +link "http://a.b" @1:1-1:13 + text "http://a.b" @1:2-1:12 +text " " @1:13-1:14 +link "mailto:a@b.c" @1:14-1:21 + text "a@b.c" @1:15-1:20 +text " " @1:21-1:22 +html "<span>" @1:22-1:28 +text " " @1:28-1:29 +html "<!-- c -->" @1:29-1:39 +text " " @1:39-1:40 +html "<br/>" @1:40-1:45 +"# +)] +#[case::inline_links( + "[a](http://x \"t\") [b][c] [d][] [e] ![f](g) ![h][i]", + "<p><a href=\"http://x\" title=\"t\">a</a> [b][c] [d][] [e] <img src=\"g\" alt=\"f\" /> ![h][i]</p>", + r#" +link "http://x" title="t" @1:1-1:18 + text "a" @1:2-1:3 +text " [b][c] [d][] [e] " @1:18-1:36 +image "g" alt="f" @1:36-1:43 +text " ![h][i]" @1:43-1:51 +"# +)] +#[case::inline_link_forms( + "[a](<b c> 'x') [d]( e ) [f](g (h)) [i](j \"k\\\"l\") [m](n&amp;o)", + "<p><a href=\"b%20c\" title=\"x\">a</a> <a href=\"e\">d</a> <a href=\"g\" title=\"h\">f</a> <a href=\"j\" title=\"k&quot;l\">i</a> <a href=\"n&amp;o\">m</a></p>", + r#" +link "b c" title="x" @1:1-1:15 + text "a" @1:2-1:3 +text " " @1:15-1:16 +link "e" @1:16-1:24 + text "d" @1:17-1:18 +text " " @1:24-1:25 +link "g" title="h" @1:25-1:35 + text "f" @1:26-1:27 +text " " @1:35-1:36 +link "j" title="k\"l" @1:36-1:49 + text "i" @1:37-1:38 +text " " @1:49-1:50 +link "n&o" @1:50-1:62 + text "m" @1:51-1:52 +"# +)] +#[case::inline_link_multiline( + "[a](b\n\"t\") [c](d", + "<p><a href=\"b\" title=\"t\">a</a> [c](d</p>", + r#" +link "b" title="t" @1:1-2:5 + text "a" @1:2-1:3 +text " [c](d" @2:5-2:11 +"# +)] +#[case::inline_image_alt( + "![a *b* `c`](d) ![e ![f](g)](h)", + "<p><img src=\"d\" alt=\"a b c\" /> <img src=\"h\" alt=\"e f\" /></p>", + r#" +image "d" alt="a b c" @1:1-1:16 +text " " @1:16-1:17 +image "h" alt="e f" @1:17-1:32 +"# +)] +#[case::inline_link_in_link( + "[a [b](c) d](e)", + "<p>[a <a href=\"c\">b</a> d](e)</p>", + r#" +text "[a " @1:1-1:4 +link "c" @1:4-1:10 + text "b" @1:5-1:6 +text " d](e)" @1:10-1:16 +"# +)] +#[case::inline_emphasis_link( + "*[a](b)* **[c][a]**", + "<p><em><a href=\"b\">a</a></em> <strong>[c][a]</strong></p>", + r#" +emphasis @1:1-1:9 + link "b" @1:2-1:8 + text "a" @1:3-1:4 +text " " @1:9-1:10 +strong @1:10-1:20 + text "[c][a]" @1:12-1:18 +"# +)] +#[case::inline_html( + "<a href=\"x\">t</a> <a\nhref=x>", + "<p><a href=\"x\">t</a> <a\nhref=x></p>", + r#" +html "<a href=\"x\">" @1:1-1:13 +text "t" @1:13-1:14 +html "</a>" @1:14-1:18 +text " " @1:18-1:19 +html "<a\nhref=x>" @1:19-2:8 +"# +)] +#[case::inline_html_misc( + "<http://a b> <a+b@c> <ab> </a> <?x?> <![CDATA[x]]> <!X y>", + "<p>&lt;<a href=\"http://a\">http://a</a> b&gt; <a href=\"mailto:a+b@c\">a+b@c</a> <ab> </a> <?x?> <![CDATA[x]]> <!X y></p>", + r#" +text "<" @1:1-1:2 +link "http://a" @1:2-1:10 + text "http://a" @1:2-1:10 +text " b> " @1:10-1:14 +link "mailto:a+b@c" @1:14-1:21 + text "a+b@c" @1:15-1:20 +text " " @1:21-1:22 +html "<ab>" @1:22-1:26 +text " " @1:26-1:27 +html "</a>" @1:27-1:31 +text " " @1:31-1:32 +html "<?x?>" @1:32-1:37 +text " " @1:37-1:38 +html "<![CDATA[x]]>" @1:38-1:51 +text " " @1:51-1:52 +html "<!X y>" @1:52-1:58 +"# +)] +#[case::inline_nested_emphasis( + "*a **b** c* **a *b* c**", + "<p><em>a <strong>b</strong> c</em> <strong>a <em>b</em> c</strong></p>", + r#" +emphasis @1:1-1:12 + text "a " @1:2-1:4 + strong @1:4-1:9 + text "b" @1:6-1:7 + text " c" @1:9-1:11 +text " " @1:12-1:13 +strong @1:13-1:24 + text "a " @1:15-1:17 + emphasis @1:17-1:20 + text "b" @1:18-1:19 + text " c" @1:20-1:22 +"# +)] +#[case::inline_intraword( + "a_b_c a*b*c _a_b", + "<p>a_b_c a<em>b</em>c _a_b</p>", + r#" +text "a_b_c a" @1:1-1:8 +emphasis @1:8-1:11 + text "b" @1:9-1:10 +text "c _a_b" @1:11-1:17 +"# +)] +#[case::inline_unmatched( + "*a **b _c ~d [e ![f", + "<p>*a **b _c ~d [e ![f</p>", + r#"text "*a **b _c ~d [e ![f" @1:1-1:20"# +)] +#[case::inline_rule_of_three( + "*foo**bar**baz* ***a** b*", + "<p><em>foo<strong>bar</strong>baz</em> <em><strong>a</strong> b</em></p>", + r#" +emphasis @1:1-1:16 + text "foo" @1:2-1:5 + strong @1:5-1:12 + text "bar" @1:7-1:10 + text "baz" @1:12-1:15 +text " " @1:16-1:17 +emphasis @1:17-1:26 + strong @1:18-1:23 + text "a" @1:20-1:21 + text " b" @1:23-1:25 +"# +)] +#[case::inline_heading( + "# a *b* `c`\n", + "<h1>a <em>b</em> <code>c</code></h1>\n", + r#" +heading 1 @1:1-1:12 + text "a " @1:3-1:5 + emphasis @1:5-1:8 + text "b" @1:6-1:7 + text " " @1:8-1:9 + code_inline "c" @1:9-1:12 +"# +)] +#[case::inline_setext_heading( + "a *b*\nc\n===\n", + "<h1>a <em>b</em>\nc</h1>\n", + r#" +heading 1 @1:1-3:4 + text "a " @1:1-1:3 + emphasis @1:3-1:6 + text "b" @1:4-1:5 + text "\nc" @1:6-2:2 +"# +)] +#[case::inline_list_item( + "- a *b*\n c **d**\n", + "<ul>\n<li>a <em>b</em>\nc <strong>d</strong></li>\n</ul>\n", + r#" +list level=0 index=0 marker=- @1:3-2:10 + text "a " @1:3-1:5 + emphasis @1:5-1:8 + text "b" @1:6-1:7 + text "\nc " @1:8-2:5 + strong @2:5-2:10 + text "d" @2:7-2:8 +"# +)] +#[case::inline_table_cell( + "| a *b* | `c\\|d` |\n|-|-|\n", + "<table>\n<thead>\n<tr>\n<th>a <em>b</em></th>\n<th><code>c|d</code></th>\n</tr>\n</thead>\n</table>\n", + r#" +cell 0:0 @1:1-1:9 + text "a " @1:3-1:5 + emphasis @1:5-1:8 + text "b" @1:6-1:7 +cell 0:1 @1:9-1:19 + code_inline "c|d" @1:11-1:17 +align none,none @2:1-2:1 +"# +)] +#[case::inline_multibyte( + "あ*い*う `え` [お](か)", + "<p>あ<em>い</em>う <code>え</code> <a href=\"%E3%81%8B\">お</a></p>", + r#" +text "あ" @1:1-1:4 +emphasis @1:4-1:9 + text "い" @1:5-1:8 +text "う " @1:9-1:13 +code_inline "え" @1:13-1:18 +text " " @1:18-1:19 +link "か" @1:19-1:29 + text "お" @1:20-1:23 +"# +)] +#[case::inline_crlf_break( + "a \r\nb\r\n", + "<p>a<br />\r\nb</p>\r\n", + r#" +text "a" @1:1-1:2 +break @1:2-2:1 +text "b" @2:1-2:2 +"# +)] +#[case::math_multiline( + "x$\nx$", + "<p>x<code class=\"language-math math-inline\"> x</code></p>", + r#" +text "x" @1:1-1:2 +math_inline "\nx" @1:2-2:3 +"# +)] +#[case::decl_email( + "x<!a@b.co>", + "<p>x<!a@b.co></p>", + r#" +text "x" @1:1-1:2 +html "<!a@b.co>" @1:2-1:11 +"# +)] +#[case::code_multiline( + "x`\nx`", + "<p>x<code> x</code></p>", + r#" +text "x" @1:1-1:2 +code_inline "\nx" @1:2-2:3 +"# +)] +#[case::def_basic("[a]: b", "", r#"definition "a" "b" label="a" @1:1-1:7"#)] +#[case::def_title("[a]: <b c> 'T'", "", r#"definition "a" "b c" label="a" title="T" @1:1-1:15"#)] +#[case::def_next_lines("[a]:\nb\n\"t\"", "", r#"definition "a" "b" label="a" title="t" @1:1-3:4"#)] +#[case::def_title_junk_next_line( + "[a]: b\n\"t\" x", + "<p>&quot;t&quot; x</p>", + r#" +definition "a" "b" label="a" @1:1-1:7 +text "\"t\" x" @2:1-2:6 +"# +)] +#[case::def_title_junk_same_line( + "[a]: b \"t\" x", + "<p>[a]: b &quot;t&quot; x</p>", + r#"text "[a]: b \"t\" x" @1:1-1:13"# +)] +#[case::def_then_paragraph( + "[a]: b\nc", + "<p>c</p>", + r#" +definition "a" "b" label="a" @1:1-1:7 +text "c" @2:1-2:2 +"# +)] +#[case::def_two_then_text( + "[a]: b\n[c]: d\ntext", + "<p>text</p>", + r#" +definition "a" "b" label="a" @1:1-1:7 +definition "c" "d" label="c" @2:1-2:7 +text "text" @3:1-3:5 +"# +)] +#[case::def_not_at_start("text\n[a]: b", "<p>text\n[a]: b</p>", r#"text "text\n[a]: b" @1:1-2:7"#)] +#[case::def_then_setext_equals( + "[a]: b\n=== ", + "<p>===</p>", + r#" +definition "a" "b" label="a" @1:1-1:7 +text "===" @2:1-2:4 +"# +)] +#[case::def_then_setext_dashes( + "[a]: b\n---", + "<hr />", + r#" +definition "a" "b" label="a" @1:1-1:7 +hr marker='-' @2:1-2:4 +"# +)] +#[case::def_indent1(" [a]: b", "", r#"definition "a" "b" label="a" @1:1-1:8"#)] +#[case::def_indent3(" [a]: b", "", r#"definition "a" "b" label="a" @1:1-1:10"#)] +#[case::def_indent4(" [a]: b", "<pre><code>[a]: b\n</code></pre>", r#"code "[a]: b" @1:1-1:11"#)] +#[case::def_in_quote( + "> [a]: b", + "<blockquote>\n</blockquote>", + r#" +blockquote @1:1-1:9 + definition "a" "b" label="a" @1:3-1:9 +"# +)] +#[case::def_in_list( + "- [a]: b", + "<ul>\n<li></li>\n</ul>", + r#" +list level=0 index=0 marker=- @1:3-1:9 + definition "a" "b" label="a" @1:3-1:9 +"# +)] +#[case::def_no_dest("[a]:", "<p>[a]:</p>", r#"text "[a]:" @1:1-1:5"#)] +#[case::def_empty_label("[]: b", "<p>[]: b</p>", r#"text "[]: b" @1:1-1:6"#)] +#[case::def_blank_label("[ ]: b", "<p>[ ]: b</p>", r#"text "[ ]: b" @1:1-1:7"#)] +#[case::def_escaped_label("[a\\]b]: c", "", r#"definition "a\\]b" "c" label="a]b" @1:1-1:10"#)] +#[case::def_multiline_title("[a]: b \"t\nt\"", "", r#"definition "a" "b" label="a" title="t\nt" @1:1-2:3"#)] +#[case::def_title_indented("[a]: b\n \"t\"", "", r#"definition "a" "b" label="a" title="t" @1:1-2:6"#)] +#[case::def_no_space("[a]:b", "", r#"definition "a" "b" label="a" @1:1-1:6"#)] +#[case::def_dest_junk("[a]: b c", "<p>[a]: b c</p>", r#"text "[a]: b c" @1:1-1:9"#)] +#[case::def_duplicate_labels( + "[A b]: c\n[a b]: d", + "", + r#" +definition "a b" "c" label="A b" @1:1-1:10 +definition "a b" "d" label="a b" @2:1-2:9 +"# +)] +#[case::def_empty_angle("[a]: <>", "", r#"definition "a" "" label="a" @1:1-1:8"#)] +#[case::def_quote_in_dest("[a]: 'x", "", r#"definition "a" "'x" label="a" @1:1-1:8"#)] +#[case::def_use_shortcut( + "[a]: /u\n\n[a] [a][] [b][a] [a][b] [A b]", + "<p><a href=\"/u\">a</a> <a href=\"/u\">a</a> <a href=\"/u\">b</a> [a][b] [A b]</p>", + r#" +definition "a" "/u" label="a" @1:1-1:8 +link_ref "a" label="a" @3:1-3:4 + text "a" @3:2-3:3 +text " " @3:4-3:5 +link_ref "a" label="a" @3:5-3:10 + text "a" @3:6-3:7 +text " " @3:10-3:11 +link_ref "a" label="a" @3:11-3:17 + text "b" @3:12-3:13 +text " [a][b] [A b]" @3:17-3:30 +"# +)] +#[case::def_use_normalized( + "[ab cd]: /u\n\n[AB CD] [ab\ncd]", + "<p><a href=\"/u\">AB CD</a> <a href=\"/u\">ab\ncd</a></p>", + r#" +definition "ab cd" "/u" label="ab cd" @1:1-1:12 +link_ref "ab cd" label="AB CD" @3:1-3:10 + text "AB CD" @3:2-3:9 +text " " @3:10-3:11 +link_ref "ab cd" label="ab\ncd" @3:11-4:4 + text "ab\ncd" @3:12-4:3 +"# +)] +#[case::def_use_before( + "[a] and [b][a]\n\n[a]: /u \"T\"", + "<p><a href=\"/u\" title=\"T\">a</a> and <a href=\"/u\" title=\"T\">b</a></p>\n", + r#" +link_ref "a" label="a" @1:1-1:4 + text "a" @1:2-1:3 +text " and " @1:4-1:9 +link_ref "a" label="a" @1:9-1:15 + text "b" @1:10-1:11 +definition "a" "/u" label="a" title="T" @3:1-3:12 +"# +)] +#[case::def_use_in_link_text( + "[a]: b\n\n[a](x) [a]", + "<p><a href=\"x\">a</a> <a href=\"b\">a</a></p>", + r#" +definition "a" "b" label="a" @1:1-1:7 +link "x" @3:1-3:7 + text "a" @3:2-3:3 +text " " @3:7-3:8 +link_ref "a" label="a" @3:8-3:11 + text "a" @3:9-3:10 +"# +)] +#[case::def_use_image( + "[x]: /u\n\n![x] ![y][x] ![x][]", + "<p><img src=\"/u\" alt=\"x\" /> <img src=\"/u\" alt=\"y\" /> <img src=\"/u\" alt=\"x\" /></p>", + r#" +definition "x" "/u" label="x" @1:1-1:8 +image_ref "x" alt="x" label="x" @3:1-3:5 +text " " @3:5-3:6 +image_ref "x" alt="y" label="x" @3:6-3:13 +text " " @3:13-3:14 +image_ref "x" alt="x" label="x" @3:14-3:20 +"# +)] +#[case::def_link_in_emphasis( + "[a]: b\n\n*[a]* **[c][a]**", + "<p><em><a href=\"b\">a</a></em> <strong><a href=\"b\">c</a></strong></p>", + r#" +definition "a" "b" label="a" @1:1-1:7 +emphasis @3:1-3:6 + link_ref "a" label="a" @3:2-3:5 + text "a" @3:3-3:4 +text " " @3:6-3:7 +strong @3:7-3:17 + link_ref "a" label="a" @3:9-3:15 + text "c" @3:10-3:11 +"# +)] +#[case::fn_basic( + "[^a]: b", + "", + r#" +footnote "a" @1:1-1:8 + text "b" @1:7-1:8 +"# +)] +#[case::fn_lazy( + "[^a]: b\nc", + "", + r#" +footnote "a" @1:1-2:2 + text "b\nc" @1:7-2:2 +"# +)] +#[case::fn_indented_continuation( + "[^a]: b\n c", + "", + r#" +footnote "a" @1:1-2:6 + text "b\nc" @1:7-2:6 +"# +)] +#[case::fn_two_paragraphs( + "[^a]: b\n\n c", + "", + r#" +footnote "a" @1:1-3:6 + text "b" @1:7-1:8 + text "c" @3:5-3:6 +"# +)] +#[case::fn_two_paragraphs_then_text( + "[^a]: b\n\n c\n\nd", + "<p>d</p>", + r#" +footnote "a" @1:1-4:1 + text "b" @1:7-1:8 + text "c" @3:5-3:6 +text "d" @5:1-5:2 +"# +)] +#[case::fn_two_space_continuation( + "[^a]: b\n c", + "", + r#" +footnote "a" @1:1-2:4 + text "b\nc" @1:7-2:4 +"# +)] +#[case::fn_empty("[^a]:", "", r#"footnote "a" @1:1-1:6"#)] +#[case::fn_empty_space("[^a]: ", "", r#"footnote "a" @1:1-1:7"#)] +#[case::fn_content_next_line( + "[^a]:\n b", + "", + r#" +footnote "a" @1:1-2:6 + text "b" @2:5-2:6 +"# +)] +#[case::fn_two_adjacent( + "[^a]: b\n[^c]: d", + "", + r#" +footnote "a" @1:1-1:8 + text "b" @1:7-1:8 +footnote "c" @2:1-2:8 + text "d" @2:7-2:8 +"# +)] +#[case::fn_two_separated( + "[^a]: b\n\n[^c]: d", + "", + r#" +footnote "a" @1:1-2:1 + text "b" @1:7-1:8 +footnote "c" @3:1-3:8 + text "d" @3:7-3:8 +"# +)] +#[case::fn_label_space("[^ a]: b", "", r#"definition "^ a" "b" label="^ a" @1:1-1:9"#)] +#[case::fn_label_inner_space("[^a b]: b", "", r#"definition "^a b" "b" label="^a b" @1:1-1:10"#)] +#[case::fn_label_empty("[^]: b", "", r#"definition "^" "b" label="^" @1:1-1:7"#)] +#[case::fn_indent1( + " [^a]: b", + "", + r#" +footnote "a" @1:1-1:9 + text "b" @1:8-1:9 +"# +)] +#[case::fn_indent4(" [^a]: b", "<pre><code>[^a]: b\n</code></pre>", r#"code "[^a]: b" @1:1-1:12"#)] +#[case::fn_in_quote( + "> [^a]: b", + "<blockquote>\n</blockquote>", + r#" +blockquote @1:1-1:10 + footnote "a" @1:3-1:10 + text "b" @1:9-1:10 +"# +)] +#[case::fn_in_list( + "- [^a]: b", + "<ul>\n<li></li>\n</ul>", + r#" +list level=0 index=0 marker=- @1:3-1:10 + footnote "a" @1:3-1:10 + text "b" @1:9-1:10 +"# +)] +#[case::fn_then_quote( + "[^a]: b\n> q", + "<blockquote>\n<p>q</p>\n</blockquote>", + r#" +footnote "a" @1:1-1:8 + text "b" @1:7-1:8 +blockquote @2:1-2:4 + text "q" @2:3-2:4 +"# +)] +#[case::fn_then_list( + "[^a]: b\n- x", + "<ul>\n<li>x</li>\n</ul>", + r#" +footnote "a" @1:1-1:8 + text "b" @1:7-1:8 +list level=0 index=0 marker=- @2:3-2:4 + text "x" @2:3-2:4 +"# +)] +#[case::fn_heading( + "[^a]: # h", + "", + r#" +footnote "a" @1:1-1:10 + heading 1 @1:7-1:10 + text "h" @1:9-1:10 +"# +)] +#[case::fn_list( + "[^a]: - x\n - y", + "", + r#" +footnote "a" @1:1-2:8 + list level=0 index=0 marker=- @1:9-1:10 + text "x" @1:9-1:10 + list level=0 index=1 marker=- @2:7-2:8 + text "y" @2:7-2:8 +"# +)] +#[case::fn_fence( + "[^a]: b\n ```\n x\n ```", + "", + r#" +footnote "a" @1:1-4:8 + text "b" @1:7-1:8 + code "x" fence @2:5-4:8 +"# +)] +#[case::fn_ref_case( + "[^A]: b\n\nx[^a]", + "<p>x<sup><a href=\"#user-content-fn-a\" id=\"user-content-fnref-a\" data-footnote-ref=\"\" aria-describedby=\"footnote-label\">1</a></sup></p>\n<section data-footnotes=\"\" class=\"footnotes\"><h2 id=\"footnote-label\" class=\"sr-only\">Footnotes</h2>\n<ol>\n<li id=\"user-content-fn-a\">\n<p>b <a href=\"#user-content-fnref-a\" data-footnote-backref=\"\" aria-label=\"Back to content\" class=\"data-footnote-backref\">↩</a></p>\n</li>\n</ol>\n</section>\n", + r#" +footnote "a" @1:1-2:1 + text "b" @1:7-1:8 +text "x" @3:1-3:2 +footnote_ref "a" label="a" @3:2-3:6 +"# +)] +#[case::fn_no_space( + "[^a]:b", + "", + r#" +footnote "a" @1:1-1:7 + text "b" @1:6-1:7 +"# +)] +#[case::fn_interrupts_paragraph( + "text\n[^a]: b", + "<p>text</p>\n", + r#" +text "text" @1:1-1:5 +footnote "a" @2:1-2:8 + text "b" @2:7-2:8 +"# +)] +#[case::fn_then_dashes( + "[^a]: b\n---", + "<hr />", + r#" +footnote "a" @1:1-1:8 + text "b" @1:7-1:8 +hr marker='-' @2:1-2:4 +"# +)] +#[case::fn_then_equals( + "[^a]: b\n===", + "", + r#" +footnote "a" @1:1-2:4 + text "b\n===" @1:7-2:4 +"# +)] +#[case::fn_ref_basic( + "[^a]\n\n[^a]: note", + "<p><sup><a href=\"#user-content-fn-a\" id=\"user-content-fnref-a\" data-footnote-ref=\"\" aria-describedby=\"footnote-label\">1</a></sup></p>\n<section data-footnotes=\"\" class=\"footnotes\"><h2 id=\"footnote-label\" class=\"sr-only\">Footnotes</h2>\n<ol>\n<li id=\"user-content-fn-a\">\n<p>note <a href=\"#user-content-fnref-a\" data-footnote-backref=\"\" aria-label=\"Back to content\" class=\"data-footnote-backref\">↩</a></p>\n</li>\n</ol>\n</section>\n", + r#" +footnote_ref "a" label="a" @1:1-1:5 +footnote "a" @3:1-3:11 + text "note" @3:7-3:11 +"# +)] +#[case::fn_ref_no_def("[^b] no def", "<p>[^b] no def</p>", r#"text "[^b] no def" @1:1-1:12"#)] +#[case::fn_ref_inline( + "x[^a] and [^a]!\n\n[^a]: note", + "<p>x<sup><a href=\"#user-content-fn-a\" id=\"user-content-fnref-a\" data-footnote-ref=\"\" aria-describedby=\"footnote-label\">1</a></sup> and <sup><a href=\"#user-content-fn-a\" id=\"user-content-fnref-a-2\" data-footnote-ref=\"\" aria-describedby=\"footnote-label\">1</a></sup>!</p>\n<section data-footnotes=\"\" class=\"footnotes\"><h2 id=\"footnote-label\" class=\"sr-only\">Footnotes</h2>\n<ol>\n<li id=\"user-content-fn-a\">\n<p>note <a href=\"#user-content-fnref-a\" data-footnote-backref=\"\" aria-label=\"Back to content\" class=\"data-footnote-backref\">↩</a> <a href=\"#user-content-fnref-a-2\" data-footnote-backref=\"\" aria-label=\"Back to content\" class=\"data-footnote-backref\">↩<sup>2</sup></a></p>\n</li>\n</ol>\n</section>\n", + r#" +text "x" @1:1-1:2 +footnote_ref "a" label="a" @1:2-1:6 +text " and " @1:6-1:11 +footnote_ref "a" label="a" @1:11-1:15 +text "!" @1:15-1:16 +footnote "a" @3:1-3:11 + text "note" @3:7-3:11 +"# +)] +#[case::fn_ref_in_link_text( + "[a[^a]](b)\n\n[^a]: note", + "<p>[a<sup><a href=\"#user-content-fn-a\" id=\"user-content-fnref-a\" data-footnote-ref=\"\" aria-describedby=\"footnote-label\">1</a></sup>](b)</p>\n<section data-footnotes=\"\" class=\"footnotes\"><h2 id=\"footnote-label\" class=\"sr-only\">Footnotes</h2>\n<ol>\n<li id=\"user-content-fn-a\">\n<p>note <a href=\"#user-content-fnref-a\" data-footnote-backref=\"\" aria-label=\"Back to content\" class=\"data-footnote-backref\">↩</a></p>\n</li>\n</ol>\n</section>\n", + r#" +text "[a" @1:1-1:3 +footnote_ref "a" label="a" @1:3-1:7 +text "](b)" @1:7-1:11 +footnote "a" @3:1-3:11 + text "note" @3:7-3:11 +"# +)] +#[case::html_div( + "<div>\nx\n</div>\n\ny", + "<div>\nx\n</div>\n<p>y</p>", + r#" +html "<div>\nx\n</div>" @1:1-3:7 +text "y" @5:1-5:2 +"# +)] +#[case::html_indented(" <div>\n x\n", " <div>\n x\n", r#"html " <div>\n x" @1:1-2:4"#)] +#[case::html_oneline("<div>x</div>", "<div>x</div>", r#"html "<div>x</div>" @1:1-1:13"#)] +#[case::html_script( + "<script>\nx\n\ny\n</script>\nz", + "&lt;script>\nx\n\ny\n&lt;/script>\n<p>z</p>", + r#" +html "<script>\nx\n\ny\n</script>" @1:1-5:10 +text "z" @6:1-6:2 +"# +)] +#[case::html_pre( + "<pre>\na\n\nb</pre>\nc", + "<pre>\na\n\nb</pre>\n<p>c</p>", + r#" +html "<pre>\na\n\nb</pre>" @1:1-4:8 +text "c" @5:1-5:2 +"# +)] +#[case::html_style_inline_end( + "<style>x</style> y\nz", + "&lt;style>x&lt;/style> y\n<p>z</p>", + r#" +html "<style>x</style> y" @1:1-1:19 +text "z" @2:1-2:2 +"# +)] +#[case::html_comment( + "<!-- c\n\n d -->\nx", + "<!-- c\n\n d -->\n<p>x</p>", + r#" +html "<!-- c\n\n d -->" @1:1-3:7 +text "x" @4:1-4:2 +"# +)] +#[case::html_instruction( + "<?php\nx ?>\ny", + "<?php\nx ?>\n<p>y</p>", + r#" +html "<?php\nx ?>" @1:1-2:5 +text "y" @3:1-3:2 +"# +)] +#[case::html_declaration( + "<!DOCTYPE html>\nx", + "<!DOCTYPE html>\n<p>x</p>", + r#" +html "<!DOCTYPE html>" @1:1-1:16 +text "x" @2:1-2:2 +"# +)] +#[case::html_cdata( + "<![CDATA[\n\nx]]>\ny", + "<![CDATA[\n\nx]]>\n<p>y</p>", + r#" +html "<![CDATA[\n\nx]]>" @1:1-3:5 +text "y" @4:1-4:2 +"# +)] +#[case::html_complete_tag("<a href=\"x\">\ny", "<a href=\"x\">\ny", r#"html "<a href=\"x\">\ny" @1:1-2:2"#)] +#[case::html_tag_then_text( + "<a href=\"x\">y", + "<p><a href=\"x\">y</p>", + r#" +html "<a href=\"x\">" @1:1-1:13 +text "y" @1:13-1:14 +"# +)] +#[case::html_interrupt_basic( + "text\n<div>", + "<p>text</p>\n<div>", + r#" +text "text" @1:1-1:5 +html "<div>" @2:1-2:6 +"# +)] +#[case::html_no_interrupt_complete( + "text\n<a href=\"x\">", + "<p>text\n<a href=\"x\"></p>", + r#" +text "text\n" @1:1-2:1 +html "<a href=\"x\">" @2:1-2:13 +"# +)] +#[case::html_custom_element("<x-y>\nz", "<x-y>\nz", r#"html "<x-y>\nz" @1:1-2:2"#)] +#[case::html_closing("</div>\nz", "</div>\nz", r#"html "</div>\nz" @1:1-2:2"#)] +#[case::html_multiline_open( + "<div\nclass=\"a\">\nz", + "<div\nclass=\"a\">\nz", + r#"html "<div\nclass=\"a\">\nz" @1:1-3:2"# +)] +#[case::html_in_quote( + "> <div>\n> x", + "<blockquote>\n<div>\nx\n</blockquote>", + r#" +blockquote @1:1-2:4 + html "<div>\nx" @1:3-2:4 +"# +)] +#[case::html_in_list( + "- <div>\n x", + "<ul>\n<li>\n<div>\nx\n</li>\n</ul>", + r#" +list level=0 index=0 marker=- @1:3-2:4 + html "<div>\nx" @1:3-2:4 +"# +)] +#[case::html_uppercase("<DIV>\nx", "<DIV>\nx", r#"html "<DIV>\nx" @1:1-2:2"#)] +#[case::html_self_closing("<div/>\nx", "<div/>\nx", r#"html "<div/>\nx" @1:1-2:2"#)] +#[case::html_ins("<ins>\nx", "<ins>\nx", r#"html "<ins>\nx" @1:1-2:2"#)] +#[case::html_br("<br>\nx", "<br>\nx", r#"html "<br>\nx" @1:1-2:2"#)] +#[case::html_textarea( + "<textarea>\nx\n\ny</textarea>", + "&lt;textarea>\nx\n\ny&lt;/textarea>", + r#"html "<textarea>\nx\n\ny</textarea>" @1:1-4:13"# +)] +#[case::html_table_two_blocks( + "<table>\n<tr>\n\n<td>", + "<table>\n<tr>\n<td>", + r#" +html "<table>\n<tr>" @1:1-2:5 +html "<td>" @4:1-4:5 +"# +)] +#[case::html_img_self_close( + "<img src=\"x\" />\nz", + "<img src=\"x\" />\nz", + r#"html "<img src=\"x\" />\nz" @1:1-2:2"# +)] +#[case::html_img_then_text( + "<img src=\"x\" /> y", + "<p><img src=\"x\" /> y</p>", + r#" +html "<img src=\"x\" />" @1:1-1:16 +text " y" @1:16-1:18 +"# +)] +#[case::html_multiline_tag_inline( + "<a\nb>\nz", + "<p><a\nb>\nz</p>", + r#" +html "<a\nb>" @1:1-2:3 +text "\nz" @2:3-3:2 +"# +)] +#[case::html_indent4(" <div>", "<pre><code>&lt;div&gt;\n</code></pre>", r#"code "<div>" @1:1-1:10"#)] +#[case::html_comment_then_text( + "<!--x-->y\nz", + "<!--x-->y\n<p>z</p>", + r#" +html "<!--x-->y" @1:1-1:10 +text "z" @2:1-2:2 +"# +)] +#[case::html_then_code_line("<div>\n code", "<div>\n code", r#"html "<div>\n code" @1:1-2:9"#)] +#[case::html_comment_short( + "<!-->\nx", + "<!-->\n<p>x</p>", + r#" +html "<!-->" @1:1-1:6 +text "x" @2:1-2:2 +"# +)] +#[case::html_instruction_short( + "<?>\nx", + "<?>\n<p>x</p>", + r#" +html "<?>" @1:1-1:4 +text "x" @2:1-2:2 +"# +)] +#[case::html_lazy_quote( + "> <div>\nx", + "<blockquote>\n<div>\n</blockquote>\n<p>x</p>", + r#" +blockquote @1:1-1:8 + html "<div>" @1:3-1:8 +text "x" @2:1-2:2 +"# +)] +#[case::html_attr_forms( + "<a b c=d e='f' g=\"h\">\nx", + "<a b c=d e='f' g=\"h\">\nx", + r#"html "<a b c=d e='f' g=\"h\">\nx" @1:1-2:2"# +)] +#[case::html_bad_attr("<a b=>\nx", "<p>&lt;a b=&gt;\nx</p>", r#"text "<a b=>\nx" @1:1-2:2"#)] +#[case::fm_yaml( + "---\na: b\n---\ntext", + "<p>text</p>", + r#" +yaml "a: b" @1:1-3:4 +text "text" @4:1-4:5 +"# +)] +#[case::fm_yaml_only("---\na: b\n---", "", r#"yaml "a: b" @1:1-3:4"#)] +#[case::fm_empty("---\n---", "", r#"yaml "" @1:1-2:4"#)] +#[case::fm_unclosed( + "---\na\n", + "<hr />\n<p>a</p>\n", + r#" +hr marker='-' @1:1-1:4 +text "a" @2:1-2:2 +"# +)] +#[case::fm_blank_content( + "---\n\n---\nx", + "<p>x</p>", + r#" +yaml "" @1:1-3:4 +text "x" @4:1-4:2 +"# +)] +#[case::fm_toml( + "+++\na = 1\n+++\nx", + "<p>x</p>", + r#" +toml "a = 1" @1:1-3:4 +text "x" @4:1-4:2 +"# +)] +#[case::fm_indented( + " ---\na\n---", + "<hr />\n<h2>a</h2>", + r#" +hr marker='-' @1:1-1:5 +heading 2 @2:1-3:4 + text "a" @2:1-2:2 +"# +)] +#[case::fm_close_trailing_space( + "---\na\n--- \nx", + "<p>x</p>", + r#" +yaml "a" @1:1-3:5 +text "x" @4:1-4:2 +"# +)] +#[case::fm_close_longer( + "---\na\n----\nx", + "<hr />\n<h2>a</h2>\n<p>x</p>", + r#" +hr marker='-' @1:1-1:4 +heading 2 @2:1-3:5 + text "a" @2:1-2:2 +text "x" @4:1-4:2 +"# +)] +#[case::fm_open_trailing_space("--- \na\n---", "", r#"yaml "a" @1:1-3:4"#)] +#[case::fm_open_junk( + "---x\na\n---", + "<h2>---x\na</h2>", + r#" +heading 2 @1:1-3:4 + text "---x\na" @1:1-2:2 +"# +)] +#[case::fm_not_first( + "text\n---\na\n---", + "<h2>text</h2>\n<h2>a</h2>", + r#" +heading 2 @1:1-2:4 + text "text" @1:1-1:5 +heading 2 @3:1-4:4 + text "a" @3:1-3:2 +"# +)] +#[case::fm_blank_before( + "\n---\na\n---", + "<hr />\n<h2>a</h2>", + r#" +hr marker='-' @2:1-2:4 +heading 2 @3:1-4:4 + text "a" @3:1-3:2 +"# +)] +#[case::fm_multiline_content( + "---\na: 1\n\nb: 2\n---\n# h", + "<h1>h</h1>", + r#" +yaml "a: 1\n\nb: 2" @1:1-5:4 +heading 1 @6:1-6:4 + text "h" @6:3-6:4 +"# +)] +#[case::fm_hr_after( + "---\na\n---\n\n---", + "<hr />", + r#" +yaml "a" @1:1-3:4 +hr marker='-' @5:1-5:4 +"# +)] +#[case::math_basic( + "$$\na\n$$", + "<pre><code class=\"language-math math-display\">a\n</code></pre>", + r#"math "a" @1:1-3:3"# +)] +#[case::math_then_text( + "$$\na\n$$\nx", + "<pre><code class=\"language-math math-display\">a\n</code></pre>\n<p>x</p>", + r#" +math "a" @1:1-3:3 +text "x" @4:1-4:2 +"# +)] +#[case::math_meta( + "$$ meta\na\n$$", + "<pre><code class=\"language-math math-display\">a\n</code></pre>", + r#"math "a" @1:1-3:3"# +)] +#[case::math_unclosed( + "$$\na", + "<pre><code class=\"language-math math-display\">a\n</code></pre>\n", + r#"math "a" @1:1-2:2"# +)] +#[case::math_longer_open( + "$$$\na\n$$", + "<pre><code class=\"language-math math-display\">a\n$$\n</code></pre>\n", + r#"math "a\n$$" @1:1-3:3"# +)] +#[case::math_longer_close( + "$$\na\n$$$$", + "<pre><code class=\"language-math math-display\">a\n</code></pre>", + r#"math "a" @1:1-3:5"# +)] +#[case::math_indented( + " $$\n a\n b\n $$", + "<pre><code class=\"language-math math-display\">a\n b\n</code></pre>", + r#"math "a\n b" @1:3-4:5"# +)] +#[case::math_empty( + "$$\n$$", + "<pre><code class=\"language-math math-display\"></code></pre>", + r#"math "" @1:1-2:3"# +)] +#[case::math_inline_not_block( + "$$a$$", + "<p><code class=\"language-math math-inline\">a</code></p>", + r#"math_inline "a" @1:1-1:6"# +)] +#[case::math_info_no_close( + "$$a\n$$", + "<pre><code class=\"language-math math-display\"></code></pre>", + r#"math "" @1:1-2:3"# +)] +#[case::math_in_quote( + "> $$\n> a\n> $$", + "<blockquote>\n<pre><code class=\"language-math math-display\">a\n</code></pre>\n</blockquote>", + r#" +blockquote @1:1-3:5 + math "a" @1:3-3:5 +"# +)] +#[case::math_in_list( + "- $$\n a\n $$", + "<ul>\n<li>\n<pre><code class=\"language-math math-display\">a\n</code></pre>\n</li>\n</ul>", + r#" +list level=0 index=0 marker=- @1:3-3:5 + math "a" @1:3-3:5 +"# +)] +#[case::math_close_junk( + "$$\na\n$$ x", + "<pre><code class=\"language-math math-display\">a\n$$ x\n</code></pre>\n", + r#"math "a\n$$ x" @1:1-3:5"# +)] +#[case::math_interrupts_paragraph( + "text\n$$\na\n$$", + "<p>text</p>\n<pre><code class=\"language-math math-display\">a\n</code></pre>", + r#" +text "text" @1:1-1:5 +math "a" @2:1-4:3 +"# +)] +#[case::math_blank_inside( + "$$\n\na\n$$", + "<pre><code class=\"language-math math-display\">\na\n</code></pre>", + r#"math "\na" @1:1-4:3"# +)] +#[case::math_single_dollar_line( + "$ $\nx", + "<p><code class=\"language-math math-inline\"> </code>\nx</p>", + r#" +math_inline " " @1:1-1:4 +text "\nx" @1:4-2:2 +"# +)] +#[case::crlf_frontmatter( + "---\r\na\r\n---\r\nx", + "<p>x</p>", + r#" +yaml "a" @1:1-3:4 +text "x" @4:1-4:2 +"# +)] +#[case::break_end_indented( + "a \n b", + "<p>a<br />\nb</p>", + r#" +text "a" @1:1-1:2 +break @1:2-2:1 +text "b" @2:3-2:4 +"# +)] +#[case::text_end_indented_continuation( + "*a*\n <x>", + "<p><em>a</em>\n<x></p>", + r#" +emphasis @1:1-1:4 + text "a" @1:2-1:3 +text "\n" @1:4-2:1 +html "<x>" @2:3-2:6 +"# +)] +#[case::soft_end_before_node( + "a\n `b`", + "<p>a\n<code>b</code></p>", + r#" +text "a\n" @1:1-2:1 +code_inline "b" @2:3-2:6 +"# +)] +#[case::break_in_quote( + "> a \n> b", + "<blockquote>\n<p>a<br />\nb</p>\n</blockquote>", + r#" +blockquote @1:1-2:4 + text "a" @1:3-1:4 + break @1:4-2:1 + text "b" @2:3-2:4 +"# +)] +#[case::text_eol_in_footnote( + "[^a]: x \n [a]", + "", + r#" +footnote "a" @1:1-2:6 + text "x\n[a]" @1:7-2:6 +"# +)] +#[case::math_whitespace_content( + " $$\n \n $$", + "<pre><code class=\"language-math math-display\">\n</code></pre>", + r#"math "" @1:3-3:5"# +)] +#[case::math_whitespace_content2( + " $$\n \n $$", + "<pre><code class=\"language-math math-display\">\n</code></pre>", + r#"math "" @1:3-3:5"# +)] +#[case::table_indented_no_pipe( + " a|b\n|-|:-:|", + "<table>\n<thead>\n<tr>\n<th>a</th>\n<th align=\"center\">b</th>\n</tr>\n</thead>\n</table>", + r#" +cell 0:0 @1:1-1:4 + text "a" @1:3-1:4 +cell 0:1 @1:4-1:6 + text "b" @1:5-1:6 +align none,center @2:1-2:1 +"# +)] +#[case::tab_nested_list( + "- a\n\t- b\n\t\t- c\n", + "<ul>\n<li>a\n<ul>\n<li>b\n<ul>\n<li>c</li>\n</ul>\n</li>\n</ul>\n</li>\n</ul>\n", + r#" +list level=0 index=0 marker=- @1:3-1:4 + text "a" @1:3-1:4 +list level=1 index=0 marker=- @2:7-2:8 + text "b" @2:7-2:8 +list level=2 index=0 marker=- @3:11-3:12 + text "c" @3:11-3:12 +"# +)] +#[case::tab_nested_ordered( + "1. a\n\t1. b\n\t\t1. c\n", + "<ol>\n<li>a\n<ol>\n<li>b\n<ol>\n<li>c</li>\n</ol>\n</li>\n</ol>\n</li>\n</ol>\n", + r#" +list level=0 index=0 ordered start=1 marker=. @1:4-1:5 + text "a" @1:4-1:5 +list level=1 index=0 ordered start=1 marker=. @2:8-2:9 + text "b" @2:8-2:9 +list level=2 index=0 ordered start=1 marker=. @3:12-3:13 + text "c" @3:12-3:13 +"# +)] +#[case::tab_list_code( + "- a\n\n\t\tcode\n", + "<ul>\n<li>\n<p>a</p>\n<pre><code> code\n</code></pre>\n</li>\n</ul>\n", + r#" +list level=0 index=0 marker=- @1:3-3:13 + text "a" @1:3-1:4 + code " code" @3:3-3:13 +"# +)] +#[case::tab_in_text( + "a\tb\t*c*\n", + "<p>a\tb\t<em>c</em></p>\n", + r#" +text "a\tb\t" @1:1-1:9 +emphasis @1:9-1:12 + text "c" @1:10-1:11 +"# +)] +#[case::tab_code_block( + "\tcode\n\t\tmore\n", + "<pre><code>code\n\tmore\n</code></pre>\n", + r#"code "code\n\tmore" @1:1-2:13"# +)] +#[case::tab_after_marker( + "-\ta\n", + "<ul>\n<li>a</li>\n</ul>\n", + r#" +list level=0 index=0 marker=- @1:5-1:6 + text "a" @1:5-1:6 +"# +)] +#[case::atx_leading_sequences( + "# # a", + "<h1># a</h1>", + r##" +heading 1 @1:1-1:6 + text "# a" @1:3-1:6 +"## +)] +#[case::atx_leading_sequences2( + "## # a", + "<h2># a</h2>", + r##" +heading 2 @1:1-1:7 + text "# a" @1:4-1:7 +"## +)] +#[case::atx_only_sequences( + "# # #", + "<h1>#</h1>", + r##" +heading 1 @1:1-1:6 + text "#" @1:3-1:4 +"## +)] +#[case::atx_no_space_sequence( + "# #a", + "<h1>#a</h1>", + r##" +heading 1 @1:1-1:5 + text "#a" @1:3-1:5 +"## +)] +#[case::atx_double_sequence( + "# ## a", + "<h1>## a</h1>", + r###" +heading 1 @1:1-1:7 + text "## a" @1:3-1:7 +"### +)] +#[case::atx_tab_sequence( + "#\t# a", + "<h1># a</h1>", + r##" +heading 1 @1:1-1:8 + text "# a" @1:5-1:8 +"## +)] +#[case::atx_closing_and_leading( + "# # a #", + "<h1># a</h1>", + r##" +heading 1 @1:1-1:8 + text "# a" @1:3-1:6 +"## +)] +#[case::atx_inner_hash( + "# a # b", + "<h1>a # b</h1>", + r#" +heading 1 @1:1-1:8 + text "a # b" @1:3-1:8 +"# +)] +#[case::quote_footnote_trailing_space( + "- a\n > [^a]: x\n ", + "<ul>\n<li>a\n<blockquote>\n</blockquote>\n</li>\n</ul>", + r#" +list level=0 index=0 marker=- @1:3-3:3 + text "a" @1:3-1:4 + blockquote @2:3-3:3 + footnote "a" @2:5-3:3 + text "x" @2:11-2:12 +"# +)] +#[case::indented_code_trailing_blank(" code\n\n\n", "<pre><code>code\n</code></pre>\n", r#"code "code" @1:1-1:9"#)] +#[case::bom_heading( + "\u{feff}# a\n", + "<h1>a</h1>\n", + r#" +heading 1 @1:4-1:7 + text "a" @1:6-1:7 +"# +)] +#[case::bom_frontmatter("\u{feff}---\na: b\n---\n", "", r#"yaml "a: b" @1:4-3:4"#)] +#[case::bom_text("\u{feff}a\n", "<p>a</p>\n", r#"text "a" @1:4-1:5"#)] +#[case::www_multibyte_after_ww("wwß x", "<p>wwß x</p>", r#"text "wwß x" @1:1-1:7"#)] +#[case::www_combining("ww\u{301}.a.com", "<p>ww\u{301}.a.com</p>", r#"text "ww\u{301}.a.com" @1:1-1:11"#)] +#[case::www_cjk( + "www.あ.com/パス x", + "<p><a href=\"http://www.%E3%81%82.com/%E3%83%91%E3%82%B9\">www.あ.com/パス</a> x</p>", + r#" +link "http://www.あ.com/パス" @1:1-1:19 + text "www.あ.com/パス" @1:1-1:19 +text " x" @1:19-1:21 +"# +)] +#[case::http_cjk( + "http://あ.jp/パス ok", + "<p><a href=\"http://%E3%81%82.jp/%E3%83%91%E3%82%B9\">http://あ.jp/パス</a> ok</p>", + r#" +link "http://あ.jp/パス" @1:1-1:21 + text "http://あ.jp/パス" @1:1-1:21 +text " ok" @1:21-1:24 +"# +)] +#[case::email_cjk_domain("a@あ.com", "<p>a@あ.com</p>", r#"text "a@あ.com" @1:1-1:10"#)] +#[case::email_cjk_local("あ@b.com", "<p>あ@b.com</p>", r#"text "あ@b.com" @1:1-1:10"#)] +#[case::html_inline_continuation( + "<img a\n b=\"c\">", + "<p><img a\nb=\"c\"></p>", + r#"html "<img a\nb=\"c\">" @1:1-2:12"# +)] +#[case::html_inline_continuation_tab( + "<img a\n\t\tb=\"c\">", + "<p><img a\nb=\"c\"></p>", + r#"html "<img a\nb=\"c\">" @1:1-2:15"# +)] +#[case::table_trailing_space( + "| a | b | \n|-|-| \n| c | d | \n", + "<table>\n<thead>\n<tr>\n<th>a</th>\n<th>b</th>\n</tr>\n</thead>\n<tbody>\n<tr>\n<td>c</td>\n<td>d</td>\n</tr>\n</tbody>\n</table>\n", + r#" +cell 0:0 @1:1-1:5 + text "a" @1:3-1:4 +cell 0:1 @1:5-1:12 + text "b" @1:7-1:8 +align none,none @2:1-2:1 +cell 1:0 @3:1-3:5 + text "c" @3:3-3:4 +cell 1:1 @3:5-3:12 + text "d" @3:7-3:8 +"# +)] +#[case::table_no_pipes_trailing_space( + "a | b\n--|--\nd | e \n", + "<table>\n<thead>\n<tr>\n<th>a</th>\n<th>b</th>\n</tr>\n</thead>\n<tbody>\n<tr>\n<td>d</td>\n<td>e</td>\n</tr>\n</tbody>\n</table>\n", + r#" +cell 0:0 @1:1-1:3 + text "a" @1:1-1:2 +cell 0:1 @1:3-1:6 + text "b" @1:5-1:6 +align none,none @2:1-2:1 +cell 1:0 @3:1-3:3 + text "d" @3:1-3:2 +cell 1:1 @3:3-3:7 + text "e" @3:5-3:6 +"# +)] +#[case::table_trailing_tab( + "| a | b |\t\n|-|-|\n| c | d | \n", + "<table>\n<thead>\n<tr>\n<th>a</th>\n<th>b</th>\n</tr>\n</thead>\n<tbody>\n<tr>\n<td>c</td>\n<td>d</td>\n</tr>\n</tbody>\n</table>\n", + r#" +cell 0:0 @1:1-1:5 + text "a" @1:3-1:4 +cell 0:1 @1:5-1:13 + text "b" @1:7-1:8 +align none,none @2:1-2:1 +cell 1:0 @3:1-3:5 + text "c" @3:3-3:4 +cell 1:1 @3:5-3:13 + text "d" @3:7-3:8 +"# +)] +#[case::table_cjk( + "| あ | 🎉 |\n|:-|-:|\n| é | ß |\n", + "<table>\n<thead>\n<tr>\n<th align=\"left\">あ</th>\n<th align=\"right\">🎉</th>\n</tr>\n</thead>\n<tbody>\n<tr>\n<td align=\"left\">é</td>\n<td align=\"right\">ß</td>\n</tr>\n</tbody>\n</table>\n", + r#" +cell 0:0 @1:1-1:7 + text "あ" @1:3-1:6 +cell 0:1 @1:7-1:15 + text "🎉" @1:9-1:13 +align left,right @2:1-2:1 +cell 1:0 @3:1-3:6 + text "é" @3:3-3:5 +cell 1:1 @3:6-3:12 + text "ß" @3:8-3:10 +"# +)] +#[case::table_single_column_colon( + "a\n:-\nb\n", + "<table>\n<thead>\n<tr>\n<th align=\"left\">a</th>\n</tr>\n</thead>\n<tbody>\n<tr>\n<td align=\"left\">b</td>\n</tr>\n</tbody>\n</table>\n", + r#" +cell 0:0 @1:1-1:2 + text "a" @1:1-1:2 +align left @2:1-2:1 +cell 1:0 @3:1-3:2 + text "b" @3:1-3:2 +"# +)] +#[case::table_single_column_right( + "a\n---:\nb\nc\n", + "<table>\n<thead>\n<tr>\n<th align=\"right\">a</th>\n</tr>\n</thead>\n<tbody>\n<tr>\n<td align=\"right\">b</td>\n</tr>\n<tr>\n<td align=\"right\">c</td>\n</tr>\n</tbody>\n</table>\n", + r#" +cell 0:0 @1:1-1:2 + text "a" @1:1-1:2 +align right @2:1-2:1 +cell 1:0 @3:1-3:2 + text "b" @3:1-3:2 +cell 2:0 @4:1-4:2 + text "c" @4:1-4:2 +"# +)] +#[case::table_double_colon_not_delimiter("a\n::-\n", "<p>a\n::-</p>\n", r#"text "a\n::-" @1:1-2:4"#)] +#[case::table_body_empty_marker( + "a|b\n|-|-|\n*\n", + "<table>\n<thead>\n<tr>\n<th>a</th>\n<th>b</th>\n</tr>\n</thead>\n</table>\n<ul>\n<li></li>\n</ul>\n", + r#" +cell 0:0 @1:1-1:2 + text "a" @1:1-1:2 +cell 0:1 @1:2-1:4 + text "b" @1:3-1:4 +align none,none @2:1-2:1 +list level=0 index=0 marker=* @3:1-3:2 +"# +)] +#[case::table_body_ordered_marker( + "a|b\n|-|-|\n2. x\n", + "<table>\n<thead>\n<tr>\n<th>a</th>\n<th>b</th>\n</tr>\n</thead>\n</table>\n<ol start=\"2\">\n<li>x</li>\n</ol>\n", + r#" +cell 0:0 @1:1-1:2 + text "a" @1:1-1:2 +cell 0:1 @1:2-1:4 + text "b" @1:3-1:4 +align none,none @2:1-2:1 +list level=0 index=0 ordered start=2 marker=. @3:4-3:5 + text "x" @3:4-3:5 +"# +)] +#[case::table_body_empty_ordered( + "a|b\n|-|-|\n1.\n", + "<table>\n<thead>\n<tr>\n<th>a</th>\n<th>b</th>\n</tr>\n</thead>\n</table>\n<ol>\n<li></li>\n</ol>\n", + r#" +cell 0:0 @1:1-1:2 + text "a" @1:1-1:2 +cell 0:1 @1:2-1:4 + text "b" @1:3-1:4 +align none,none @2:1-2:1 +list level=0 index=0 ordered start=1 marker=. @3:1-3:3 +"# +)] +#[case::paragraph_after_partial_tab( + "- a\n\n\t[P](h) x\n", + "<ul>\n<li>\n<p>a</p>\n<p><a href=\"h\">P</a> x</p>\n</li>\n</ul>\n", + r#" +list level=0 index=0 marker=- @1:3-3:13 + text "a" @1:3-1:4 + link "h" @3:5-3:11 + text "P" @3:6-3:7 + text " x" @3:11-3:13 +"# +)] +#[case::fence_after_partial_tab( + "- a\n\n\t```ts\n\t\tx\n\t```\n", + "<ul>\n<li>\n<p>a</p>\n<pre><code class=\"language-ts\">\tx\n</code></pre>\n</li>\n</ul>\n", + r#" +list level=0 index=0 marker=- @1:3-5:8 + text "a" @1:3-1:4 + code "\tx" lang="ts" fence @3:5-5:8 +"# +)] +#[case::math_after_partial_tab( + "- a\n\n\t$$\n\tm\n\t$$\n", + "<ul>\n<li>\n<p>a</p>\n<pre><code class=\"language-math math-display\">m\n</code></pre>\n</li>\n</ul>\n", + r#" +list level=0 index=0 marker=- @1:3-5:7 + text "a" @1:3-1:4 + math "m" @3:5-5:7 +"# +)] +#[case::quote_after_partial_tab( + "- a\n\n\t> q\n", + "<ul>\n<li>\n<p>a</p>\n<blockquote>\n<p>q</p>\n</blockquote>\n</li>\n</ul>\n", + r#" +list level=0 index=0 marker=- @1:3-3:8 + text "a" @1:3-1:4 + blockquote @3:3-3:8 + text "q" @3:7-3:8 +"# +)] +#[case::heading_after_partial_tab( + "- a\n\n\t# h\n", + "<ul>\n<li>\n<p>a</p>\n<h1>h</h1>\n</li>\n</ul>\n", + r#" +list level=0 index=0 marker=- @1:3-3:8 + text "a" @1:3-1:4 + heading 1 @3:3-3:8 + text "h" @3:7-3:8 +"# +)] +#[case::control_character_references( + "&#1;&#x7f;&#x85;&#xB;&#xC;&#x9f;&#xa0;&#xd800;&#x110000;&#0;", + "<p>����\u{c}�\u{a0}���</p>", + r#"text "����\u{c}�\u{a0}���" @1:1-1:61"# +)] +#[case::footnote_escaped_bracket("[^a\\]: x", "<p>[^a]: x</p>", r#"text "[^a]: x" @1:1-1:9"#)] +#[case::footnote_escaped_backslash( + "[^a\\\\]: x", + "", + r#" +footnote "a\\\\" @1:1-1:10 + text "x" @1:9-1:10 +"# +)] +#[case::footnote_escaped_open( + "[^a\\[]: x", + "", + r#" +footnote "a\\[" @1:1-1:10 + text "x" @1:9-1:10 +"# +)] +#[case::footnote_unescaped_open("[^a[]: x", "<p>[^a[]: x</p>", r#"text "[^a[]: x" @1:1-1:9"#)] +#[case::footnote_empty_then_blank("[^a]: \n\n", "", r#"footnote "a" @1:1-2:1"#)] +#[case::footnote_empty_then_spaces("[^a]: \n \n", "", r#"footnote "a" @1:1-2:5"#)] +#[case::footnote_empty_then_text( + "[^a]: \n\nb", + "<p>b</p>", + r#" +footnote "a" @1:1-2:1 +text "b" @3:1-3:2 +"# +)] +#[case::definition_nul_destination( + "[a]: \0\n\n[a]\n", + "<p><a href=\"%EF%BF%BD\">a</a></p>\n", + r#" +definition "a" "\0" label="a" @1:1-1:7 +link_ref "a" label="a" @3:1-3:4 + text "a" @3:2-3:3 +"# +)] +#[case::link_nul_destination( + "[a](x\0y)", + "<p><a href=\"x%EF%BF%BDy\">a</a></p>", + r#" +link "x\0y" @1:1-1:9 + text "a" @1:2-1:3 +"# +)] +#[case::definition_then_empty_marker( + "[a]: b\n-", + "<p>-</p>", + r#" +definition "a" "b" label="a" @1:1-1:7 +text "-" @2:1-2:2 +"# +)] +#[case::definition_then_item( + "[a]: b\n- x", + "<ul>\n<li>x</li>\n</ul>", + r#" +definition "a" "b" label="a" @1:1-1:7 +list level=0 index=0 marker=- @2:3-2:4 + text "x" @2:3-2:4 +"# +)] +#[case::definition_then_ordered( + "[a]: b\n1.", + "<p>1.</p>", + r#" +definition "a" "b" label="a" @1:1-1:7 +text "1." @2:1-2:3 +"# +)] +#[case::emphasis_cjk( + "*あ*。*い*", + "<p><em>あ</em>。<em>い</em></p>", + r#" +emphasis @1:1-1:6 + text "あ" @1:2-1:5 +text "。" @1:6-1:9 +emphasis @1:9-1:14 + text "い" @1:10-1:13 +"# +)] +#[case::strong_cjk( + "**日本語**です", + "<p><strong>日本語</strong>です</p>", + r#" +strong @1:1-1:14 + text "日本語" @1:3-1:12 +text "です" @1:14-1:20 +"# +)] +#[case::underscore_accent( + "_é_ _ß_", + "<p><em>é</em> <em>ß</em></p>", + r#" +emphasis @1:1-1:5 + text "é" @1:2-1:4 +text " " @1:5-1:6 +emphasis @1:6-1:10 + text "ß" @1:7-1:9 +"# +)] +#[case::fullwidth_markers("*a* _b_", "<p>*a* _b_</p>", r#"text "*a* _b_" @1:1-1:16"#)] +#[case::emphasis_after_cjk_punctuation( + "。*a*、**b**", + "<p>。<em>a</em>、<strong>b</strong></p>", + r#" +text "。" @1:1-1:4 +emphasis @1:4-1:7 + text "a" @1:5-1:6 +text "、" @1:7-1:10 +strong @1:10-1:15 + text "b" @1:12-1:13 +"# +)] +#[case::combining_mark( + "e\u{301}*a*", + "<p>e\u{301}<em>a</em></p>", + r#" +text "e\u{301}" @1:1-1:4 +emphasis @1:4-1:7 + text "a" @1:5-1:6 +"# +)] +#[case::emoji_zwj( + "👨\u{200d}👩\u{200d}👧 *a*", + "<p>👨\u{200d}👩\u{200d}👧 <em>a</em></p>", + r#" +text "👨\u{200d}👩\u{200d}👧 " @1:1-1:20 +emphasis @1:20-1:23 + text "a" @1:21-1:22 +"# +)] +#[case::fence_cjk_info( + "```あ\n日本語\n```\n", + "<pre><code class=\"language-あ\">日本語\n</code></pre>\n", + r#"code "日本語" lang="あ" fence @1:1-3:4"# +)] +#[case::fence_in_quote_emoji( + "> ```🎉\n> x\n> ```\n", + "<blockquote>\n<pre><code class=\"language-🎉\">x\n</code></pre>\n</blockquote>\n", + r#" +blockquote @1:1-3:6 + code "x" lang="🎉" fence @1:3-3:6 +"# +)] +#[case::fence_in_list_cjk( + "- ```あ\n 日本語\n ```\n", + "<ul>\n<li>\n<pre><code class=\"language-あ\">日本語\n</code></pre>\n</li>\n</ul>\n", + r#" +list level=0 index=0 marker=- @1:3-3:6 + code "日本語" lang="あ" fence @1:3-3:6 +"# +)] +#[case::math_cjk( + "$$\nあ\n$$\n\n$い$\n", + "<pre><code class=\"language-math math-display\">あ\n</code></pre>\n<p><code class=\"language-math math-inline\">い</code></p>\n", + r#" +math "あ" @1:1-3:3 +math_inline "い" @5:1-5:6 +"# +)] +#[case::heading_cjk( + "# 見出し\n\n見出し2\n===\n", + "<h1>見出し</h1>\n<h1>見出し2</h1>\n", + r#" +heading 1 @1:1-1:12 + text "見出し" @1:3-1:12 +heading 1 @3:1-4:4 + text "見出し2" @3:1-3:11 +"# +)] +#[case::link_cjk( + "[あ](http://い.jp/う \"え\")", + "<p><a href=\"http://%E3%81%84.jp/%E3%81%86\" title=\"え\">あ</a></p>", + r#" +link "http://い.jp/う" title="え" @1:1-1:31 + text "あ" @1:2-1:5 +"# +)] +#[case::reference_cjk( + "[あ]\n\n[あ]: /u\n", + "<p><a href=\"/u\">あ</a></p>\n", + r#" +link_ref "あ" label="あ" @1:1-1:6 + text "あ" @1:2-1:5 +definition "あ" "/u" label="あ" @3:1-3:10 +"# +)] +#[case::footnote_cjk( + "あ[^い]\n\n[^い]: う\n", + "<p>あ<sup><a href=\"#user-content-fn-%E3%81%84\" id=\"user-content-fnref-%E3%81%84\" data-footnote-ref=\"\" aria-describedby=\"footnote-label\">1</a></sup></p>\n<section data-footnotes=\"\" class=\"footnotes\"><h2 id=\"footnote-label\" class=\"sr-only\">Footnotes</h2>\n<ol>\n<li id=\"user-content-fn-%E3%81%84\">\n<p>う <a href=\"#user-content-fnref-%E3%81%84\" data-footnote-backref=\"\" aria-label=\"Back to content\" class=\"data-footnote-backref\">↩</a></p>\n</li>\n</ol>\n</section>\n", + r#" +text "あ" @1:1-1:4 +footnote_ref "い" label="い" @1:4-1:10 +footnote "い" @3:1-3:12 + text "う" @3:9-3:12 +"# +)] +#[case::ideographic_space( + "a\u{3000}b\n\u{3000}c", + "<p>a\u{3000}b\n\u{3000}c</p>", + r#"text "a\u{3000}b\n\u{3000}c" @1:1-2:5"# +)] +#[case::nbsp_around_emphasis( + "a\u{a0}*b*\u{a0}c", + "<p>a\u{a0}<em>b</em>\u{a0}c</p>", + r#" +text "a\u{a0}" @1:1-1:4 +emphasis @1:4-1:7 + text "b" @1:5-1:6 +text "\u{a0}c" @1:7-1:10 +"# +)] +#[case::entity_cjk_context( + "あ&amp;い&#12354;&#x1F389;", + "<p>あ&amp;いあ🎉</p>", + r#"text "あ&いあ🎉" @1:1-1:29"# +)] +#[case::autolink_cjk( + "<http://あ.jp/パス>", + "<p><a href=\"http://%E3%81%82.jp/%E3%83%91%E3%82%B9\">http://あ.jp/パス</a></p>", + r#" +link "http://あ.jp/パス" @1:1-1:23 + text "http://あ.jp/パス" @1:2-1:22 +"# +)] +#[case::code_span_cjk( + "`あ`と``い`う``", + "<p><code>あ</code>と<code>い`う</code></p>", + r#" +code_inline "あ" @1:1-1:6 +text "と" @1:6-1:9 +code_inline "い`う" @1:9-1:20 +"# +)] +#[case::hard_break_cjk( + "あ \nい\\\nう", + "<p>あ<br />\nい<br />\nう</p>", + r#" +text "あ" @1:1-1:4 +break @1:4-2:1 +text "い" @2:1-2:4 +break @2:4-3:1 +text "う" @3:1-3:4 +"# +)] +#[case::nul_in_text("a\0b", "<p>a�b</p>", r#"text "a\0b" @1:1-1:4"#)] +fn markdown(#[case] input: &str, #[case] html: &str, #[case] expected: &str) { + let nodes = Markdown::from_markdown_str(input).unwrap().nodes; + assert_eq!(tree(&nodes), expected.trim_matches('\n'), "{input:?}"); + assert_eq!(mq_markdown::to_html(input), html, "{input:?}"); +} + +#[rstest] +#[case::jsx_self("<a />", "jsx_flow <a> @1:1-1:6")] +#[case::jsx_attrs( + "<a b=\"c&amp;d\" e='f' g={h} {...i} j />", + r#"jsx_flow <a b="c&d" e="f" g={h} {...i} j> @1:1-1:39"# +)] +#[case::jsx_member("<a.b.c />", "jsx_flow <a.b.c> @1:1-1:10")] +#[case::jsx_namespace("<a:b c:d=\"e\" />", r#"jsx_flow <a:b c:d="e"> @1:1-1:16"#)] +#[case::jsx_fragment("<></>", "jsx_flow <> @1:1-1:6")] +#[case::jsx_text_pair( + "<a>x</a>", + r#" +jsx_text <a> @1:1-1:9 + text "x" @1:4-1:5 +"# +)] +#[case::jsx_flow_pair( + "<a>\n\nx\n\n</a>", + r#" +jsx_flow <a> @1:1-5:5 + text "x" @3:1-3:2 +"# +)] +#[case::jsx_flow_indented_child( + "<a>\n x\n</a>", + r#" +jsx_flow <a> @1:1-3:5 + text "x" @2:3-2:4 +"# +)] +#[case::jsx_inline( + "a <b>c</b> d", + r#" +text "a " @1:1-1:3 +jsx_text <b> @1:3-1:11 + text "c" @1:6-1:7 +text " d" @1:11-1:13 +"# +)] +#[case::jsx_inline_self( + "a <b/> d", + r#" +text "a " @1:1-1:3 +jsx_text <b> @1:3-1:7 +text " d" @1:7-1:9 +"# +)] +#[case::expr_flow("{a}", r#"expression_flow "a" @1:1-1:4"#)] +#[case::expr_nested("{a {b} c}", r#"expression_flow "a {b} c" @1:1-1:10"#)] +#[case::expr_multiline("{a\nb}", r#"expression_flow "a\nb" @1:1-2:3"#)] +#[case::expr_multiline_indented("{a\n b\n c}", r#"expression_flow "a\nb\n c" @1:1-3:6"#)] +#[case::expr_text( + "x {a} y", + r#" +text "x " @1:1-1:3 +expression_text "a" @1:3-1:6 +text " y" @1:6-1:8 +"# +)] +#[case::expr_then_tag( + "{a} <b/>", + r#" +expression_flow "a" @1:1-1:4 +jsx_flow <b> @1:5-1:9 +"# +)] +#[case::tag_then_expr( + "<b/> {a}", + r#" +jsx_flow <b> @1:1-1:5 +expression_flow "a" @1:6-1:9 +"# +)] +#[case::tag_then_text( + "<b/> x", + r#" +jsx_text <b> @1:1-1:5 +text " x" @1:5-1:7 +"# +)] +#[case::expr_then_text( + "{a}x", + r#" +expression_text "a" @1:1-1:4 +text "x" @1:4-1:5 +"# +)] +#[case::jsx_in_expr_child( + "<a>{b}</a>", + r#" +jsx_flow <a> @1:1-1:11 + expression_flow "b" @1:4-1:7 +"# +)] +#[case::jsx_unclosed_text("<a>b", "error: Expected a closing tag for `<a>` before the end of the content")] +#[case::jsx_mismatch( + "<a></b>", + "error: Unexpected closing tag `</b>`, expected corresponding closing tag for `<a>` (1:1)" +)] +#[case::lt_space("a < b", r#"text "a < b" @1:1-1:6"#)] +#[case::lt_digit("a <3", "error: Unexpected character `3` (U+0033) before name")] +#[case::attr_no_value("<a b=>", "error: Unexpected character `>` (U+003E) before attribute value")] +#[case::attr_space_after_eq("<b e= \"f\"/>", r#"jsx_flow <b e="f"> @1:1-1:12"#)] +#[case::attr_spaces_around_eq( + "a <b e = \"f\"/>.", + r#" +text "a " @1:1-1:3 +jsx_text <b e="f"> @1:3-1:15 +text "." @1:15-1:16 +"# +)] +#[case::attr_line_before_eq( + "<b c\n= \"x\">c</b>", + r#" +jsx_text <b c="x"> @1:1-2:12 + text "c" @2:7-2:8 +"# +)] +#[case::attr_namespace_spaces( + "a <b xml :\tlang\n= \"de-CH\" foo:bar>c</b>.", + r#" +text "a " @1:1-1:3 +jsx_text <b xml:lang="de-CH" foo:bar> @1:3-2:24 + text "c" @2:19-2:20 +text "." @2:24-2:25 +"# +)] +#[case::attr_spaced_names( + "a <b a b : c d : e = \"f\" g/>.", + r#" +text "a " @1:1-1:3 +jsx_text <b a b:c d:e="f" g> @1:3-1:29 +text "." @1:29-1:30 +"# +)] +#[case::attr_expr_space_after_eq("<b e= {f}/>", "jsx_flow <b e={f}> @1:1-1:12")] +#[case::jsx_nested_flow( + "<a>\n<b>\n</b>\n</a>", + r#" +jsx_flow <a> @1:1-4:5 + jsx_flow <b> @2:1-3:5 +"# +)] +#[case::jsx_in_quote( + "> <a>\n> x\n> </a>", + r#" +blockquote @1:1-3:7 + jsx_flow <a> @1:3-3:7 + text "x" @2:3-2:4 +"# +)] +#[case::jsx_in_list( + "- <a>\n x\n </a>", + r#" +list level=0 index=0 marker=- @1:3-3:7 + jsx_flow <a> @1:3-3:7 + text "x" @2:3-2:4 +"# +)] +#[case::attr_quote_inside("<a b='c\"d' />", r#"jsx_flow <a b="c\"d"> @1:1-1:14"#)] +#[case::two_tags( + "<a/><b/>", + r#" +jsx_flow <a> @1:1-1:5 +jsx_flow <b> @1:5-1:9 +"# +)] +#[case::two_text_elements( + "<a>x</a><b>y</b>", + r#" +jsx_text <a> @1:1-1:9 + text "x" @1:4-1:5 +jsx_text <b> @1:9-1:17 + text "y" @1:12-1:13 +"# +)] +#[case::interrupt_paragraph_tag( + "a\n<b/>", + r#" +text "a" @1:1-1:2 +jsx_flow <b> @2:1-2:5 +"# +)] +#[case::interrupt_paragraph_expr( + "a\n{b}", + r#" +text "a" @1:1-1:2 +expression_flow "b" @2:1-2:4 +"# +)] +#[case::two_expressions( + "{a}\n{b}", + r#" +expression_flow "a" @1:1-1:4 +expression_flow "b" @2:1-2:4 +"# +)] +#[case::expr_in_flow_element( + "<a>\n\n{b}\n\n</a>", + r#" +jsx_flow <a> @1:1-5:5 + expression_flow "b" @3:1-3:4 +"# +)] +#[case::indented_tag(" <a/>", "jsx_flow <a> @1:4-1:8")] +#[case::very_indented_tag(" <a/>", "jsx_flow <a> @1:5-1:9")] +#[case::heading_jsx( + "# h <a>b</a>", + r#" +heading 1 @1:1-1:13 + text "h " @1:3-1:5 + jsx_text <a> @1:5-1:13 + text "b" @1:8-1:9 +"# +)] +#[case::emphasis_crossing( + "*<a>x</a>*", + r#" +emphasis @1:1-1:11 + jsx_text <a> @1:2-1:10 + text "x" @1:5-1:6 +"# +)] +#[case::emphasis_crossing_bad( + "*<a>x*</a>", + "error: Expected a closing tag for `<a>` before the end of the content" +)] +#[case::link_jsx( + "[<a>x</a>](y)", + r#" +link "y" @1:1-1:14 + jsx_text <a> @1:2-1:10 + text "x" @1:5-1:6 +"# +)] +#[case::code_not_jsx( + "`<a/>` {b}", + r#" +code_inline "<a/>" @1:1-1:7 +text " " @1:7-1:8 +expression_text "b" @1:8-1:11 +"# +)] +#[case::escaped("\\<a/> \\{b}", r#"text "<a/> {b}" @1:2-1:11"#)] +#[case::attr_entities("<a b=\"&lt;&#x41;\" />", r#"jsx_flow <a b="<A"> @1:1-1:21"#)] +#[case::attr_multiline_literal("<a b='x\ny' />", r#"jsx_flow <a b="x\ny"> @1:1-2:6"#)] +#[case::closing_spaces( + "<a >x</a >", + r#" +jsx_text <a> @1:1-1:13 + text "x" @1:6-1:7 +"# +)] +#[case::lt_space_name("< a>", r#"text "< a>" @1:1-1:5"#)] +#[case::self_closing_space("<a / >", "jsx_flow <a> @1:1-1:7")] +#[case::attr_on_next_line("<a\nb />", "jsx_flow <a b> @1:1-2:5")] +#[case::dashed_name("<a-b />", "jsx_flow <a-b> @1:1-1:8")] +#[case::dashed_attr("<a b-c=\"d\" />", r#"jsx_flow <a b-c="d"> @1:1-1:14"#)] +#[case::bad_name_char("<A_b$.c-d />", "error: Unexpected character `$` (U+0024) after name")] +#[case::expr_unclosed( + "{a", + "error: Unexpected end of file in expression, expected a corresponding closing brace for `{`" +)] +#[case::expr_empty("{}", r#"expression_flow "" @1:1-1:3"#)] +#[case::expr_space("{ }", r#"expression_flow " " @1:1-1:4"#)] +#[case::empty_flow_element("<a>\n</a>", "jsx_flow <a> @1:1-2:5")] +#[case::text_element_unclosed_line( + "<a>x\n</a>", + "error: Expected a closing tag for `<a>` before the end of the content" +)] +#[case::stray_close("</a>", "error: Unexpected closing slash `/` in tag, expected an open tag first")] +#[case::stray_close_text("x </a>", "error: Unexpected closing slash `/` in tag, expected an open tag first")] +#[case::element_with_emphasis( + "<a>*b*</a>", + r#" +jsx_text <a> @1:1-1:11 + emphasis @1:4-1:7 + text "b" @1:5-1:6 +"# +)] +#[case::flow_element_with_emphasis( + "<a>\n*b*\n</a>", + r#" +jsx_flow <a> @1:1-3:5 + emphasis @2:1-2:4 + text "b" @2:2-2:3 +"# +)] +#[case::flow_element_with_list( + "<a>\n- b\n</a>", + r#" +jsx_flow <a> @1:1-3:5 + list level=0 index=0 marker=- @2:3-2:4 + text "b" @2:3-2:4 +"# +)] +#[case::list_items_unbalanced( + "- <a>\n- </a>", + "error: Expected a closing tag for `<a>` (1:3) before the end of its container" +)] +#[case::spaced_equals("<a b = \"c\" />", r#"jsx_flow <a b="c"> @1:1-1:14"#)] +#[case::heading( + "# h\n\ntext *a* [b](c) `d`\n", + r#" +heading 1 @1:1-1:4 + text "h" @1:3-1:4 +text "text " @3:1-3:6 +emphasis @3:6-3:9 + text "a" @3:7-3:8 +text " " @3:9-3:10 +link "c" @3:10-3:16 + text "b" @3:11-3:12 +text " " @3:16-3:17 +code_inline "d" @3:17-3:20 +"# +)] +#[case::indented_is_text(" not code\n", r#"text "not code" @1:5-1:13"#)] +#[case::indented_heading( + " # h\n", + r#" +heading 1 @1:1-1:9 + text "h" @1:8-1:9 +"# +)] +#[case::no_autolink("<http://a.b>", "error: Unexpected character `/` (U+002F) before local name")] +#[case::no_gfm( + "| a |\n|-|\n\n~a~ www.a.b [^a]\n\n- [ ] a\n", + r#" +text "| a |\n|-|" @1:1-2:4 +text "~a~ www.a.b [^a]" @4:1-4:17 +list level=0 index=0 marker=- @6:3-6:8 + text "[ ] a" @6:3-6:8 +"# +)] +#[case::no_math( + "$a$\n\n$$\na\n$$\n", + r#" +text "$a$" @1:1-1:4 +text "$$\na\n$$" @3:1-5:3 +"# +)] +#[case::frontmatter("---\na\n---\n", r#"yaml "a" @1:1-3:4"#)] +#[case::frontmatter_toml( + "+++\na = 1\n+++\n\n<A />", + r#" +toml "a = 1" @1:1-3:4 +jsx_flow <A> @5:1-5:6 +"# +)] +#[case::frontmatter_unclosed( + "---\na\n", + r#" +hr marker='-' @1:1-1:4 +text "a" @2:1-2:2 +"# +)] +#[case::esm( + "import a from 'b'\n\nexport const c = 1\n", + r#" +esm "import a from 'b'" @1:1-1:18 +esm "export const c = 1" @3:1-3:19 +"# +)] +#[case::definition( + "[a]: /u\n\n[a]\n", + r#" +definition "a" "/u" label="a" @1:1-1:8 +link_ref "a" label="a" @3:1-3:4 + text "a" @3:2-3:3 +"# +)] +#[case::list_deep_indent( + "- a\n", + r#" +list level=0 index=0 marker=- @1:8-1:9 + text "a" @1:8-1:9 +"# +)] +#[case::fence("```rust\ncode\n```\n", r#"code "code" lang="rust" fence @1:1-3:4"#)] +#[case::lone_cr_after_lt("<\r/-x", r#"text "<\r/-x" @1:1-2:4"#)] +#[case::crlf_after_lt("<\r\n/-x", r#"text "<\r\n/-x" @1:1-2:4"#)] +#[case::expression_lone_cr("{\r }", r#"expression_flow "\r" @1:1-2:3"#)] +#[case::expression_cr_indent("{a\r b\r\n c\n d}", r#"expression_flow "a\r b\r\nc\nd" @1:1-4:5"#)] +#[case::tilde_not_delimiter("*~*a", r#"text "*~*a" @1:1-1:5"#)] +#[case::tilde_strong_not_delimiter("**~**あ", r#"text "**~**あ" @1:1-1:9"#)] +#[case::dollar_after_name("<a$/>", "error: Unexpected character `$` (U+0024) after name")] +#[case::dollar_after_space("<a $b/>", "jsx_flow <a $b> @1:1-1:8")] +#[case::cjk_tag_name("<あ b=\"c\" />", r#"jsx_flow <あ b="c"> @1:1-1:14"#)] +#[case::fullwidth_underscore_name("<a_b />", "jsx_flow <a_b> @1:1-1:10")] +#[case::jsx_text_cjk( + "あ <b>い</b> う", + r#" +text "あ " @1:1-1:5 +jsx_text <b> @1:5-1:15 + text "い" @1:8-1:11 +text " う" @1:15-1:19 +"# +)] +#[case::expression_cjk( + "{あ} い", + r#" +expression_text "あ" @1:1-1:6 +text " い" @1:6-1:10 +"# +)] +fn mdx(#[case] input: &str, #[case] expected: &str) { + let actual = match Markdown::from_mdx_str(input) { + Ok(markdown) => tree(&markdown.nodes), + Err(error) => format!("error: {error}"), + }; + assert_eq!(actual, expected.trim_matches('\n'), "{input:?}"); +} From ae7178766df8012b289679820b47d213bc4b4b14 Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Fri, 9 Oct 2026 12:03:00 +0900 Subject: [PATCH 54/88] =?UTF-8?q?=E2=9A=A1=20perf(markdown):=20compute=20t?= =?UTF-8?q?ab=20columns=20from=20a=20per-line=20tab=20table?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Columns of lines with tabs were recomputed from the line start for every position, which made long lines with many inline nodes quadratic. --- crates/mq-markdown/src/parser/block.rs | 5 +- crates/mq-markdown/src/parser/line.rs | 82 ++++++++++++------- crates/mq-markdown/src/parser/tree.rs | 42 +++++++--- .../mq-markdown/tests/parser_limits_tests.rs | 3 + 4 files changed, 91 insertions(+), 41 deletions(-) diff --git a/crates/mq-markdown/src/parser/block.rs b/crates/mq-markdown/src/parser/block.rs index 93cda599d..3c557359f 100644 --- a/crates/mq-markdown/src/parser/block.rs +++ b/crates/mq-markdown/src/parser/block.rs @@ -9,7 +9,7 @@ use super::definition; use super::flavor::Flavor; use super::html_flow::{self, Kind as HtmlKind}; use super::inline; -use super::line::{Indent, Line, split_lines}; +use super::line::{Indent, Line, split_lines, tab_stops}; use super::mdx_flow::{FlowOutcome, absorbed_until, blocks_lazy_continuation, looks_like_mdx_flow, probe_mdx_flow}; use super::table; mod rule; @@ -37,7 +37,8 @@ struct Interrupt { } pub(super) fn parse(src: &str, flavor: Flavor, read_frontmatter: bool) -> Vec<Block> { - let mut lines = split_lines(src, flavor); + let stops = tab_stops(src); + let mut lines = split_lines(src, flavor, &stops); // A byte order mark at the start of the document is not content. if let Some(first) = lines.first_mut().filter(|line| line.text.starts_with('\u{feff}')) { *first = first.skip('\u{feff}'.len_utf8()); diff --git a/crates/mq-markdown/src/parser/line.rs b/crates/mq-markdown/src/parser/line.rs index b036924c8..7c4c3a050 100644 --- a/crates/mq-markdown/src/parser/line.rs +++ b/crates/mq-markdown/src/parser/line.rs @@ -26,16 +26,16 @@ pub(super) struct Line<'a> { pub(super) item_end: bool, /// Whether the line continues a paragraph without its container prefix. pub(super) lazy: bool, - /// The whole line as in the document, to compute columns when it contains tabs. - pub(super) origin: &'a str, - /// Whether `origin` contains a tab. - pub(super) tabs: bool, + /// The tabs of the line, to compute columns without scanning the line again. + pub(super) stops: &'a [TabStop], + /// Byte offset of the line in the document, to compare with `stops`. + pub(super) base: usize, /// Columns left over from a tab that a container consumed only in part. They count as leading /// whitespace of `text`. pub(super) pad: usize, /// The dialect of the document. pub(super) flavor: Flavor, - /// For `*`, `-` and `_`: one past the offset in `origin` of the last byte that is neither that + /// For `*`, `-` and `_`: one past the offset in the line of the last byte that is neither that /// character, nor a space or tab, or 0 if there is none. A thematic break is a run of one of them /// and whitespace to the end of the line, so this tells at once, from any offset, that it is not. pub(super) others: [usize; 3], @@ -53,10 +53,16 @@ impl<'a> Line<'a> { /// The zero-based visual column where `text` starts in the document. fn start_column(&self) -> usize { - if self.tabs { - visual_column(self.origin[..self.column].chars(), 0) - } else { - self.column + self.visual_column(self.column) + } + + /// The zero-based visual column of the byte at `offset` in the line. + fn visual_column(&self, offset: usize) -> usize { + let at = self.base + offset; + let passed = self.stops.partition_point(|stop| stop.at < at); + match passed.checked_sub(1).map(|index| self.stops[index]) { + Some(stop) => stop.after + (at - stop.at - 1), + None => offset, } } @@ -92,11 +98,7 @@ impl<'a> Line<'a> { /// to the next multiple of four, like `markdown-rs` counts them. pub(super) fn point(&self, byte: usize) -> Point { let offset = self.column + byte; - let mut column = if self.tabs { - visual_column(self.origin[..offset].chars(), 0) + 1 - } else { - offset + 1 - }; + let mut column = self.visual_column(offset) + 1; // Whitespace at the start of a line inside a tab starts before the end of that tab, content // starts where that tab ends. if byte == 0 { @@ -175,16 +177,31 @@ impl<'a> Line<'a> { } } -/// The zero-based column reached after `chars`, starting at column `start`, where tabs go to the next -/// multiple of four and other characters advance by their length in bytes. -pub(super) fn visual_column(chars: impl Iterator<Item = char>, start: usize) -> usize { - chars.fold(start, |column, char| { - if char == '\t' { - (column / 4 + 1) * 4 - } else { - column + char.len_utf8() +/// A tab in the document and the zero-based column right after it. +#[derive(Clone, Copy)] +pub(super) struct TabStop { + at: usize, + after: usize, +} + +/// Every tab in `src` in order. +pub(super) fn tab_stops(src: &str) -> Vec<TabStop> { + let mut stops = Vec::new(); + // The byte offset and the column right after the last tab or line start. + let (mut anchor, mut anchor_column) = (0, 0); + for (index, byte) in src.bytes().enumerate() { + match byte { + b'\n' | b'\r' => (anchor, anchor_column) = (index + 1, 0), + b'\t' => { + let column = anchor_column + (index - anchor); + let after = (column / 4 + 1) * 4; + stops.push(TabStop { at: index, after }); + (anchor, anchor_column) = (index + 1, after); + } + _ => {} } - }) + } + stops } /// The `Line::others` of `text`. @@ -203,10 +220,19 @@ fn last_others(text: &str) -> [usize; 3] { } /// Splits `src` into lines on `\n`, `\r\n` and `\r`. A trailing terminator does not add a line. -pub(super) fn split_lines(src: &str, flavor: Flavor) -> Vec<Line<'_>> { +pub(super) fn split_lines<'a>(src: &'a str, flavor: Flavor, stops: &'a [TabStop]) -> Vec<Line<'a>> { let bytes = src.as_bytes(); let mut lines = Vec::new(); let (mut start, mut index) = (0, 0); + let mut next_stop = 0; + // The tabs between `start` and `end`. + let mut take_stops = |end: usize| { + let first = next_stop; + while stops.get(next_stop).is_some_and(|stop| stop.at < end) { + next_stop += 1; + } + &stops[first..next_stop] + }; while index < bytes.len() { let end = index; @@ -226,8 +252,8 @@ pub(super) fn split_lines(src: &str, flavor: Flavor) -> Vec<Line<'_>> { eof: index == bytes.len(), item_end: false, lazy: false, - origin: &src[start..end], - tabs: src[start..end].contains('\t'), + stops: take_stops(end), + base: start, pad: 0, flavor, others: last_others(&src[start..end]), @@ -244,8 +270,8 @@ pub(super) fn split_lines(src: &str, flavor: Flavor) -> Vec<Line<'_>> { eof: true, item_end: false, lazy: false, - origin: &src[start..], - tabs: src[start..].contains('\t'), + stops: take_stops(src.len()), + base: start, pad: 0, flavor, others: last_others(&src[start..]), diff --git a/crates/mq-markdown/src/parser/tree.rs b/crates/mq-markdown/src/parser/tree.rs index 356c5b9b8..23aec4e9c 100644 --- a/crates/mq-markdown/src/parser/tree.rs +++ b/crates/mq-markdown/src/parser/tree.rs @@ -4,7 +4,6 @@ //! it runs in [`super::resolve`] once all blocks are known. use super::error::MdxError; -use super::line::visual_column; use super::mdx::TagKind; use super::scan::eol_len; use crate::node::{HeadingDepth, ListMarker, MdxAttributeContent, Node, Point, Position, TableAlignKind}; @@ -24,8 +23,9 @@ pub(super) struct InlineSource { pub(super) lines: Vec<LineStart>, /// Whether this is the content of a table cell, where `\|` in code stands for `|`. pub(super) table: bool, - /// Whether `text` contains a tab, which makes columns differ from byte offsets. - tabs: bool, + /// The tabs in `text` and the document column right after each, which make columns differ from + /// byte offsets. + tabs: Vec<(usize, usize)>, } impl InlineSource { @@ -44,11 +44,32 @@ impl InlineSource { text.push_str(eol); } } - Self { - tabs: text.contains('\t'), + let mut source = Self { + tabs: Vec::new(), text, lines: starts, table: false, + }; + source.index_tabs(); + source + } + + /// Finds the tabs of every line and the column after them. + fn index_tabs(&mut self) { + self.tabs.clear(); + if !self.text.contains('\t') { + return; + } + for (index, line) in self.lines.iter().enumerate() { + let end = self.lines.get(index + 1).map_or(self.text.len(), |next| next.offset); + // The offset and the zero-based column right after the last tab or line start. + let (mut anchor, mut anchor_column) = (line.offset, line.point.column - 1); + for (at, _) in self.text[line.offset..end].match_indices('\t') { + let at = line.offset + at; + let after = ((anchor_column + (at - anchor)) / 4 + 1) * 4; + self.tabs.push((at, after)); + (anchor, anchor_column) = (at + 1, after); + } } } @@ -81,7 +102,6 @@ impl InlineSource { fn drain_to(&mut self, end: usize) { let new_point = self.point(end); self.text.replace_range(..end, ""); - self.tabs = self.text.contains('\t'); let index = self.line_index(end); let first = LineStart { @@ -96,6 +116,7 @@ impl InlineSource { point: line.point.clone(), }); self.lines = std::iter::once(first).chain(rest).collect(); + self.index_tabs(); } /// The index of the line that `offset` is on. @@ -121,11 +142,10 @@ impl InlineSource { /// The position of the byte at `offset` in `text`. pub(super) fn point(&self, offset: usize) -> Point { let LineStart { offset: start, point } = &self.lines[self.line_index(offset)]; - let passed = &self.text[*start..offset]; - let column = if self.tabs && passed.contains('\t') { - visual_column(passed.chars(), point.column - 1) + 1 - } else { - point.column + passed.len() + let passed = self.tabs.partition_point(|&(at, _)| at < offset); + let column = match passed.checked_sub(1).map(|index| self.tabs[index]) { + Some((at, after)) if at >= *start => after + (offset - at - 1) + 1, + _ => point.column + (offset - start), }; Point { line: point.line, diff --git a/crates/mq-markdown/tests/parser_limits_tests.rs b/crates/mq-markdown/tests/parser_limits_tests.rs index a0beb0747..e24376667 100644 --- a/crates/mq-markdown/tests/parser_limits_tests.rs +++ b/crates/mq-markdown/tests/parser_limits_tests.rs @@ -77,6 +77,9 @@ fn elements_nested_within_the_limit_parse() { #[case::many_emphasis_pairs_and_an_email("*a* b@c.de ".repeat(30_000))] #[case::backtick_runs_of_every_length((1..3_000).map(|length| format!("e{}", "`".repeat(length))).collect::<String>())] #[case::dollar_runs_of_every_length((1..3_000).map(|length| format!("e{}", "$".repeat(length))).collect::<String>())] +#[case::tabs_between_inlines(format!("x{}\n", "\t*a* ".repeat(40_000)))] +#[case::tabs_between_inlines_in_a_quote(format!("> x{}\n", "\t*a* ".repeat(40_000)))] +#[case::tabs_between_inlines_in_a_table(format!("|a|\n|-|\n|x{}|\n", "\t*a* ".repeat(40_000)))] #[case::unclosed_comments("a <!--".repeat(300_000))] #[case::unclosed_comments_after_a_closer(format!("</{}", "<!--".repeat(300_000)))] #[case::unclosed_instructions("a <?".repeat(300_000))] From 8737baefff4a1974a4df9d4c025e1bd4261a2a7d Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Fri, 9 Oct 2026 12:08:00 +0900 Subject: [PATCH 55/88] =?UTF-8?q?=F0=9F=90=9B=20fix(markdown):=20end=20an?= =?UTF-8?q?=20empty=20list=20item=20at=20its=20last=20content=20line?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An empty task item took the blank lines after it into its position, so the writer joined the next paragraph to it and a reparse made that paragraph the item's content. --- crates/mq-markdown/src/markdown.rs | 3 +++ crates/mq-markdown/src/parser/block.rs | 7 +++---- crates/mq-markdown/src/parser/resolve.rs | 5 ++++- crates/mq-markdown/src/parser/tree.rs | 2 ++ 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/crates/mq-markdown/src/markdown.rs b/crates/mq-markdown/src/markdown.rs index 73792cb1e..673c52154 100644 --- a/crates/mq-markdown/src/markdown.rs +++ b/crates/mq-markdown/src/markdown.rs @@ -567,6 +567,9 @@ mod tests { #[case::star_rule_under_text_needs_no_blank_line("Foo\n***\n", 2, "Foo\n***\n")] // items whose marker sits alone on its own line must stay tight, not go loose #[case::list_tight_despite_marker_on_own_line("-\n foo\n-\n bar", 2, "- foo\n- bar\n")] + // an empty task item must not take the paragraph after it + #[case::empty_task_item_then_paragraph("- [ ] \n\npara", 2, "- [ ] \n\npara\n")] + #[case::empty_checked_ordered_task_item_then_paragraph("1. [x] \n\npara", 2, "1. [x] \n\npara\n")] // an indented code block right after a list must not read back as list continuation #[case::code_after_list_uses_fence(" - one\n\n two", 2, "- one\n\n```\n two\n```\n")] // a block quote nested in a list item needs its continuation line's own indent diff --git a/crates/mq-markdown/src/parser/block.rs b/crates/mq-markdown/src/parser/block.rs index 3c557359f..f883d87f5 100644 --- a/crates/mq-markdown/src/parser/block.rs +++ b/crates/mq-markdown/src/parser/block.rs @@ -683,12 +683,10 @@ fn list_item( last.item_end = item_end; } + let content_end = inner.last().map_or_else(|| lines[start].end(), Line::end); let position = Position { start: lines[start].point(0), - end: blanks - .last() - .or(inner.last()) - .map_or_else(|| lines[start].end(), Line::end), + end: blanks.last().map_or_else(|| content_end.clone(), Line::end), }; let mut children = parse_blocks(&inner, depth + 1, interrupting); // The checkbox is the start of the text of the first paragraph. @@ -704,6 +702,7 @@ fn list_item( checked, children, position, + content_end, }; (item, index) diff --git a/crates/mq-markdown/src/parser/resolve.rs b/crates/mq-markdown/src/parser/resolve.rs index a11b13672..1d5cd5374 100644 --- a/crates/mq-markdown/src/parser/resolve.rs +++ b/crates/mq-markdown/src/parser/resolve.rs @@ -244,7 +244,10 @@ fn list_nodes(list: ListBlock, level: Level, doc: Document<'_>, nodes: &mut Vec< start: first.start.clone(), end: last.end.clone(), }, - _ => item.position, + _ => Position { + start: item.position.start, + end: item.content_end, + }, }; nodes.push(Node::List(List { diff --git a/crates/mq-markdown/src/parser/tree.rs b/crates/mq-markdown/src/parser/tree.rs index 23aec4e9c..43ec22353 100644 --- a/crates/mq-markdown/src/parser/tree.rs +++ b/crates/mq-markdown/src/parser/tree.rs @@ -214,6 +214,8 @@ pub(super) struct Item { pub(super) checked: Option<bool>, pub(super) children: Vec<Block>, pub(super) position: Position, + /// Where the last line of the item ends, before the blank lines that `position` runs over. + pub(super) content_end: Point, } pub(super) enum TableItem { From 6f8ab76126fd03ba8aaf553d38c8c104cea96fd2 Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Fri, 9 Oct 2026 12:13:00 +0900 Subject: [PATCH 56/88] =?UTF-8?q?=F0=9F=90=9B=20fix(markdown):=20keep=20pa?= =?UTF-8?q?ragraphs=20apart=20in=20blocks=20without=20positions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Without positions, the paragraphs of a list item or a quote were written back to back and read as one. Text that follows text can only start a new paragraph, so it gets a blank line. --- crates/mq-markdown/src/markdown.rs | 12 ++++++++++++ crates/mq-markdown/src/node/render.rs | 12 +++++++++++- crates/mq-markdown/src/node/tests.rs | 4 ++-- 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/crates/mq-markdown/src/markdown.rs b/crates/mq-markdown/src/markdown.rs index 673c52154..8c7d1e45b 100644 --- a/crates/mq-markdown/src/markdown.rs +++ b/crates/mq-markdown/src/markdown.rs @@ -651,6 +651,18 @@ mod tests { assert_eq!(first, "[https://example.com](https://example.com)\n"); } + #[rstest] + #[case::paragraphs_in_an_item("- a\n\n b\n\n c\n", "- a\n\n b\n\n c\n")] + // the order of a paragraph after a nested list is lost, but its content is not + #[case::paragraph_after_a_nested_list("- a\n\n - n\n\n para\n", "- a\n\n para\n - n\n")] + #[case::paragraphs_in_an_ordered_item("1. a\n\n b\n", "1. a\n\n b\n")] + #[case::paragraphs_in_a_quote_in_an_item("- > a\n >\n > b\n", "- > a\n > \n > b\n")] + fn test_paragraphs_stay_apart_without_positions(#[case] input: &str, #[case] expected: &str) { + let mut md = Markdown::from_markdown_str(input).unwrap(); + md.nodes.iter_mut().for_each(Node::strip_positions); + assert_eq!(md.to_string(), expected); + } + #[test] fn test_markdown_to_html() { let md = "# Hello".parse::<Markdown>().unwrap(); diff --git a/crates/mq-markdown/src/node/render.rs b/crates/mq-markdown/src/node/render.rs index 3691a73ed..88e883ae7 100644 --- a/crates/mq-markdown/src/node/render.rs +++ b/crates/mq-markdown/src/node/render.rs @@ -686,10 +686,12 @@ pub(crate) fn render_values_block(values: &[Node], options: &RenderOptions, them let mut result = String::new(); let mut pre_position: Option<Position> = None; let mut index = 0; + // The last node written, to tell paragraphs from the inline nodes of one when there are no positions. + let mut previous: Option<&Node> = None; while index < values.len() { // The cells of a table are laid out together, as at the top level. - let (rendered, position, next) = if is_table_part(&values[index]) { + let (rendered, position, next, last) = if is_table_part(&values[index]) { let end = values[index..] .iter() .position(|node| !is_table_part(node)) @@ -710,6 +712,7 @@ pub(crate) fn render_values_block(values: &[Node], options: &RenderOptions, them table.render_with_theme(theme).trim_end_matches('\n').to_string(), position, end, + &run[run.len() - 1], ) } else { let value = &values[index]; @@ -717,8 +720,10 @@ pub(crate) fn render_values_block(values: &[Node], options: &RenderOptions, them render_before(value, values.get(index + 1), options, theme), value.position(), index + 1, + value, ) }; + let first = &values[index]; index = next; if let Some(pos) = position { @@ -729,8 +734,13 @@ pub(crate) fn render_values_block(values: &[Node], options: &RenderOptions, them pre_position = Some(pos); result.push_str(&"\n".repeat(new_line_count)); } else { + // Text never follows text inside one paragraph, so without positions that is a new one. + if pre_position.is_none() && matches!((previous, first), (Some(Node::Text(_)), Node::Text(_))) { + result.push_str("\n\n"); + } pre_position = None; } + previous = Some(last); result.push_str(&rendered); } result diff --git a/crates/mq-markdown/src/node/tests.rs b/crates/mq-markdown/src/node/tests.rs index 1aba7116d..7252437e8 100644 --- a/crates/mq-markdown/src/node/tests.rs +++ b/crates/mq-markdown/src/node/tests.rs @@ -2554,7 +2554,7 @@ fn test_attr_value_as_i64(#[case] value: AttrValue, #[case] expected: Option<i64 position: None }), "> [!INFO]\n> line one\n> line two" )] -// two separate position-less Text values are concatenated inline (no implicit newline) +// two separate position-less Text values are paragraphs: the inline nodes of one never are two Texts #[case( Node::Callout(Callout { fold: None, kind: "INFO".to_string(), title: None, values: vec![ @@ -2562,7 +2562,7 @@ fn test_attr_value_as_i64(#[case] value: AttrValue, #[case] expected: Option<i64 Node::Text(Text { value: "part b".to_string(), position: None }), ], position: None }), - "> [!INFO]\n> part apart b" + "> [!INFO]\n> part a\n> \n> part b" )] fn test_callout_render(#[case] node: Node, #[case] expected: &str) { assert_eq!(node.to_string_with(&RenderOptions::default()), expected); From 5cee621121376b9fede1690042a5c32b6dea2c7c Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Fri, 9 Oct 2026 12:18:00 +0900 Subject: [PATCH 57/88] =?UTF-8?q?=F0=9F=90=9B=20fix(markdown):=20render=20?= =?UTF-8?q?the=20children=20of=20JSX=20elements=20as=20Markdown=20in=20HTM?= =?UTF-8?q?L?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The HTML output is made by reading the written Markdown again. The tag lines and their children were not separated by blank lines, so they read as one HTML block that holds raw text. --- crates/mq-markdown/src/markdown.rs | 2 +- crates/mq-markdown/src/node.rs | 4 ++-- crates/mq-markdown/src/node/render.rs | 22 +++++++++++++++++++ .../mq-markdown/tests/html_rendering_tests.rs | 11 ++++++++++ 4 files changed, 36 insertions(+), 3 deletions(-) diff --git a/crates/mq-markdown/src/markdown.rs b/crates/mq-markdown/src/markdown.rs index 8c7d1e45b..dfb9305b0 100644 --- a/crates/mq-markdown/src/markdown.rs +++ b/crates/mq-markdown/src/markdown.rs @@ -313,7 +313,7 @@ impl Markdown { } pub fn to_html(&self) -> String { - to_html(&self.to_string()) + to_html(&crate::node::for_html(|| self.to_string())) } pub fn to_text(&self) -> String { diff --git a/crates/mq-markdown/src/node.rs b/crates/mq-markdown/src/node.rs index fab572755..8e90558fe 100644 --- a/crates/mq-markdown/src/node.rs +++ b/crates/mq-markdown/src/node.rs @@ -16,8 +16,8 @@ pub mod attr_value; mod expand; mod render; pub(crate) use render::{ - indent_lines, list_own_prefix_width, reindent_all_lines, render_before, render_cell_values, render_values_block, - values_to_string, + for_html, indent_lines, list_own_prefix_width, reindent_all_lines, render_before, render_cell_values, + render_values_block, values_to_string, }; use render::{render_link_destination, render_link_title}; #[cfg(test)] diff --git a/crates/mq-markdown/src/node/render.rs b/crates/mq-markdown/src/node/render.rs index 88e883ae7..c330e6cf2 100644 --- a/crates/mq-markdown/src/node/render.rs +++ b/crates/mq-markdown/src/node/render.rs @@ -1,6 +1,24 @@ //! Rendering nodes back to Markdown. use super::*; +use std::cell::Cell; + +thread_local! { + /// Set while the output is read again as Markdown to make HTML of it, not as MDX. + static HTML_TARGET: Cell<bool> = const { Cell::new(false) }; +} + +/// Runs `render` with the JSX elements written so that their children are Markdown in HTML. +pub(crate) fn for_html<R>(render: impl FnOnce() -> R) -> R { + struct Reset(bool); + impl Drop for Reset { + fn drop(&mut self) { + HTML_TARGET.set(self.0); + } + } + let _reset = Reset(HTML_TARGET.replace(true)); + render() +} impl Node { pub fn to_string_with(&self, options: &RenderOptions) -> String { @@ -373,6 +391,10 @@ impl Node { let children = render_values_block(&mdx_jsx_flow_element.children, options, theme); // The lines of an expression are indented as they are written, which indenting them // again would add to each time they are read. + if HTML_TARGET.get() { + // Blank lines end the tags as HTML blocks, so the children are Markdown. + return format!("<{}{}>\n\n{}\n\n</{}>", name, attributes, children, name); + } let children = if mdx_jsx_flow_element.children.iter().any(has_multiline_expression) { children } else { diff --git a/crates/mq-markdown/tests/html_rendering_tests.rs b/crates/mq-markdown/tests/html_rendering_tests.rs index 8e6b9bc87..2c6a8ef34 100644 --- a/crates/mq-markdown/tests/html_rendering_tests.rs +++ b/crates/mq-markdown/tests/html_rendering_tests.rs @@ -454,3 +454,14 @@ fn to_html_of_a_document_equals_to_html_of_its_source(#[case] input: &str) { let document: Markdown = input.parse().unwrap(); assert_eq!(document.to_html(), to_html(input)); } + +#[rstest] +#[case::children_are_markdown("<Card title=\"a\">\ntext *em*\n</Card>\n", &["<Card title=\"a\">", "<p>text <em>em</em></p>", "</Card>"])] +#[case::nested_elements("<A>\n<B>\n*x*\n</B>\n</A>\n", &["<A>", "<B>", "<p><em>x</em></p>", "</B>", "</A>"])] +#[case::in_a_list_item("- <A>\n *x*\n </A>\n", &["<A>", "<p><em>x</em></p>", "</A>"])] +fn mdx_jsx_flow_children_are_rendered_as_markdown(#[case] input: &str, #[case] expected: &[&str]) { + let html = Markdown::from_mdx_str(input).unwrap().to_html(); + for part in expected { + assert!(html.lines().any(|line| line == *part), "{part:?} in {html:?}"); + } +} From 6591417447b3f5afd19ebfedb73e3941c54cc42e Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Fri, 9 Oct 2026 12:23:00 +0900 Subject: [PATCH 58/88] =?UTF-8?q?=E2=9C=85=20test(markdown):=20run=20the?= =?UTF-8?q?=20spec=20tests=20with=20wikilinks=20and=20embeds,=20and=20week?= =?UTF-8?q?ly=20in=20CI?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [[...]] is read as a wikilink or an embed when those features are on, which the specs do not have. The known differences list them by feature, and test-spec runs both configurations. Also fixes the typo "lazyness" in the mdast tests. --- .github/workflows/gfm-spec.yml | 4 +++- crates/mq-markdown/src/parser.rs | 31 +++++++++++++++++++++++-- crates/mq-markdown/tests/mdast_tests.rs | 8 +++---- justfile | 1 + 4 files changed, 37 insertions(+), 7 deletions(-) diff --git a/.github/workflows/gfm-spec.yml b/.github/workflows/gfm-spec.yml index 60d34e313..1e4d74d5f 100644 --- a/.github/workflows/gfm-spec.yml +++ b/.github/workflows/gfm-spec.yml @@ -1,4 +1,4 @@ -name: GFM spec round-trip fidelity +name: Markdown spec conformance on: schedule: @@ -21,5 +21,7 @@ jobs: with: just-version: 1.58.0 - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - name: Render the spec examples + run: just test-spec - name: Run GFM spec round-trip check run: just test-gfm-spec diff --git a/crates/mq-markdown/src/parser.rs b/crates/mq-markdown/src/parser.rs index a128dfcc1..6621cec62 100644 --- a/crates/mq-markdown/src/parser.rs +++ b/crates/mq-markdown/src/parser.rs @@ -622,6 +622,29 @@ mod tests { 607, 608, ]; + /// Examples of the GFM spec with `[[...]]` in them that the `wikilink` feature reads as a wikilink + /// (544, 555), and the one that either the `wikilink` or the `embed` feature changes (586). + const GFM_WIKILINK_DIFFERENCES: &[usize] = &[544, 555]; + const GFM_OBSIDIAN_DIFFERENCES: &[usize] = &[586]; + + /// The same for the `CommonMark` 0.31.2 spec (548, 559 and 590). + const COMMONMARK_WIKILINK_DIFFERENCES: &[usize] = &[548, 559]; + const COMMONMARK_OBSIDIAN_DIFFERENCES: &[usize] = &[590]; + + /// The examples that differ from the spec: `known`, and those that the enabled features read as + /// wikilinks and embeds. + fn expected_differences(known: &[usize], wikilink: &[usize], either: &[usize]) -> Vec<usize> { + let mut all = known.to_vec(); + if cfg!(feature = "wikilink") { + all.extend_from_slice(wikilink); + } + if cfg!(any(feature = "wikilink", feature = "embed")) { + all.extend_from_slice(either); + } + all.sort_unstable(); + all + } + /// Examples of the `CommonMark` 0.31.2 spec that are not rendered as in the spec, by number: /// /// - a document that starts with `---` has frontmatter (96, 98) @@ -665,7 +688,7 @@ mod tests { fn spec_examples_render_as_in_the_spec() { assert_eq!( differing_examples(&spec_text("SPEC_FILE", SPEC_URL), 600), - KNOWN_DIFFERENCES, + expected_differences(KNOWN_DIFFERENCES, GFM_WIKILINK_DIFFERENCES, GFM_OBSIDIAN_DIFFERENCES), "examples that are not rendered as in the spec" ); } @@ -677,7 +700,11 @@ mod tests { fn commonmark_spec_examples_render_as_in_the_spec() { assert_eq!( differing_examples(&spec_text("COMMONMARK_SPEC_FILE", COMMONMARK_SPEC_URL), 600), - COMMONMARK_KNOWN_DIFFERENCES, + expected_differences( + COMMONMARK_KNOWN_DIFFERENCES, + COMMONMARK_WIKILINK_DIFFERENCES, + COMMONMARK_OBSIDIAN_DIFFERENCES + ), "examples that are not rendered as in the spec" ); } diff --git a/crates/mq-markdown/tests/mdast_tests.rs b/crates/mq-markdown/tests/mdast_tests.rs index aa294ac4c..a7e38fd26 100644 --- a/crates/mq-markdown/tests/mdast_tests.rs +++ b/crates/mq-markdown/tests/mdast_tests.rs @@ -463,9 +463,9 @@ fn check(input: &str, expected: &[&str], parse: fn(&str) -> miette::Result<Markd #[case(" { a }\nb", &["flowexpr { a }", r#"T"b""#])] // should support lists after non-expressions (wooorm/markdown-rs#11) #[case("a\n\n* b", &[r#"T"a""#, "item 0 u - t(", r#"T"b""#, ")"])] -// should not support lazyness (2) +// should not support laziness (2) #[case("> a\n{b}", &["blockquote(", r#"T"a""#, ")", "flowexpr {b}"])] -// should not support lazyness (3) +// should not support laziness (3) #[case("> {a}\nb", &["blockquote(", "flowexpr {a}", ")", r#"T"b""#])] // should support mdx expressions (flow) as `MdxFlowExpression`s in mdast #[case("{alpha +\nbravo}", &["flowexpr {alpha +\nbravo}"])] @@ -847,9 +847,9 @@ fn mdx(#[case] input: &str, #[case] expected: &[&str]) { #[case("{a")] // should crash if no closing brace is found (2) #[case("{b { c }")] -// should not support lazyness (1) +// should not support laziness (1) #[case("> {a\nb}")] -// should not support lazyness (4) +// should not support laziness (4) #[case("> {\n> a\nb}")] // should crash on an incorrect spread that looks like an assignment #[case("<a {b=c}={} d>")] diff --git a/justfile b/justfile index 8ddf67b24..639c30204 100644 --- a/justfile +++ b/justfile @@ -127,6 +127,7 @@ test-gfm-spec: # Render every example of the CommonMark, GFM and GFM extensions specs (fetches them over the network) test-spec: cargo test -p mq-markdown --lib -- --ignored spec_examples + cargo test -p mq-markdown --lib --features obsidian -- --ignored spec_examples test-doc: cargo test --doc --workspace From ecc45e389d06ab4f2ddf568b5fdb63afa1fa1e38 Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Fri, 9 Oct 2026 12:28:00 +0900 Subject: [PATCH 59/88] =?UTF-8?q?=F0=9F=90=9B=20fix(markdown):=20keep=20th?= =?UTF-8?q?e=20table=20run=20out=20of=20the=20recursive=20block=20renderer?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The paragraph separation grew the frame of render_values_block enough for a quote nested 20000 deep to overflow the stack of a test thread with all features on. --- crates/mq-markdown/src/node/render.rs | 72 ++++++++++++++++----------- 1 file changed, 42 insertions(+), 30 deletions(-) diff --git a/crates/mq-markdown/src/node/render.rs b/crates/mq-markdown/src/node/render.rs index c330e6cf2..67d1214f8 100644 --- a/crates/mq-markdown/src/node/render.rs +++ b/crates/mq-markdown/src/node/render.rs @@ -704,7 +704,6 @@ pub(crate) fn render_values(values: &[Node], options: &RenderOptions, theme: &Co /// so applying column-based spacing causes double-indentation. This variant preserves /// blank lines between values (via newline counting) but ignores the column position. pub(crate) fn render_values_block(values: &[Node], options: &RenderOptions, theme: &ColorTheme<'_>) -> String { - let is_table_part = |node: &Node| matches!(node, Node::TableCell(_) | Node::TableAlign(_)); let mut result = String::new(); let mut pre_position: Option<Position> = None; let mut index = 0; @@ -712,40 +711,17 @@ pub(crate) fn render_values_block(values: &[Node], options: &RenderOptions, them let mut previous: Option<&Node> = None; while index < values.len() { + let first = &values[index]; // The cells of a table are laid out together, as at the top level. - let (rendered, position, next, last) = if is_table_part(&values[index]) { - let end = values[index..] - .iter() - .position(|node| !is_table_part(node)) - .map_or(values.len(), |offset| index + offset); - let run = &values[index..end]; - let table = crate::Markdown { - nodes: run.to_vec(), - options: options.clone(), - }; - let position = match (run[0].position(), run[run.len() - 1].position()) { - (Some(first), Some(last)) => Some(Position { - start: first.start, - end: last.end, - }), - _ => None, - }; - ( - table.render_with_theme(theme).trim_end_matches('\n').to_string(), - position, - end, - &run[run.len() - 1], - ) + let (rendered, position, next) = if is_table_part(first) { + render_table_run(values, index, options, theme) } else { - let value = &values[index]; ( - render_before(value, values.get(index + 1), options, theme), - value.position(), + render_before(first, values.get(index + 1), options, theme), + first.position(), index + 1, - value, ) }; - let first = &values[index]; index = next; if let Some(pos) = position { @@ -762,12 +738,48 @@ pub(crate) fn render_values_block(values: &[Node], options: &RenderOptions, them } pre_position = None; } - previous = Some(last); + previous = Some(&values[index - 1]); result.push_str(&rendered); } result } +fn is_table_part(node: &Node) -> bool { + matches!(node, Node::TableCell(_) | Node::TableAlign(_)) +} + +/// Renders the cells and the delimiter row from `values[index]` on as one table, with its position and +/// the index after it. Apart from the recursion, so that nesting does not pay for its locals. +#[inline(never)] +fn render_table_run( + values: &[Node], + index: usize, + options: &RenderOptions, + theme: &ColorTheme<'_>, +) -> (String, Option<Position>, usize) { + let end = values[index..] + .iter() + .position(|node| !is_table_part(node)) + .map_or(values.len(), |offset| index + offset); + let run = &values[index..end]; + let table = crate::Markdown { + nodes: run.to_vec(), + options: options.clone(), + }; + let position = match (run[0].position(), run[run.len() - 1].position()) { + (Some(first), Some(last)) => Some(Position { + start: first.start, + end: last.end, + }), + _ => None, + }; + ( + table.render_with_theme(theme).trim_end_matches('\n').to_string(), + position, + end, + ) +} + /// Renders a link/image destination, auto-upgrading to `<...>` (escaping `\ < >`) /// when the bare form would be unsafe (empty, or contains whitespace/control/`< >`); /// otherwise bare, escaping `\ ( )` since unescaped parens must balance. From 789da0f8119887fb5d239593bcc7bed01ae84fe1 Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Fri, 9 Oct 2026 12:33:00 +0900 Subject: [PATCH 60/88] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor(markdown):?= =?UTF-8?q?=20share=20the=20lazy=20continuation=20checks=20with=20the=20bl?= =?UTF-8?q?ock=20rules?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The line start check for MDX flow content is one function for the rule and both continuation checks. Quotes now track footnote definitions as containers, and a line with fewer containers than the open paragraph is a lazy line for its inner ones. Property tests check that a quote and plain lines agree on whether a lazy line joins the paragraph. --- crates/mq-markdown/src/parser/block.rs | 11 +- crates/mq-markdown/src/parser/block/rule.rs | 4 +- crates/mq-markdown/src/parser/mdx_flow.rs | 23 ++-- .../mq-markdown/tests/html_rendering_tests.rs | 11 ++ .../tests/parser_property_tests.rs | 101 +++++++++++++++++- 5 files changed, 133 insertions(+), 17 deletions(-) diff --git a/crates/mq-markdown/src/parser/block.rs b/crates/mq-markdown/src/parser/block.rs index f883d87f5..c65db8f3d 100644 --- a/crates/mq-markdown/src/parser/block.rs +++ b/crates/mq-markdown/src/parser/block.rs @@ -304,6 +304,12 @@ impl<'a> LeafState<'a> { containers += 1; continue; } + Some(LineStart::Footnote(marker)) if containers < self.budget => { + line = line.skip(marker.content); + containers += 1; + self.paragraph = false; + continue; + } // A marker in or inside the container of the open paragraph has to be able to interrupt it. Some(LineStart::Item(marker)) if containers < self.budget @@ -325,8 +331,9 @@ impl<'a> LeafState<'a> { return; } self.header = rest.contains('|').then(|| table::row_cells(&line)); - // A lazy line belongs to the paragraph that is already open, in its container. - if !(line.lazy && self.paragraph) { + // A lazy line belongs to the paragraph that is already open, in its container. So does a + // line that has fewer containers than that paragraph, as it is lazy for the inner ones. + if !(self.paragraph && (line.lazy || containers < self.paragraph_containers)) { self.paragraph_containers = containers; } self.paragraph = true; diff --git a/crates/mq-markdown/src/parser/block/rule.rs b/crates/mq-markdown/src/parser/block/rule.rs index a3f0bdb74..902345ce0 100644 --- a/crates/mq-markdown/src/parser/block/rule.rs +++ b/crates/mq-markdown/src/parser/block/rule.rs @@ -8,7 +8,7 @@ use super::{Interrupt, LineStart, atx_heading, blockquote, footnote, html_block, use crate::node::{HorizontalRule, HorizontalRuleMarker, MdxJsEsm, Node, Position}; use crate::parser::code::{fenced_code, indented_code}; use crate::parser::line::{Indent, Line}; -use crate::parser::mdx_flow::{FlowOutcome, mdx_flow}; +use crate::parser::mdx_flow::{FlowOutcome, may_start_flow, mdx_flow}; use crate::parser::table; use crate::parser::tree::Block; @@ -313,7 +313,7 @@ pub(super) struct MdxFlow; impl BlockRule for MdxFlow { fn parse(cx: &Cx<'_, '_>, blocks: &mut Vec<Block>) -> Option<Step> { - if !(cx.line.flavor.has_jsx() && matches!(cx.rest.as_bytes().first(), Some(b'<' | b'{'))) { + if !may_start_flow(cx.line) { return None; } let FlowOutcome::Flow(next) = mdx_flow(cx.lines, cx.index, blocks) else { diff --git a/crates/mq-markdown/src/parser/mdx_flow.rs b/crates/mq-markdown/src/parser/mdx_flow.rs index 92f033f17..f74af40c6 100644 --- a/crates/mq-markdown/src/parser/mdx_flow.rs +++ b/crates/mq-markdown/src/parser/mdx_flow.rs @@ -167,25 +167,24 @@ pub(super) fn blocks_lazy_continuation(lines: &[Line<'_>], index: usize) -> bool ) } -pub(super) fn probe_mdx_flow(lines: &[Line<'_>], index: usize) -> Option<FlowOutcome> { - let line = &lines[index]; - (line.flavor.has_jsx() - && !line.lazy +/// Whether `line` can start flow content: it is MDX and begins with a tag or an expression. The rule +/// that parses flow content and the checks of what a paragraph can continue all start from this. +pub(super) fn may_start_flow(line: &Line<'_>) -> bool { + line.flavor.has_jsx() && matches!( line.text.trim_start_matches([' ', '\t']).as_bytes().first(), Some(b'<' | b'{') - )) - .then(|| mdx_flow(lines, index, &mut Vec::new())) + ) +} + +pub(super) fn probe_mdx_flow(lines: &[Line<'_>], index: usize) -> Option<FlowOutcome> { + let line = &lines[index]; + (may_start_flow(line) && !line.lazy).then(|| mdx_flow(lines, index, &mut Vec::new())) } /// Whether a single MDX line looks like flow content, for tracking what a paragraph can continue. pub(super) fn looks_like_mdx_flow(line: &Line<'_>) -> bool { - if !line.flavor.has_jsx() - || !matches!( - line.text.trim_start_matches([' ', '\t']).as_bytes().first(), - Some(b'<' | b'{') - ) - { + if !may_start_flow(line) { return false; } // The line ending counts, unless the document ends here. diff --git a/crates/mq-markdown/tests/html_rendering_tests.rs b/crates/mq-markdown/tests/html_rendering_tests.rs index 2c6a8ef34..425203907 100644 --- a/crates/mq-markdown/tests/html_rendering_tests.rs +++ b/crates/mq-markdown/tests/html_rendering_tests.rs @@ -465,3 +465,14 @@ fn mdx_jsx_flow_children_are_rendered_as_markdown(#[case] input: &str, #[case] e assert!(html.lines().any(|line| line == *part), "{part:?} in {html:?}"); } } + +#[rstest] +#[case::footnote_in_a_quote( + "> [^n]: f\n> ===\nlazy\n\n[^n]\n", + "<p>f\n===\nlazy <a href=\"#user-content-fnref-n\"" +)] +#[case::line_lazy_for_an_inner_quote("> > q\n> a\n> ===\nlazy\n", "<p>q\na\n===\nlazy</p>")] +fn lazy_lines_continue_the_paragraph_of_the_innermost_container(#[case] input: &str, #[case] expected: &str) { + let html = to_html(input); + assert!(html.contains(expected), "{expected:?} in {html:?}"); +} diff --git a/crates/mq-markdown/tests/parser_property_tests.rs b/crates/mq-markdown/tests/parser_property_tests.rs index ca605c55b..4e52c6e01 100644 --- a/crates/mq-markdown/tests/parser_property_tests.rs +++ b/crates/mq-markdown/tests/parser_property_tests.rs @@ -4,7 +4,7 @@ //! letters do not change the structure, every position is on a character boundary, and the HTML is //! well-formed with safe URLs. -use mq_markdown::{Markdown, Node, to_html}; +use mq_markdown::{Markdown, Node, ParseOptions, to_html}; use proptest::prelude::*; fn parse(input: &str) -> Vec<Node> { @@ -563,3 +563,102 @@ proptest! { prop_assert!(positions.windows(2).all(|pair| pair[0] < pair[1]), "{html}"); } } + +/// Lines that start, continue and end the blocks that decide whether a paragraph is open. Footnote +/// definitions are left out: a quote hoists them, so their text is not found in the same place. +/// Definitions are left out too: a quote does not know that a paragraph is only definitions, so it reads a line of `=` +/// under one as a setext underline, where outside a quote it is text. Indented code and an +/// empty item are left out for the same reason as blank lines after an item. +const LAZY_LINES: &[&str] = &[ + "a", "b c", "", "# h", "***", "---", "===", "> q", "- i", "1. i", "2. i", " more", "```", "~~~", "$$", "<div>", + "</div>", "<!-- c", "-->", "<?x", "?>", "<p>", "<span>", +]; + +/// The lines of MDX that start, continue and end flow content. A fence is left out of the lines with +/// them: it is text inside an expression that spans lines, which a quote cannot tell line by line. +const LAZY_MDX_LINES: &[&str] = &[ + "<A>", + "</A>", + "<A />", + "<A", + "b>", + "<A b={", + "}>", + "{x}", + "{", + "{x} <B />", + "<A> t", + "t {x}", + "<a b='", +]; + +/// Whether the text `lazy` has been joined to the paragraph before it. +fn lazy_is_joined(nodes: Vec<Node>) -> bool { + walk(nodes) + .iter() + .any(|node| matches!(node, Node::Text(text) if text.value.contains("\nlazy"))) +} + +fn quoted(lines: &[&str]) -> String { + lines + .iter() + .map(|line| format!(">{}{line}", if line.is_empty() { "" } else { " " })) + .collect::<Vec<_>>() + .join("\n") +} + +/// Block parsing tracks which paragraph is open in a quote on its own, to tell lazy continuation. It has +/// to decide as parsing the lines without the quote does. +fn lazy_line_agrees(lines: &[&str], parse: impl Fn(&str) -> Option<Vec<Node>>) -> Result<(), TestCaseError> { + if lines.last().is_none_or(|line| line.is_empty()) { + return Ok(()); + } + // A quote does not track the indentation of the items that are open, so it reads a line that an + // item holds after a blank line as code, or as text where the item has a setext underline. + let first_item = lines + .iter() + .position(|line| line.starts_with("- ") || line.starts_with(char::is_numeric)); + if first_item.is_some_and(|item| lines[item..].contains(&"")) { + return Ok(()); + } + let plain = format!("{}\nlazy", lines.join("\n")); + let in_quote = format!("{}\nlazy", quoted(lines)); + // MDX does not allow a lazy line inside flow content, which is an error only in a quote. + let (Some(in_quote_nodes), Some(plain_nodes)) = (parse(&in_quote), parse(&plain)) else { + return Ok(()); + }; + prop_assert_eq!( + lazy_is_joined(in_quote_nodes), + lazy_is_joined(plain_nodes), + "{:?} against {:?}", + in_quote, + plain + ); + Ok(()) +} + +proptest! { + #[test] + fn lazy_continuation_in_a_quote_agrees_with_a_paragraph_outside( + lines in prop::collection::vec(prop::sample::select(LAZY_LINES), 1..6) + ) { + lazy_line_agrees(&lines, |input| { + Markdown::from_markdown_str_with(input, ParseOptions { frontmatter: false }).ok().map(|md| md.nodes) + })?; + } + + #[test] + fn lazy_continuation_in_a_quote_agrees_with_a_paragraph_outside_in_mdx( + lines in prop::collection::vec( + prop_oneof![ + prop::sample::select(LAZY_LINES).prop_filter("a fence", |line| !matches!(*line, "```" | "~~~" | "$$")), + prop::sample::select(LAZY_MDX_LINES) + ], + 1..6 + ) + ) { + lazy_line_agrees(&lines, |input| { + Markdown::from_mdx_str_with(input, ParseOptions { frontmatter: false }).ok().map(|md| md.nodes) + })?; + } +} From 708c6c8cd909d3d32cd2214df1abc69b63b65c7f Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Fri, 9 Oct 2026 12:38:00 +0900 Subject: [PATCH 61/88] =?UTF-8?q?=E2=9C=85=20test(markdown):=20check=20tha?= =?UTF-8?q?t=20the=20HTML=20shows=20what=20the=20callout=20and=20link=20no?= =?UTF-8?q?des=20hold?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The HTML renderer already reads callout headers with take_header and takes wikilink and embed targets from the parsed nodes, so there is no second reading to merge. These tests keep it so. --- crates/mq-markdown/tests/obsidian_tests.rs | 52 ++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/crates/mq-markdown/tests/obsidian_tests.rs b/crates/mq-markdown/tests/obsidian_tests.rs index bc3ae7bc1..6d6abf633 100644 --- a/crates/mq-markdown/tests/obsidian_tests.rs +++ b/crates/mq-markdown/tests/obsidian_tests.rs @@ -422,3 +422,55 @@ fn html_renders_callouts(#[case] input: &str, #[case] expected: &str) { assert_eq!(mq_markdown::to_html(input), expected); assert_eq!(Markdown::from_markdown_str(input).unwrap().to_html(), expected); } + +/// The HTML and the nodes are made from one reading of the source: what the callout node says is what +/// the HTML shows. +#[rstest] +#[case::plain("> [!NOTE]\n> body\n")] +#[case::title("> [!warning] Heads *up*\n> body\n")] +#[case::folded("> [!tip]- Fold me\n> body\n")] +#[case::open("> [!faq]+\n> body\n")] +#[case::kind_with_dash("> [!my-type_1] T\n")] +#[case::not_a_callout("> [! NOTE]\n> body\n")] +fn html_shows_what_the_callout_node_holds(#[case] input: &str) { + let nodes = input.parse::<Markdown>().unwrap().nodes; + let html = mq_markdown::to_html(input); + let Some(callout) = find_callout(&nodes) else { + assert!(!html.contains("callout"), "{html}"); + return; + }; + assert!( + html.contains(&format!("data-callout=\"{}\"", callout.kind.to_lowercase())), + "{html}" + ); + let tag = if callout.fold.is_some() { + "<details" + } else { + "<div class=\"callout\"" + }; + assert!(html.starts_with(tag), "{html}"); + assert_eq!(html.contains(" open>"), callout.fold == Some('+'), "{html}"); + if let Some(title) = &callout.title { + assert!(html.contains(&title.replace('<', "&lt;")), "{title} in {html}"); + } +} + +#[rstest] +#[case::target("[[Page]]")] +#[case::text("[[Page|shown]]")] +#[case::heading("[[Page#Heading]]")] +#[case::embed_note("![[Note]]")] +#[case::embed_image("![[pic.png|alt|100]]")] +fn html_shows_what_the_link_node_holds(#[case] input: &str) { + let nodes = input.parse::<Markdown>().unwrap().nodes; + let html = mq_markdown::to_html(input); + let target = nodes + .iter() + .find_map(|node| match node { + Node::WikiLink(link) => Some(link.target.clone()), + Node::Embed(embed) => Some(embed.target.clone()), + _ => None, + }) + .unwrap(); + assert!(html.contains(&target), "{target} in {html}"); +} From f69947565f3b2c689255d3bd1ab71eace2689788 Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Fri, 9 Oct 2026 12:43:00 +0900 Subject: [PATCH 62/88] =?UTF-8?q?=F0=9F=92=A5=20feat(markdown)!:=20remove?= =?UTF-8?q?=20ListMarker::Continuation=20and=20make=20the=20options=20and?= =?UTF-8?q?=20errors=20extensible?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The continuation of an item is a flag next to the reordered nodes, so ListMarker has the five markers that a document can write and as_char returns a char. MdxFound and MdxPlace are non_exhaustive, and ParseOptions too, with with_frontmatter to set it. --- crates/mq-markdown/src/markdown.rs | 23 ++++++++++-- crates/mq-markdown/src/markdown/list_order.rs | 37 +++++++++++++++---- crates/mq-markdown/src/node.rs | 22 ++++------- crates/mq-markdown/src/node/tests.rs | 3 +- crates/mq-markdown/src/parser/error.rs | 2 + .../mq-markdown/tests/mdx_roundtrip_tests.rs | 2 +- .../tests/parser_property_tests.rs | 4 +- crates/mq-markdown/tests/parser_tree_tests.rs | 7 +--- crates/mq-run/src/cli.rs | 4 +- 9 files changed, 65 insertions(+), 39 deletions(-) diff --git a/crates/mq-markdown/src/markdown.rs b/crates/mq-markdown/src/markdown.rs index dfb9305b0..40c5445fa 100644 --- a/crates/mq-markdown/src/markdown.rs +++ b/crates/mq-markdown/src/markdown.rs @@ -5,8 +5,8 @@ use crate::html_to_markdown::ConversionOptions; #[cfg(feature = "html-to-markdown")] use crate::node::ListStyle; use crate::node::{ - Code, ColorTheme, ListMarker, Node, Position, RenderOptions, TableAlign, TableCell, indent_lines, - list_own_prefix_width, reindent_all_lines, render_before, render_cell_values, render_values_block, + Code, ColorTheme, Node, Position, RenderOptions, TableAlign, TableCell, indent_lines, list_own_prefix_width, + reindent_all_lines, render_before, render_cell_values, render_values_block, }; #[cfg(any(feature = "json", feature = "html-to-markdown"))] use miette::miette; @@ -18,12 +18,22 @@ mod table_layout; /// What reading Markdown or MDX turns on. #[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] pub struct ParseOptions { /// Read YAML between `---` lines, or TOML between `+++` lines, at the start of the input as frontmatter. /// Without it these lines are a rule and text, or a heading. pub frontmatter: bool, } +impl ParseOptions { + /// Sets whether frontmatter at the start of the input is read as such. + #[must_use] + pub fn with_frontmatter(mut self, frontmatter: bool) -> Self { + self.frontmatter = frontmatter; + self + } +} + impl Default for ParseOptions { fn default() -> Self { Self { frontmatter: true } @@ -85,7 +95,12 @@ impl Markdown { let mut list_indent_stack: Vec<usize> = Vec::new(); let reordered = list_order::reorder(&self.nodes); - let nodes: &[Node] = reordered.as_deref().unwrap_or(&self.nodes); + let nodes: &[Node] = reordered.as_ref().map_or(&self.nodes, |reordered| &reordered.nodes); + let is_continuation = |index: usize| { + reordered + .as_ref() + .is_some_and(|reordered| reordered.continuation[index]) + }; let mut buffer = String::with_capacity(nodes.len() * 50); for (i, node) in nodes.iter().enumerate() { @@ -178,7 +193,7 @@ impl Markdown { let prev_node = i.checked_sub(1).and_then(|j| nodes.get(j)); let value = if let Node::List(list) = node - && list.marker == Some(ListMarker::Continuation) + && is_continuation(i) { // The rest of an item after its nested items, aligned with the content of the item. list_indent_stack.truncate(list.level as usize + 1); diff --git a/crates/mq-markdown/src/markdown/list_order.rs b/crates/mq-markdown/src/markdown/list_order.rs index 115ce3eec..189608224 100644 --- a/crates/mq-markdown/src/markdown/list_order.rs +++ b/crates/mq-markdown/src/markdown/list_order.rs @@ -4,10 +4,30 @@ //! that item come after them in the source, like a paragraph that follows a nested list. Rendering //! the nodes as they are would write those children before the nested items. -use crate::node::{List, ListMarker, Node, Position}; +use crate::node::{List, Node, Position}; + +/// Nodes in source order. +pub(super) struct Reordered { + pub(super) nodes: Vec<Node>, + /// For each node, whether it holds the rest of an item that follows the items nested in it. It is + /// rendered as its content, indented to the content of the item, without a marker. + pub(super) continuation: Vec<bool>, +} + +impl Reordered { + fn push(&mut self, node: Node) { + self.nodes.push(node); + self.continuation.push(false); + } + + fn push_continuation(&mut self, node: Node) { + self.nodes.push(node); + self.continuation.push(true); + } +} /// Returns the nodes in source order, or `None` when they already are. -pub(super) fn reorder(nodes: &[Node]) -> Option<Vec<Node>> { +pub(super) fn reorder(nodes: &[Node]) -> Option<Reordered> { // Some child of an item starts after the item nested right after it. let needed = nodes.windows(2).any(|pair| match (&pair[0], &pair[1]) { (Node::List(item), Node::List(next)) if next.level > item.level => { @@ -22,7 +42,10 @@ pub(super) fn reorder(nodes: &[Node]) -> Option<Vec<Node>> { if !needed { return None; } - let mut out = Vec::with_capacity(nodes.len() + 4); + let mut out = Reordered { + nodes: Vec::with_capacity(nodes.len() + 4), + continuation: Vec::with_capacity(nodes.len() + 4), + }; reorder_into(nodes, &mut out); Some(out) } @@ -31,7 +54,7 @@ fn start_line(node: &Node) -> Option<usize> { node.position().map(|position| position.start.line) } -fn reorder_into(nodes: &[Node], out: &mut Vec<Node>) { +fn reorder_into(nodes: &[Node], out: &mut Reordered) { let mut index = 0; while index < nodes.len() { let node = &nodes[index]; @@ -91,12 +114,12 @@ fn reorder_into(nodes: &[Node], out: &mut Vec<Node>) { child += 1; } if child > from { - out.push(continuation(item, &item.values[from..child])); + out.push_continuation(continuation(item, &item.values[from..child])); } reorder_into(segment, out); } if child < item.values.len() { - out.push(continuation(item, &item.values[child..])); + out.push_continuation(continuation(item, &item.values[child..])); } index += 1 + nested; } @@ -106,8 +129,6 @@ fn continuation(item: &List, values: &[Node]) -> Node { Node::List(List { values: values.to_vec(), position: span(values), - // Rendered as its content, indented to the content of the item, without a marker. - marker: Some(ListMarker::Continuation), ..item.clone() }) } diff --git a/crates/mq-markdown/src/node.rs b/crates/mq-markdown/src/node.rs index 8e90558fe..7404d3394 100644 --- a/crates/mq-markdown/src/node.rs +++ b/crates/mq-markdown/src/node.rs @@ -203,23 +203,17 @@ pub enum ListMarker { /// `)` after the number of an ordered item. #[cfg_attr(feature = "json", serde(rename = ")"))] Paren, - /// What the rest of an item that follows the items nested in it is rendered with. It has no marker - /// of its own, and the parser never produces it. - #[doc(hidden)] - #[cfg_attr(feature = "json", serde(rename = "\0"))] - Continuation, } impl ListMarker { - /// The character the marker is written with, or `None` for [`ListMarker::Continuation`]. - pub fn as_char(self) -> Option<char> { + /// The character the marker is written with. + pub fn as_char(self) -> char { match self { - Self::Dash => Some('-'), - Self::Plus => Some('+'), - Self::Star => Some('*'), - Self::Period => Some('.'), - Self::Paren => Some(')'), - Self::Continuation => None, + Self::Dash => '-', + Self::Plus => '+', + Self::Star => '*', + Self::Period => '.', + Self::Paren => ')', } } @@ -241,7 +235,7 @@ impl ListMarker { Self::Dash => Some(ListStyle::Dash), Self::Plus => Some(ListStyle::Plus), Self::Star => Some(ListStyle::Star), - Self::Period | Self::Paren | Self::Continuation => None, + Self::Period | Self::Paren => None, } } } diff --git a/crates/mq-markdown/src/node/tests.rs b/crates/mq-markdown/src/node/tests.rs index 7252437e8..eb08f66fc 100644 --- a/crates/mq-markdown/src/node/tests.rs +++ b/crates/mq-markdown/src/node/tests.rs @@ -11,7 +11,7 @@ use rstest::rstest; #[case::digit('1', None)] fn list_marker_from_char(#[case] char: char, #[case] expected: Option<ListMarker>) { assert_eq!(ListMarker::from_char(char), expected); - assert_eq!(expected.and_then(ListMarker::as_char), expected.map(|_| char)); + assert_eq!(expected.map(ListMarker::as_char), expected.map(|_| char)); } #[rstest] @@ -20,7 +20,6 @@ fn list_marker_from_char(#[case] char: char, #[case] expected: Option<ListMarker #[case::star(ListMarker::Star, Some(ListStyle::Star))] #[case::period(ListMarker::Period, None)] #[case::paren(ListMarker::Paren, None)] -#[case::continuation(ListMarker::Continuation, None)] fn list_marker_style(#[case] marker: ListMarker, #[case] expected: Option<ListStyle>) { assert_eq!(marker.style(), expected); } diff --git a/crates/mq-markdown/src/parser/error.rs b/crates/mq-markdown/src/parser/error.rs index 79f717ca6..bc909078e 100644 --- a/crates/mq-markdown/src/parser/error.rs +++ b/crates/mq-markdown/src/parser/error.rs @@ -6,6 +6,7 @@ use std::fmt; /// What a JSX tag has where something else must be. #[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[non_exhaustive] pub enum MdxFound { Char(char), EndOfFile, @@ -22,6 +23,7 @@ impl fmt::Display for MdxFound { /// The part of a JSX tag where an unexpected character is. #[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[non_exhaustive] pub enum MdxPlace { BeforeName, AfterName, diff --git a/crates/mq-markdown/tests/mdx_roundtrip_tests.rs b/crates/mq-markdown/tests/mdx_roundtrip_tests.rs index 84f5dfa21..32c338a9e 100644 --- a/crates/mq-markdown/tests/mdx_roundtrip_tests.rs +++ b/crates/mq-markdown/tests/mdx_roundtrip_tests.rs @@ -134,7 +134,7 @@ fn mdx_without_a_closing_line_has_no_frontmatter() { fn frontmatter_can_be_left_to_the_text(#[case] mdx: bool) { use mq_markdown::ParseOptions; - let options = ParseOptions { frontmatter: false }; + let options = ParseOptions::default().with_frontmatter(false); let input = "---\ntitle: x\n---\n\n+++\na\n+++\n"; let markdown = if mdx { Markdown::from_mdx_str_with(input, options) diff --git a/crates/mq-markdown/tests/parser_property_tests.rs b/crates/mq-markdown/tests/parser_property_tests.rs index 4e52c6e01..6a1b16d27 100644 --- a/crates/mq-markdown/tests/parser_property_tests.rs +++ b/crates/mq-markdown/tests/parser_property_tests.rs @@ -643,7 +643,7 @@ proptest! { lines in prop::collection::vec(prop::sample::select(LAZY_LINES), 1..6) ) { lazy_line_agrees(&lines, |input| { - Markdown::from_markdown_str_with(input, ParseOptions { frontmatter: false }).ok().map(|md| md.nodes) + Markdown::from_markdown_str_with(input, ParseOptions::default().with_frontmatter(false)).ok().map(|md| md.nodes) })?; } @@ -658,7 +658,7 @@ proptest! { ) ) { lazy_line_agrees(&lines, |input| { - Markdown::from_mdx_str_with(input, ParseOptions { frontmatter: false }).ok().map(|md| md.nodes) + Markdown::from_mdx_str_with(input, ParseOptions::default().with_frontmatter(false)).ok().map(|md| md.nodes) })?; } } diff --git a/crates/mq-markdown/tests/parser_tree_tests.rs b/crates/mq-markdown/tests/parser_tree_tests.rs index d4dd74506..f06473bb7 100644 --- a/crates/mq-markdown/tests/parser_tree_tests.rs +++ b/crates/mq-markdown/tests/parser_tree_tests.rs @@ -4,7 +4,7 @@ //! are not empty or default, and `@line:column-line:column`. use mq_markdown::{ - HorizontalRuleMarker, ListMarker, Markdown, MdxAttributeContent, MdxAttributeValue, Node, Position, TableAlignKind, + HorizontalRuleMarker, Markdown, MdxAttributeContent, MdxAttributeValue, Node, Position, TableAlignKind, }; use rstest::rstest; @@ -51,10 +51,7 @@ fn node_line(node: &Node) -> (String, &[Node]) { ), Node::Blockquote(quote) => (format!("blockquote{}", position(&quote.position)), &quote.values), Node::List(list) => { - let marker = list.marker.map(|marker| match marker { - ListMarker::Continuation => "continuation".to_string(), - marker => marker.as_char().unwrap_or_default().to_string(), - }); + let marker = list.marker.map(|marker| marker.as_char().to_string()); ( format!( "list level={} index={}{}{}{}{}{}{}", diff --git a/crates/mq-run/src/cli.rs b/crates/mq-run/src/cli.rs index d1eda97f9..2b4bae0f6 100644 --- a/crates/mq-run/src/cli.rs +++ b/crates/mq-run/src/cli.rs @@ -2062,9 +2062,7 @@ impl Cli { /// How Markdown and MDX input is read. fn parse_options(&self) -> mq_markdown::ParseOptions { - mq_markdown::ParseOptions { - frontmatter: !self.input.program.no_frontmatter, - } + mq_markdown::ParseOptions::default().with_frontmatter(!self.input.program.no_frontmatter) } fn validate_csv_options(&self) -> miette::Result<()> { From ee00d0fd786493856c18abbe8dc3e117f3efc2a8 Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Fri, 9 Oct 2026 12:48:00 +0900 Subject: [PATCH 63/88] =?UTF-8?q?=F0=9F=93=9D=20docs(markdown):=20describe?= =?UTF-8?q?=20the=20parser=20by=20the=20specs=20and=20mdast,=20and=20docum?= =?UTF-8?q?ent=20the=20HTML,=20list=20style=20and=20frontmatter=20behavior?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The comments that named the old parser now say what CommonMark, GFM and mdast do. A new book page covers callouts, wikilinks and the tag filter in HTML output, the list style default, and --no-frontmatter. listStyle is no longer defaulted to dash in mq-web and mq-nodejs, so they keep the markers of the document as the CLI does. Em dashes are gone from the comments of this change. --- crates/mq-check/src/constraint.rs | 30 +++---- crates/mq-markdown/src/node/expand.rs | 2 +- crates/mq-markdown/src/node/render.rs | 6 +- crates/mq-markdown/src/node/tests.rs | 18 ++-- crates/mq-markdown/src/parser.rs | 7 +- crates/mq-markdown/src/parser/inline/html.rs | 2 +- crates/mq-markdown/src/parser/inline/tail.rs | 2 +- crates/mq-markdown/src/parser/line.rs | 4 +- crates/mq-markdown/src/parser/mdx.rs | 2 +- crates/mq-markdown/src/parser/render_html.rs | 4 +- docs/books/src/SUMMARY.md | 1 + docs/books/src/sitemap.xml | 4 + docs/books/src/start/input-and-output.md | 90 ++++++++++++++++++++ packages/mq-nodejs/README.md | 2 +- packages/mq-nodejs/src/core.ts | 1 - packages/mq-nodejs/test/index.test.ts | 9 ++ packages/mq-web/README.md | 2 +- packages/mq-web/src/core.ts | 1 - packages/mq-web/test/index.test.ts | 9 ++ 19 files changed, 153 insertions(+), 43 deletions(-) create mode 100644 docs/books/src/start/input-and-output.md diff --git a/crates/mq-check/src/constraint.rs b/crates/mq-check/src/constraint.rs index 007634e60..9e93e2c61 100644 --- a/crates/mq-check/src/constraint.rs +++ b/crates/mq-check/src/constraint.rs @@ -149,7 +149,7 @@ pub fn generate_constraints(hir: &Hir, ctx: &mut InferenceContext) -> ChildrenIn // Pass 2.5: Process Assign symbols before other operators/calls. // This ensures that variable types are updated by assignments before // Refs and Calls in Pass 3 resolve against the (potentially stale) type. - // e.g., `var x = 10 | x = "hello" | upcase(x)` — the Assign updates + // e.g., `var x = 10 | x = "hello" | upcase(x)`: the Assign updates // x's type to String before upcase(x) resolves its argument type. for (symbol_id, kind) in &cats.assign_symbols { generate_symbol_constraints(hir, *symbol_id, kind.clone(), ctx, &children_index); @@ -675,7 +675,7 @@ pub(super) fn generate_symbol_constraints( ctx.set_symbol_type(symbol_id, last_ty); } } else { - // No HIR resolution — try builtin registry by name as fallback + // No HIR resolution: try builtin registry by name as fallback if let Some(symbol) = hir.symbol(symbol_id) && let Some(name) = &symbol.value && ctx.get_builtin_overloads(name.as_str()).is_some() @@ -765,7 +765,7 @@ pub(super) fn generate_symbol_constraints( }, )); - // No need to re-bind subsequent Refs — Assigns are processed + // No need to re-bind subsequent Refs: Assigns are processed // in Pass 2.5 (before Refs/Calls in Pass 3), so Refs will // pick up the updated variable type when they are processed. } else { @@ -912,7 +912,7 @@ pub(super) fn generate_symbol_constraints( return; } - // Homogeneous or unresolved — unify all element types + // Homogeneous or unresolved: unify all element types let elem_ty = elem_tys[0].clone(); let range = get_symbol_range(hir, symbol_id); for ty in &elem_tys[1..] { @@ -1037,7 +1037,7 @@ pub(super) fn generate_symbol_constraints( if !children.is_empty() { let range = get_symbol_range(hir, symbol_id); - // First child is the condition — mq is dynamically typed, so any + // First child is the condition: mq is dynamically typed, so any // value can be used as a condition (truthy/falsy), not just Bool. let _cond_ty = ctx.get_or_create_symbol_type(children[0]); @@ -1080,7 +1080,7 @@ pub(super) fn generate_symbol_constraints( true }; - // Check if any branch is None or a Union containing None — in mq, + // Check if any branch is None or a Union containing None: in mq, // `if (...): value else: None` (or a while-loop branch that returns // `Union(T, None)`) should not unify `value` with None; instead treat // as different types so the overall if-expression preserves the None @@ -1129,7 +1129,7 @@ pub(super) fn generate_symbol_constraints( || (has_none_branch && resolved.len() >= 2) || has_parameter_branch { - // Different concrete types across branches — use Union type. + // Different concrete types across branches: use Union type. // Include ALL resolved branch types (vars and concrete) so that // branches whose types are not yet fully resolved can still be // tracked and resolved later (e.g., `if (...): items else: None` @@ -1137,7 +1137,7 @@ pub(super) fn generate_symbol_constraints( let union_ty = Type::union(resolved.clone()); ctx.set_symbol_type(symbol_id, union_ty); } else { - // Homogeneous or unresolved — unify all branch types + // Homogeneous or unresolved: unify all branch types ctx.set_symbol_type(symbol_id, then_ty.clone()); for ty in &branch_tys[1..] { ctx.add_constraint(Constraint::Equal( @@ -1446,12 +1446,12 @@ pub(super) fn generate_symbol_constraints( }; if !all_same && concrete.len() >= 2 { - // Different concrete types in arms — use Union type + // Different concrete types in arms: use Union type let unique_types: Vec<Type> = concrete.into_iter().cloned().collect(); let union_ty = Type::union(unique_types); ctx.set_symbol_type(symbol_id, union_ty); } else if concrete.len() == 1 && !var_tys.is_empty() { - // One concrete arm (e.g. `None`) plus still-unresolved arms — union + // One concrete arm (e.g. `None`) plus still-unresolved arms: union // them instead of unifying, or the unresolved type collapses to the // concrete one. Mirrors `merge_loop_types`. let mut unified_var = var_tys[0].clone(); @@ -1535,7 +1535,7 @@ pub(super) fn generate_symbol_constraints( )); ctx.set_symbol_type(symbol_id, try_ty); } else { - // A branch type is still pending deferred resolution — decide later. + // A branch type is still pending deferred resolution: decide later. let ty_var = ctx.fresh_var(); ctx.set_symbol_type(symbol_id, Type::Var(ty_var)); ctx.add_deferred_try_catch(infer::DeferredTryCatch { @@ -1638,7 +1638,7 @@ pub(super) fn generate_symbol_constraints( Type::Var(ty_var) } } else { - // Piped type not yet resolved — defer to post-unification. + // Piped type not yet resolved: defer to post-unification. // Keep the selector so that if the piped type resolves to a Markdown node, // the right type can be returned (e.g., `md.depth` → number). let ty_var = ctx.fresh_var(); @@ -1741,7 +1741,7 @@ pub(super) fn generate_symbol_constraints( } } - // Block: pipe chain — compute type from children in pass 3. + // Block: pipe chain: compute type from children in pass 3. // Block has higher insertion order than its parent (If/Else/etc.), so in // the reverse-order pass 3 loop it is processed before its parent. This // means the If handler can see the Block's real type (e.g. Union(T, None) @@ -1926,7 +1926,7 @@ fn generate_call_constraints( let real_children = &children[..children.len() - trailing_bracket_count]; let arg_symbol_ids = if prepend_piped { - // piped input was prepended — include a placeholder + // piped input was prepended: include a placeholder let mut ids = vec![symbol_id]; // placeholder for piped arg ids.extend_from_slice(real_children); ids @@ -2037,7 +2037,7 @@ fn generate_call_constraints( // Only constrain the index to Number when it is a literal // numeric index (e.g. v[0]). For variable indices // (e.g. a String key used for Dict access), skip this - // constraint — the correct element type is resolved via + // constraint: the correct element type is resolved via // resolve_deferred_tuple_accesses once the container's // type is known. if literal_index.is_some() diff --git a/crates/mq-markdown/src/node/expand.rs b/crates/mq-markdown/src/node/expand.rs index 55600b328..388844c04 100644 --- a/crates/mq-markdown/src/node/expand.rs +++ b/crates/mq-markdown/src/node/expand.rs @@ -202,7 +202,7 @@ impl Node { let content = &text[inner_start..close]; if content.contains('[') || content.contains(']') { - // nested brackets — skip past this '[' and retry + // nested brackets: skip past this '[' and retry search_from = open + 1; continue; } diff --git a/crates/mq-markdown/src/node/render.rs b/crates/mq-markdown/src/node/render.rs index 67d1214f8..6a61d1cfa 100644 --- a/crates/mq-markdown/src/node/render.rs +++ b/crates/mq-markdown/src/node/render.rs @@ -834,7 +834,7 @@ pub(super) fn render_link_title(title: &str, style: &TitleSurroundStyle) -> Stri /// Escapes `\ [ ]` in reference labels/idents and image alt text: plain strings /// (not walked as inline nodes) that still sit inside `[...]` syntax. -/// True if `s` needs escaping beyond `escape_label`'s narrow `\ [ ]` set — unsafe +/// True if `s` needs escaping beyond `escape_label`'s narrow `\ [ ]` set: unsafe /// for a reference's implicit shortcut form, since the definition's label won't match. fn needs_broad_escaping(s: &str) -> bool { s.bytes().any(|b| { @@ -1257,7 +1257,7 @@ mod tests { } // Rendering must never panic, whatever the heading text (arbitrary Unicode, - // possibly ending in a run of `#`) — this is a UTF-8 char-boundary regression + // possibly ending in a run of `#`): this is a UTF-8 char-boundary regression // guard for the trailing-`#` escape logic. #[test] fn heading_render_never_panics_on_unicode_text( @@ -1279,7 +1279,7 @@ mod tests { } // A Link's destination, rendered through the real writer and re-parsed by the - // real markdown parser, must recover the original URL string — the concrete + // real markdown parser, must recover the original URL string: the concrete // bug class from the GFM spec-suite fidelity check (spaces/parens/angle // brackets silently corrupting or breaking the link on round-trip). #[test] diff --git a/crates/mq-markdown/src/node/tests.rs b/crates/mq-markdown/src/node/tests.rs index eb08f66fc..a30ed62bf 100644 --- a/crates/mq-markdown/src/node/tests.rs +++ b/crates/mq-markdown/src/node/tests.rs @@ -625,7 +625,7 @@ fn test_is_link_includes_wikilink(#[case] node: Node, #[case] expected: bool) { #[cfg(feature = "wikilink")] #[rstest] -// no wikilinks — returns original text node unchanged +// no wikilinks: returns original text node unchanged #[case("plain text", vec![Node::Text(Text{value: "plain text".to_string(), position: None})])] // only a wikilink #[case("[[target]]", vec![Node::WikiLink(WikiLink{target: "target".to_string(), text: None, position: None})])] @@ -659,7 +659,7 @@ fn test_is_link_includes_wikilink(#[case] node: Node, #[case] expected: bool) { ])] // unclosed [[ treated as plain text #[case("[[unclosed", vec![Node::Text(Text{value: "[[unclosed".to_string(), position: None})])] -// nested brackets invalid — treated as plain text +// nested brackets invalid: treated as plain text #[case("[[in[ner]]]", vec![Node::Text(Text{value: "[[in[ner]]]".to_string(), position: None})])] // multibyte characters in surrounding text #[case("日本語 [[ターゲット]] テキスト", vec![ @@ -708,12 +708,12 @@ fn test_wikilink_with_value(#[case] node: Node, #[case] value: &str, #[case] exp #[cfg(feature = "wikilink")] #[rstest] -// footnote with no wikilinks — values unchanged +// footnote with no wikilinks: values unchanged #[case( Node::Footnote(Footnote{ident: "1".to_string(), values: vec![Node::Text(Text{value: "plain text".to_string(), position: None})], position: None}), vec![Node::Footnote(Footnote{ident: "1".to_string(), values: vec![Node::Text(Text{value: "plain text".to_string(), position: None})], position: None})] )] -// footnote with a wikilink in text — expanded to WikiLink node +// footnote with a wikilink in text: expanded to WikiLink node #[case( Node::Footnote(Footnote{ident: "1".to_string(), values: vec![Node::Text(Text{value: "[[target]]".to_string(), position: None})], position: None}), vec![Node::Footnote(Footnote{ident: "1".to_string(), values: vec![Node::WikiLink(WikiLink{target: "target".to_string(), text: None, position: None})], position: None})] @@ -2546,7 +2546,7 @@ fn test_attr_value_as_i64(#[case] value: AttrValue, #[case] expected: Option<i64 Node::Callout(Callout { fold: None, kind: "TIP".to_string(), title: None, values: vec![], position: None }), "> [!TIP]" )] -// multiline body: single Text with embedded '\n' — each line prefixed with "> " +// multiline body: single Text with embedded '\n': each line prefixed with "> " #[case( Node::Callout(Callout { fold: None, kind: "INFO".to_string(), title: None, values: vec![Node::Text(Text { value: "line one\nline two".to_string(), position: None })], @@ -2703,11 +2703,11 @@ fn test_callout_end_to_end() { ])] // multibyte target #[case("![[ノート]]", vec![Node::Embed(Embed { target: "ノート".to_string(), display: None, position: None })])] -// no embed — plain text unchanged +// no embed: plain text unchanged #[case("plain text", vec![Node::Text(Text { value: "plain text".to_string(), position: None })])] -// unclosed embed — treated as plain text +// unclosed embed: treated as plain text #[case("![[unclosed", vec![Node::Text(Text { value: "![[unclosed".to_string(), position: None })])] -// lone ! before non-embed — treated as plain text +// lone ! before non-embed: treated as plain text #[case("! not an embed", vec![Node::Text(Text { value: "! not an embed".to_string(), position: None })])] fn test_parse_embeds_in_text(#[case] input: &str, #[case] expected: Vec<Node>) { let mut result = Vec::new(); @@ -2859,7 +2859,7 @@ fn test_callout_with_wikilink(#[case] input: &str, #[case] expected_nodes: usize )] // callout after heading #[case("# Title\n\n> [!NOTE]\n> body", 2, "# Title\n\n> [!NOTE]\n> body\n")] -// callout body with a multi-item list — second item must not be over-indented +// callout body with a multi-item list: second item must not be over-indented #[case("> [!NOTE]\n> - item 1\n> - item 2", 1, "> [!NOTE]\n> - item 1\n> - item 2\n")] fn test_callout_combined_with_other_nodes( #[case] input: &str, diff --git a/crates/mq-markdown/src/parser.rs b/crates/mq-markdown/src/parser.rs index 6621cec62..80fb0d500 100644 --- a/crates/mq-markdown/src/parser.rs +++ b/crates/mq-markdown/src/parser.rs @@ -69,9 +69,8 @@ mod tests { std::env::var(name).ok()?.parse().ok() } - /// Lines built from container prefixes and block-level bodies only, so inline syntax (not - /// implemented yet) never appears. Constructs whose markdown-rs positions depend on quirks of the - /// following lines are left to the cases of `tests/parser_tree_tests.rs`: empty items and quotes, + /// Lines built from container prefixes and block-level bodies only, so inline syntax never + /// appears. Constructs whose mdast positions depend on quirks of the following lines are left to the cases of `tests/parser_tree_tests.rs`: empty items and quotes, /// unclosed fences, indented code next to containers, ordered lists that do not start at 1, and /// lazy continuation inside nested containers, more than one container marker on a line, and runs /// of setext underlines (`===` is only used by those cases). @@ -613,7 +612,7 @@ mod tests { /// - nested `strong` of runs of `*` and `_`, which `CommonMark` 0.30 and later render as nested /// elements where 0.29 merged them (388, 416, 424, 425, 426, 463, 464, 465, 467) /// - links with a protocol other than http, https, irc, ircs, mailto and xmpp have no `href`, as - /// `markdown-rs` does it by default (496, 594, 595, 597) + /// as mdast to HTML does by default (496, 594, 595, 597) /// - GFM autolink literals, which the spec does not have (598, 604, 607, 608) /// - the GFM tag filter, which writes the tags `script`, `style` and `textarea` as text (140, 141, /// 142, 145, 147) diff --git a/crates/mq-markdown/src/parser/inline/html.rs b/crates/mq-markdown/src/parser/inline/html.rs index 85f508a76..2266f3f2d 100644 --- a/crates/mq-markdown/src/parser/inline/html.rs +++ b/crates/mq-markdown/src/parser/inline/html.rs @@ -58,7 +58,7 @@ pub(crate) fn is_autolink_email(text: &str) -> bool { } /// Parses an email address for an autolink at `start`, returning the offset of the closing `>`. -/// The characters allowed differ slightly from `CommonMark`, following `markdown-rs`. +/// The characters allowed are those of the mdast email autolink, which differ slightly from `CommonMark`. fn email(bytes: &[u8], start: usize) -> Option<usize> { let atext = |b: u8| matches!(b, b'#'..=b'\'' | b'*' | b'+' | b'-'..=b'9' | b'=' | b'?' | b'A'..=b'Z' | b'^'..=b'~'); let mut index = start; diff --git a/crates/mq-markdown/src/parser/inline/tail.rs b/crates/mq-markdown/src/parser/inline/tail.rs index f60eb604d..cba364d9a 100644 --- a/crates/mq-markdown/src/parser/inline/tail.rs +++ b/crates/mq-markdown/src/parser/inline/tail.rs @@ -130,7 +130,7 @@ pub(in crate::parser) fn destination(src: &str, pos: usize) -> Option<(String, u depth -= 1; } b' ' | b'\t' | b'\n' | b'\r' => break, - // markdown-rs lets a NUL through, though it is a control character. + // A NUL is let through, though it is a control character. byte if byte.is_ascii_control() && byte != 0 => break, _ => {} } diff --git a/crates/mq-markdown/src/parser/line.rs b/crates/mq-markdown/src/parser/line.rs index 7c4c3a050..f5c847454 100644 --- a/crates/mq-markdown/src/parser/line.rs +++ b/crates/mq-markdown/src/parser/line.rs @@ -95,7 +95,7 @@ impl<'a> Line<'a> { } /// The position of the byte at `byte` in `text`. Columns are in bytes, except that a tab advances - /// to the next multiple of four, like `markdown-rs` counts them. + /// to the next multiple of four, as mdast positions count them. pub(super) fn point(&self, byte: usize) -> Point { let offset = self.column + byte; let mut column = self.visual_column(offset) + 1; @@ -110,7 +110,7 @@ impl<'a> Line<'a> { } } - /// Like [`Line::point`] for content that `markdown-rs` places where the rest of a tab ends, instead + /// Like [`Line::point`] for content that mdast places where the rest of a tab ends, instead /// of before it: paragraphs and fences. pub(super) fn content_point(&self, byte: usize) -> Point { let mut point = self.point(byte); diff --git a/crates/mq-markdown/src/parser/mdx.rs b/crates/mq-markdown/src/parser/mdx.rs index 584fe997f..cf9c8ed9c 100644 --- a/crates/mq-markdown/src/parser/mdx.rs +++ b/crates/mq-markdown/src/parser/mdx.rs @@ -1,4 +1,4 @@ -//! MDX: JSX tags and expressions, ported from the rules of `markdown-rs` without a JavaScript parser, +//! MDX: JSX tags and expressions, as the MDX syntax extensions to CommonMark define them, without a JavaScript parser, //! so expressions are only checked for balanced braces, outside of strings, template literals and comments. use super::error::{Located, MdxErrorKind, MdxFound, MdxPlace}; diff --git a/crates/mq-markdown/src/parser/render_html.rs b/crates/mq-markdown/src/parser/render_html.rs index fd44e95ac..d37db8b35 100644 --- a/crates/mq-markdown/src/parser/render_html.rs +++ b/crates/mq-markdown/src/parser/render_html.rs @@ -1,8 +1,8 @@ //! Renders the block tree as HTML, with CommonMark, GFM, math and frontmatter, and with raw HTML //! allowed. //! -//! The output is the same as that of `markdown-rs`, line endings included: blocks start on a line of -//! their own, and the line ending that the source has after the last block is kept. +//! The output is the HTML of the CommonMark and GFM specs, line endings included: blocks start on a +//! line of their own, and the line ending that the source has after the last block is kept. use super::flavor::Flavor; use super::inline::{self, Document}; diff --git a/docs/books/src/SUMMARY.md b/docs/books/src/SUMMARY.md index 9e09d40eb..098ef0ea7 100644 --- a/docs/books/src/SUMMARY.md +++ b/docs/books/src/SUMMARY.md @@ -15,6 +15,7 @@ - [Language Bindings](start/language_bindings.md) - [Embed mq in Rust](start/embedding-rust.md) - [Example](start/example.md) + - [Input and Output Behavior](start/input-and-output.md) - [Modules](start/modules.md) - [Compiled Programs (Experimental)](start/compiled-programs.md) - [Cookbook](cookbook/index.md) diff --git a/docs/books/src/sitemap.xml b/docs/books/src/sitemap.xml index 557772002..88bdd2bc9 100644 --- a/docs/books/src/sitemap.xml +++ b/docs/books/src/sitemap.xml @@ -68,6 +68,10 @@ <loc>https://mqlang.org/book/start/example</loc> <priority>1.0</priority> </url> + <url> + <loc>https://mqlang.org/book/start/input-and-output</loc> + <priority>1.0</priority> + </url> <url> <loc>https://mqlang.org/book/start/modules</loc> <priority>1.0</priority> diff --git a/docs/books/src/start/input-and-output.md b/docs/books/src/start/input-and-output.md new file mode 100644 index 000000000..61783b6d0 --- /dev/null +++ b/docs/books/src/start/input-and-output.md @@ -0,0 +1,90 @@ +# Input and Output Behavior + +How mq reads Markdown and writes it back, and the options that change it. For the full list of flags, see the [CLI reference](../reference/cli.md). + +## HTML output + +`-F html` renders the result as HTML. + +### Callouts + +A block quote that starts with `[!TYPE]` is a callout. It becomes a `div` with the class `callout` and the type in `data-callout`. A fold marker (`-` or `+`) makes it a `details` element instead, open when the marker is `+`. + +```markdown +> [!NOTE] Title +> body + +> [!TIP]- Folded +> hidden +``` + +```html +<div class="callout" data-callout="note"> +<div class="callout-title">Title</div> +<div class="callout-content"> +<p>body</p> +</div> +</div> +<details class="callout" data-callout="tip"> +<summary class="callout-title">Folded</summary> +<div class="callout-content"> +<p>hidden</p> +</div> +</details> +``` + +### Wikilinks and embeds + +`[[target]]` and `[[target|text]]` become links with the class `internal-link`. The target is used as the `href` as written. `![[target]]` becomes an `img`, `audio`, `video` or `embed` element when the file is an image, audio, video or PDF, and a link with the classes `internal-link embed` for a note. + +### Raw HTML + +Raw HTML in the input is kept, except for the tags that GFM filters: `<script>`, `<style>`, `<textarea>` and a few others. These are escaped and show up as text. + +```markdown +<script>alert(1)</script> +``` + +```html +&lt;script>alert(1)&lt;/script> +``` + +## List style + +Without `--list-style`, each list keeps the marker it was written with (`-`, `+` or `*`). Pass `--list-style dash`, `plus` or `star` to write every list with one marker. + +```sh +$ printf '* a\n* b\n' | mq '.' +* a +* b +$ printf '* a\n* b\n' | mq --list-style dash '.' +- a +- b +``` + +The `listStyle` option of `mq-web` and `mq-nodejs` works the same way: leave it out to keep the markers. + +## Frontmatter + +A document that starts with a `---` line (or `+++`) and has a closing line is read with frontmatter. The frontmatter is not part of the rendered HTML, and `.yaml` and `.toml` select it. + +Pass `--no-frontmatter` for a document that only happens to start with `---`, such as one that opens with a horizontal rule. The lines are then read as Markdown: the first `---` is a rule, and the text up to the next `---` is a paragraph or a heading. + +```sh +$ cat doc.md +--- +not: frontmatter +--- +body +$ mq '.' doc.md +--- +not: frontmatter +--- +body +$ mq --no-frontmatter '.' doc.md +--- +## not: frontmatter +body +``` + +`--no-frontmatter` applies to Markdown and MDX input. diff --git a/packages/mq-nodejs/README.md b/packages/mq-nodejs/README.md index 10b8f6545..81323c2c6 100644 --- a/packages/mq-nodejs/README.md +++ b/packages/mq-nodejs/README.md @@ -150,7 +150,7 @@ const markdown = `- Apple - Banana - Cherry`; -// Change list style +// Change list style (without listStyle, each list keeps the marker it was written with) const starList = await run(".[]", markdown, { listStyle: "star" }); // Output: * Apple\n* Banana\n* Cherry diff --git a/packages/mq-nodejs/src/core.ts b/packages/mq-nodejs/src/core.ts index aaae146e1..9635d46cd 100644 --- a/packages/mq-nodejs/src/core.ts +++ b/packages/mq-nodejs/src/core.ts @@ -61,7 +61,6 @@ export async function run( return await wasmModule.run(code, content, { isUpdate: false, inputFormat: "markdown", - listStyle: "dash", linkUrlStyle: "none", linkTitleStyle: "paren", ...options, diff --git a/packages/mq-nodejs/test/index.test.ts b/packages/mq-nodejs/test/index.test.ts index 409d044df..95bf38763 100644 --- a/packages/mq-nodejs/test/index.test.ts +++ b/packages/mq-nodejs/test/index.test.ts @@ -16,6 +16,15 @@ describe("run", () => { expect(result).toBe("- item 1\n- item 2\n"); }); + it.each([ + ["-", "- a\n- b\n"], + ["*", "* a\n* b\n"], + ["+", "+ a\n+ b\n"], + ])("keeps the list marker %s without a list style", async (_marker, markdown) => { + const result = await run(".[]", markdown); + expect(result).toBe(markdown); + }); + it("changes list style to star", async () => { const markdown = "- First item\n- Second item\n- Third item"; const result = await run(".[]", markdown, { listStyle: "star" }); diff --git a/packages/mq-web/README.md b/packages/mq-web/README.md index 447137740..a7b3b1bf5 100644 --- a/packages/mq-web/README.md +++ b/packages/mq-web/README.md @@ -124,7 +124,7 @@ const markdown = `- Apple - Banana - Cherry`; -// Change list style +// Change list style (without listStyle, each list keeps the marker it was written with) const starList = await run(".[]", markdown, { listStyle: "star" }); // Output: * Apple\n* Banana\n* Cherry diff --git a/packages/mq-web/src/core.ts b/packages/mq-web/src/core.ts index 0b85e85a6..daa221302 100644 --- a/packages/mq-web/src/core.ts +++ b/packages/mq-web/src/core.ts @@ -80,7 +80,6 @@ export async function run( return await wasm.run(code, content, { isUpdate: false, inputFormat: "markdown", - listStyle: "dash", linkUrlStyle: "none", linkTitleStyle: "paren", ...options, diff --git a/packages/mq-web/test/index.test.ts b/packages/mq-web/test/index.test.ts index 409d044df..95bf38763 100644 --- a/packages/mq-web/test/index.test.ts +++ b/packages/mq-web/test/index.test.ts @@ -16,6 +16,15 @@ describe("run", () => { expect(result).toBe("- item 1\n- item 2\n"); }); + it.each([ + ["-", "- a\n- b\n"], + ["*", "* a\n* b\n"], + ["+", "+ a\n+ b\n"], + ])("keeps the list marker %s without a list style", async (_marker, markdown) => { + const result = await run(".[]", markdown); + expect(result).toBe(markdown); + }); + it("changes list style to star", async () => { const markdown = "- First item\n- Second item\n- Third item"; const result = await run(".[]", markdown, { listStyle: "star" }); From 4d34b3a082b6a98401f71e8871e64f0de4773605 Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Fri, 9 Oct 2026 12:53:00 +0900 Subject: [PATCH 64/88] =?UTF-8?q?=F0=9F=90=9B=20fix(markdown):=20track=20o?= =?UTF-8?q?pen=20containers=20and=20constructs=20when=20collecting=20lazy?= =?UTF-8?q?=20lines?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit LeafState kept one counter of container markers, so a quote could not tell what the lines it held were inside. It now follows a stack of the quotes, items and footnotes that are open, with the indentation of each item, and the text of a paragraph that starts like a definition, and MDX flow content that spans lines. A line under definitions only is text, an item keeps the lines it is indented for after a blank line, and a fence inside a multi-line expression is text. The property test no longer leaves any of these out. --- crates/mq-markdown/src/parser/block.rs | 173 +--------- crates/mq-markdown/src/parser/block/leaf.rs | 315 ++++++++++++++++++ crates/mq-markdown/src/parser/definition.rs | 13 + crates/mq-markdown/src/parser/mdx_flow.rs | 42 ++- .../mq-markdown/tests/html_rendering_tests.rs | 2 + .../tests/parser_property_tests.rs | 25 +- 6 files changed, 375 insertions(+), 195 deletions(-) create mode 100644 crates/mq-markdown/src/parser/block/leaf.rs diff --git a/crates/mq-markdown/src/parser/block.rs b/crates/mq-markdown/src/parser/block.rs index c65db8f3d..30ab1b86c 100644 --- a/crates/mq-markdown/src/parser/block.rs +++ b/crates/mq-markdown/src/parser/block.rs @@ -10,12 +10,14 @@ use super::flavor::Flavor; use super::html_flow::{self, Kind as HtmlKind}; use super::inline; use super::line::{Indent, Line, split_lines, tab_stops}; -use super::mdx_flow::{FlowOutcome, absorbed_until, blocks_lazy_continuation, looks_like_mdx_flow, probe_mdx_flow}; +use super::mdx_flow::{FlowOutcome, absorbed_until, blocks_lazy_continuation, probe_mdx_flow}; use super::table; +mod leaf; mod rule; use super::tree::{Block, FootnoteBlock, InlineBlock, InlineKind, InlineSource, Item, ListBlock, QuoteBlock}; use crate::node::{HeadingDepth, Html, ListMarker, Node, Point, Position, Toml, Yaml}; +use leaf::LeafState; use rule::{BlockRule, Cx, Rules, fallback}; /// Lines indented by this many columns or more are code, not other blocks. @@ -173,175 +175,6 @@ impl LineStart<'_> { } } -/// Tracks just enough of the block state of already collected lines to tell whether a paragraph -/// is open, which decides whether a following line can be a lazy continuation. -struct LeafState<'a> { - fence: Option<Fence<'a>>, - /// An HTML block that has not ended yet. - html: Option<HtmlKind>, - /// A table that has not ended yet. Its rows cannot be continued lazily. - table: bool, - /// The number of cells of the last line when it was paragraph text with a pipe, which a delimiter - /// row with as many cells can turn into a header. - header: Option<usize>, - paragraph: bool, - /// Number of container markers on the line that opened the paragraph. - paragraph_containers: usize, - /// Restrictions on the next fed line, the first line of a container that interrupted a paragraph. - restricted: Interrupt, - /// How many more containers a line can open before the depth limit, which no longer open any. - budget: usize, -} - -impl<'a> LeafState<'a> { - /// `depth` is the depth of the container that collects the lines. - fn new(restricted: Interrupt, depth: usize) -> Self { - Self { - fence: None, - html: None, - table: false, - header: None, - paragraph: false, - paragraph_containers: 0, - restricted, - budget: MAX_DEPTH.saturating_sub(depth + 1), - } - } - - /// Whether `line` without its container prefix continues the paragraph of the collected lines. - fn continues_with(&self, line: &Line<'_>) -> bool { - self.paragraph && is_lazy_continuation(line) - } - - fn feed(&mut self, line: &Line<'a>) { - let mut line = *line; - let restricted = std::mem::take(&mut self.restricted); - let mut containers = 0; - // Whether a paragraph was open when the line started, for markers deeper on the same line. - let open = self.paragraph; - loop { - let line_indent = line.indent(); - let Indent { columns, bytes: indent } = line_indent; - let rest = &line.text[indent..]; - - if let Some(fence) = &self.fence { - if columns < line.code_indent() && fence.is_closed_by(rest) { - self.fence = None; - } - return; - } - - if let Some(kind) = self.html { - let ends = match kind { - HtmlKind::Basic | HtmlKind::Complete => line.is_blank(), - _ => html_flow::ends_in(kind, line.text), - }; - if ends { - self.html = None; - } - self.paragraph = false; - return; - } - if line.is_blank() { - self.paragraph = false; - self.table = false; - self.header = None; - return; - } - let header = std::mem::take(&mut self.header); - if self.table { - if interrupts_paragraph(&line) { - self.table = false; - } else { - self.paragraph = false; - return; - } - } else if self.paragraph - && !line.lazy - && header.is_some() - && table::delimiter_cells(&line) == header - && !interrupts_paragraph(&line) - { - self.table = true; - self.paragraph = false; - return; - } - if columns >= line.code_indent() { - // Continues an open paragraph of this container, otherwise it is indented code (or - // text when the container interrupted a paragraph). - if !(self.paragraph && containers <= self.paragraph_containers) { - self.paragraph = restricted.code; - self.paragraph_containers = containers; - } - return; - } - let start = LineStart::of(&line, line_indent); - if let Some(LineStart::Fence(fence)) = start { - self.fence = Some(fence); - self.paragraph = false; - return; - } - if self.paragraph && containers == self.paragraph_containers && setext_depth(&line).is_some() { - self.paragraph = false; - return; - } - match start { - Some(LineStart::Atx(_) | LineStart::ThematicBreak) => { - self.paragraph = false; - return; - } - Some(LineStart::Html(kind)) if kind != HtmlKind::Complete || !self.paragraph => { - let closed = match kind { - HtmlKind::Basic | HtmlKind::Complete => false, - _ => html_flow::ends_in(kind, &rest[html_flow::first_line_offset(kind)..]), - }; - self.html = (!closed).then_some(kind); - self.paragraph = false; - return; - } - Some(LineStart::Blockquote) if containers < self.budget => { - line = after_blockquote_marker(line, line_indent); - containers += 1; - continue; - } - Some(LineStart::Footnote(marker)) if containers < self.budget => { - line = line.skip(marker.content); - containers += 1; - self.paragraph = false; - continue; - } - // A marker in or inside the container of the open paragraph has to be able to interrupt it. - Some(LineStart::Item(marker)) - if containers < self.budget - && (!(restricted.list || (open && containers >= self.paragraph_containers)) - || marker.interrupts_paragraph()) => - { - line = marker.content(line); - containers += 1; - // The rest of the line starts a new item. - self.paragraph = false; - continue; - } - _ => {} - } - - // MDX flow content is not part of a paragraph. - if looks_like_mdx_flow(&line) { - self.paragraph = false; - return; - } - self.header = rest.contains('|').then(|| table::row_cells(&line)); - // A lazy line belongs to the paragraph that is already open, in its container. So does a - // line that has fewer containers than that paragraph, as it is lazy for the inner ones. - if !(self.paragraph && (line.lazy || containers < self.paragraph_containers)) { - self.paragraph_containers = containers; - } - self.paragraph = true; - return; - } - } -} - /// The line without its blockquote marker and the one space or column after it, if it has a marker. fn strip_blockquote(line: Line<'_>) -> Option<Line<'_>> { let indent = line.indent(); diff --git a/crates/mq-markdown/src/parser/block/leaf.rs b/crates/mq-markdown/src/parser/block/leaf.rs new file mode 100644 index 000000000..2d2ee61bd --- /dev/null +++ b/crates/mq-markdown/src/parser/block/leaf.rs @@ -0,0 +1,315 @@ +//! Tracks the containers and the leaf block that are open in the lines a container has collected, to +//! tell whether a following line without the container prefix continues a paragraph. + +use super::{ + FOOTNOTE_INDENT, Interrupt, ItemMarker, LineStart, MAX_DEPTH, after_blockquote_marker, interrupts_paragraph, + is_lazy_continuation, setext_depth, +}; +use crate::parser::code::Fence; +use crate::parser::definition::only_definitions; +use crate::parser::html_flow::{self, Kind as HtmlKind}; +use crate::parser::line::{Indent, Line}; +use crate::parser::mdx_flow::{FlowEnd, FlowStart, flow_end, flow_start}; +use crate::parser::table; + +/// MDX flow content that spans lines is followed for this many lines, which bounds the work. +const MAX_FLOW_LINES: usize = 64; + +/// A container that is open in the collected lines, inside the one that collects them. +enum Open { + Quote, + /// A list item, whose content is indented by `width` columns. + Item { + width: usize, + /// Whether the item started empty and has no content yet: a blank line then ends it. + empty: bool, + }, + Footnote, +} + +/// MDX flow content that goes on in the next line. +struct Pending { + text: String, + lines: usize, +} + +pub(super) struct LeafState<'a> { + /// The containers that are open, outermost first. The leaf blocks below belong to the innermost. + open: Vec<Open>, + fence: Option<Fence<'a>>, + /// An HTML block that has not ended yet. + html: Option<HtmlKind>, + /// A table that has not ended yet. Its rows cannot be continued lazily. + table: bool, + /// The number of cells of the last line when it was paragraph text with a pipe, which a delimiter + /// row with as many cells can turn into a header. + header: Option<usize>, + paragraph: bool, + /// The text of the open paragraph when it starts like a definition, which a setext underline does + /// not turn into a heading when there is nothing else in it. + definitions: Option<String>, + flow: Option<Pending>, + /// Restrictions on the next fed line, the first line of a container that interrupted a paragraph. + restricted: Interrupt, + /// How many containers can still be open before the depth limit, which no longer open any. + budget: usize, +} + +impl<'a> LeafState<'a> { + /// `depth` is the depth of the container that collects the lines. + pub(super) fn new(restricted: Interrupt, depth: usize) -> Self { + Self { + open: Vec::new(), + fence: None, + html: None, + table: false, + header: None, + paragraph: false, + definitions: None, + flow: None, + restricted, + budget: MAX_DEPTH.saturating_sub(depth + 1), + } + } + + /// Whether `line` without its container prefix continues the paragraph of the collected lines. + pub(super) fn continues_with(&self, line: &Line<'_>) -> bool { + self.paragraph && is_lazy_continuation(line) + } + + pub(super) fn feed(&mut self, line: &Line<'a>) { + self.feed_line(*line); + if !self.paragraph { + self.definitions = None; + } + } + + /// Removes the prefixes of the open containers from `line`, and returns how many of them it has. + fn strip_prefixes(&self, mut line: Line<'a>) -> (usize, Line<'a>) { + for (index, open) in self.open.iter().enumerate() { + let width = match open { + Open::Quote => { + let indent = line.indent(); + if !matches!(LineStart::of(&line, indent), Some(LineStart::Blockquote)) { + return (index, line); + } + line = after_blockquote_marker(line, indent); + continue; + } + // Another blank line ends an item that started empty. + Open::Item { empty: true, .. } if line.is_blank() => return (index, line), + Open::Item { width, .. } => *width, + Open::Footnote => FOOTNOTE_INDENT, + }; + // A blank line has the prefix of an item, whatever its indentation. + if !line.is_blank() && line.indent().columns < width { + return (index, line); + } + line = line.skip_columns(width); + } + (self.open.len(), line) + } + + /// Ends the containers from the `keep`th on, and with them what they hold. + fn close_from(&mut self, keep: usize) { + self.open.truncate(keep); + self.fence = None; + self.html = None; + self.table = false; + self.header = None; + self.paragraph = false; + self.flow = None; + } + + fn feed_line(&mut self, line: Line<'a>) { + let restricted = std::mem::take(&mut self.restricted); + let mut line = line; + // A lazy line was already accepted by the container that collects the lines. + if !line.lazy { + let (matched, rest) = self.strip_prefixes(line); + if matched < self.open.len() { + // Containers that do not match end, unless the line goes on with their paragraph. + if self.paragraph && flow_start(&rest).is_none() && is_lazy_continuation(&rest) { + line = Line { lazy: true, ..rest }; + } else { + self.close_from(matched); + line = rest; + } + } else { + line = rest; + } + if !line.is_blank() { + for open in &mut self.open { + if let Open::Item { empty, .. } = open { + *empty = false; + } + } + } + } + // A paragraph that this line is in the container of, which markers have to interrupt. + let in_paragraph = self.paragraph && !line.lazy; + + loop { + let line_indent = line.indent(); + let Indent { columns, bytes: indent } = line_indent; + let rest = &line.text[indent..]; + + if let Some(fence) = &self.fence { + if columns < line.code_indent() && fence.is_closed_by(rest) { + self.fence = None; + } + return; + } + + if let Some(kind) = self.html { + let ends = match kind { + HtmlKind::Basic | HtmlKind::Complete => line.is_blank(), + _ => html_flow::ends_in(kind, line.text), + }; + if ends { + self.html = None; + } + self.paragraph = false; + return; + } + + if let Some(pending) = &mut self.flow { + pending.text.push_str(line.eol); + pending.text.push_str(line.text); + pending.lines += 1; + let end = if pending.lines < MAX_FLOW_LINES { + flow_end(&pending.text) + } else { + FlowEnd::Done + }; + if matches!(end, FlowEnd::Open) { + return; + } + self.flow = None; + self.paragraph = matches!(end, FlowEnd::Text); + return; + } + + if line.is_blank() { + self.paragraph = false; + self.table = false; + self.header = None; + return; + } + let header = std::mem::take(&mut self.header); + if self.table { + if interrupts_paragraph(&line) { + self.table = false; + } else { + self.paragraph = false; + return; + } + } else if self.paragraph + && !line.lazy + && header.is_some() + && table::delimiter_cells(&line) == header + && !interrupts_paragraph(&line) + { + self.table = true; + self.paragraph = false; + return; + } + if columns >= line.code_indent() { + // Continues an open paragraph, otherwise it is indented code (or text when the + // container interrupted a paragraph). + if !self.paragraph { + self.paragraph = restricted.code; + } else if let Some(text) = &mut self.definitions { + text.push('\n'); + text.push_str(rest); + } + return; + } + let start = LineStart::of(&line, line_indent); + if let Some(LineStart::Fence(fence)) = start { + self.fence = Some(fence); + self.paragraph = false; + return; + } + if self.paragraph && setext_depth(&line).is_some() { + // Under nothing but definitions an underline is text, unless it is a rule. + if self.definitions.as_deref().is_some_and(only_definitions) { + self.definitions = None; + self.paragraph = !matches!(start, Some(LineStart::ThematicBreak)); + } else { + self.paragraph = false; + } + return; + } + let can_open = self.open.len() < self.budget; + match start { + Some(LineStart::Atx(_) | LineStart::ThematicBreak) => { + self.paragraph = false; + return; + } + Some(LineStart::Html(kind)) if kind != HtmlKind::Complete || !self.paragraph => { + let closed = match kind { + HtmlKind::Basic | HtmlKind::Complete => false, + _ => html_flow::ends_in(kind, &rest[html_flow::first_line_offset(kind)..]), + }; + self.html = (!closed).then_some(kind); + self.paragraph = false; + return; + } + Some(LineStart::Blockquote) if can_open => { + line = after_blockquote_marker(line, line_indent); + self.open.push(Open::Quote); + self.paragraph = false; + continue; + } + Some(LineStart::Footnote(marker)) if can_open => { + line = line.skip(marker.content); + self.open.push(Open::Footnote); + self.paragraph = false; + continue; + } + // A marker in the container of the open paragraph has to be able to interrupt it. + Some(LineStart::Item(marker)) + if can_open && (!(restricted.list || in_paragraph) || marker.interrupts_paragraph()) => + { + self.open.push(item(&marker)); + line = marker.content(line); + // The rest of the line starts a new item. + self.paragraph = false; + continue; + } + _ => {} + } + + // MDX flow content is not part of a paragraph. + if let Some(start) = flow_start(&line) { + self.paragraph = false; + if matches!(start, FlowStart::Pending) { + self.flow = Some(Pending { + text: line.text.to_string(), + lines: 1, + }); + } + return; + } + self.header = rest.contains('|').then(|| table::row_cells(&line)); + if self.paragraph { + if let Some(text) = &mut self.definitions { + text.push('\n'); + text.push_str(rest); + } + } else { + self.definitions = rest.starts_with('[').then(|| rest.to_string()); + } + self.paragraph = true; + return; + } + } +} + +fn item(marker: &ItemMarker) -> Open { + Open::Item { + width: marker.width, + empty: marker.empty, + } +} diff --git a/crates/mq-markdown/src/parser/definition.rs b/crates/mq-markdown/src/parser/definition.rs index 11de63390..d409c9a9f 100644 --- a/crates/mq-markdown/src/parser/definition.rs +++ b/crates/mq-markdown/src/parser/definition.rs @@ -117,3 +117,16 @@ pub(super) fn extract(source: &InlineSource, lines: &[Position]) -> (Vec<Node>, let used = source.lines.partition_point(|line| line.offset < next_line); (nodes, used) } + +/// Whether `text`, the lines of a paragraph, is nothing but definitions. +pub(super) fn only_definitions(text: &str) -> bool { + let bytes = text.as_bytes(); + let mut pos = skip_blanks_and_eol(bytes, 0); + while pos < text.len() && bytes[pos] == b'[' { + let Some(parsed) = parse_one(text, pos) else { + return false; + }; + pos = skip_blanks_and_eol(bytes, parsed.next); + } + pos >= text.len() +} diff --git a/crates/mq-markdown/src/parser/mdx_flow.rs b/crates/mq-markdown/src/parser/mdx_flow.rs index f74af40c6..f516c94e9 100644 --- a/crates/mq-markdown/src/parser/mdx_flow.rs +++ b/crates/mq-markdown/src/parser/mdx_flow.rs @@ -4,7 +4,7 @@ use super::error::{Located, MdxError, MdxErrorKind}; use super::line::Line; use super::mdx::{self, Fallback, Parsed}; use super::tree::{Block, InlineSource, JsxTag}; -use crate::node::{MdxFlowExpression, Node, Position}; +use crate::node::{MdxFlowExpression, Node, Point, Position}; /// What the items of a line of MDX flow content add up to. pub(super) enum Flow { @@ -182,15 +182,47 @@ pub(super) fn probe_mdx_flow(lines: &[Line<'_>], index: usize) -> Option<FlowOut (may_start_flow(line) && !line.lazy).then(|| mdx_flow(lines, index, &mut Vec::new())) } -/// Whether a single MDX line looks like flow content, for tracking what a paragraph can continue. -pub(super) fn looks_like_mdx_flow(line: &Line<'_>) -> bool { +/// What a single MDX line makes of flow content, for tracking what a paragraph can continue. +pub(super) enum FlowStart { + /// The line is flow content, or an error in it. + Complete, + /// The construct goes on in the next line. + Pending, +} + +/// Whether a single MDX line looks like flow content. +pub(super) fn flow_start(line: &Line<'_>) -> Option<FlowStart> { if !may_start_flow(line) { - return false; + return None; } // The line ending counts, unless the document ends here. let mut source = InlineSource::new(std::iter::once((line.text, "", line.point(0)))); source.text.push_str(line.eol); - !matches!(flow_items(&source), Flow::Nok(_)) + match flow_items(&source) { + Flow::Nok(_) => None, + Flow::More(_) => Some(FlowStart::Pending), + _ => Some(FlowStart::Complete), + } +} + +/// Where flow content made of the lines in `text` stands after its last line. +pub(super) enum FlowEnd { + /// It goes on in the next line. + Open, + /// It is complete, or invalid. + Done, + /// It is not flow content: the lines are text. + Text, +} + +pub(super) fn flow_end(text: &str) -> FlowEnd { + let mut source = InlineSource::new(std::iter::once((text, "\n", Point { line: 1, column: 1 }))); + source.text.push('\n'); + match flow_items(&source) { + Flow::More(_) => FlowEnd::Open, + Flow::Nok(_) => FlowEnd::Text, + _ => FlowEnd::Done, + } } /// The last line of `first..=last` that a paragraph takes when flow content spans them: it stops diff --git a/crates/mq-markdown/tests/html_rendering_tests.rs b/crates/mq-markdown/tests/html_rendering_tests.rs index 425203907..368a5131d 100644 --- a/crates/mq-markdown/tests/html_rendering_tests.rs +++ b/crates/mq-markdown/tests/html_rendering_tests.rs @@ -471,6 +471,8 @@ fn mdx_jsx_flow_children_are_rendered_as_markdown(#[case] input: &str, #[case] e "> [^n]: f\n> ===\nlazy\n\n[^n]\n", "<p>f\n===\nlazy <a href=\"#user-content-fnref-n\"" )] +#[case::definition_then_underline_in_a_quote("> [r]: /u\n> ===\nlazy\n", "<p>===\nlazy</p>")] +#[case::item_content_after_a_blank_line("> - a\n>\n> b\n> ===\nlazy\n", "<h1>b</h1>")] #[case::line_lazy_for_an_inner_quote("> > q\n> a\n> ===\nlazy\n", "<p>q\na\n===\nlazy</p>")] fn lazy_lines_continue_the_paragraph_of_the_innermost_container(#[case] input: &str, #[case] expected: &str) { let html = to_html(input); diff --git a/crates/mq-markdown/tests/parser_property_tests.rs b/crates/mq-markdown/tests/parser_property_tests.rs index 6a1b16d27..79980e5c1 100644 --- a/crates/mq-markdown/tests/parser_property_tests.rs +++ b/crates/mq-markdown/tests/parser_property_tests.rs @@ -566,16 +566,12 @@ proptest! { /// Lines that start, continue and end the blocks that decide whether a paragraph is open. Footnote /// definitions are left out: a quote hoists them, so their text is not found in the same place. -/// Definitions are left out too: a quote does not know that a paragraph is only definitions, so it reads a line of `=` -/// under one as a setext underline, where outside a quote it is text. Indented code and an -/// empty item are left out for the same reason as blank lines after an item. const LAZY_LINES: &[&str] = &[ - "a", "b c", "", "# h", "***", "---", "===", "> q", "- i", "1. i", "2. i", " more", "```", "~~~", "$$", "<div>", - "</div>", "<!-- c", "-->", "<?x", "?>", "<p>", "<span>", + "a", "b c", "", "# h", "***", "---", "===", "> q", "- i", "1. i", "2. i", "-", " more", " code", "[r]: /u", + "```", "~~~", "$$", "<div>", "</div>", "<!-- c", "-->", "<?x", "?>", "<p>", "<span>", ]; -/// The lines of MDX that start, continue and end flow content. A fence is left out of the lines with -/// them: it is text inside an expression that spans lines, which a quote cannot tell line by line. +/// The lines of MDX that start, continue and end flow content. const LAZY_MDX_LINES: &[&str] = &[ "<A>", "</A>", @@ -613,14 +609,6 @@ fn lazy_line_agrees(lines: &[&str], parse: impl Fn(&str) -> Option<Vec<Node>>) - if lines.last().is_none_or(|line| line.is_empty()) { return Ok(()); } - // A quote does not track the indentation of the items that are open, so it reads a line that an - // item holds after a blank line as code, or as text where the item has a setext underline. - let first_item = lines - .iter() - .position(|line| line.starts_with("- ") || line.starts_with(char::is_numeric)); - if first_item.is_some_and(|item| lines[item..].contains(&"")) { - return Ok(()); - } let plain = format!("{}\nlazy", lines.join("\n")); let in_quote = format!("{}\nlazy", quoted(lines)); // MDX does not allow a lazy line inside flow content, which is an error only in a quote. @@ -640,7 +628,7 @@ fn lazy_line_agrees(lines: &[&str], parse: impl Fn(&str) -> Option<Vec<Node>>) - proptest! { #[test] fn lazy_continuation_in_a_quote_agrees_with_a_paragraph_outside( - lines in prop::collection::vec(prop::sample::select(LAZY_LINES), 1..6) + lines in prop::collection::vec(prop::sample::select(LAZY_LINES), 1..10) ) { lazy_line_agrees(&lines, |input| { Markdown::from_markdown_str_with(input, ParseOptions::default().with_frontmatter(false)).ok().map(|md| md.nodes) @@ -650,10 +638,7 @@ proptest! { #[test] fn lazy_continuation_in_a_quote_agrees_with_a_paragraph_outside_in_mdx( lines in prop::collection::vec( - prop_oneof![ - prop::sample::select(LAZY_LINES).prop_filter("a fence", |line| !matches!(*line, "```" | "~~~" | "$$")), - prop::sample::select(LAZY_MDX_LINES) - ], + prop_oneof![prop::sample::select(LAZY_LINES), prop::sample::select(LAZY_MDX_LINES)], 1..6 ) ) { From 81d20c74bc506400c91dbacbe7b4fbbff19817d0 Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Fri, 9 Oct 2026 12:58:00 +0900 Subject: [PATCH 65/88] =?UTF-8?q?=F0=9F=90=9B=20fix(web):=20pass=20a=20nul?= =?UTF-8?q?l=20list=20style=20to=20keep=20the=20markers=20of=20the=20docum?= =?UTF-8?q?ent?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Without the dash default, listStyle was left out and the generated Options type, which wants null, no longer accepted it. --- packages/mq-nodejs/src/core.ts | 1 + packages/mq-web/src/core.ts | 1 + 2 files changed, 2 insertions(+) diff --git a/packages/mq-nodejs/src/core.ts b/packages/mq-nodejs/src/core.ts index 9635d46cd..08cb8679d 100644 --- a/packages/mq-nodejs/src/core.ts +++ b/packages/mq-nodejs/src/core.ts @@ -61,6 +61,7 @@ export async function run( return await wasmModule.run(code, content, { isUpdate: false, inputFormat: "markdown", + listStyle: null, linkUrlStyle: "none", linkTitleStyle: "paren", ...options, diff --git a/packages/mq-web/src/core.ts b/packages/mq-web/src/core.ts index daa221302..2fcd2e7bb 100644 --- a/packages/mq-web/src/core.ts +++ b/packages/mq-web/src/core.ts @@ -80,6 +80,7 @@ export async function run( return await wasm.run(code, content, { isUpdate: false, inputFormat: "markdown", + listStyle: null, linkUrlStyle: "none", linkTitleStyle: "paren", ...options, From 34628b409a27ad937094fd541bb61dc71aada36f Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Tue, 13 Oct 2026 20:21:47 +0900 Subject: [PATCH 66/88] =?UTF-8?q?=F0=9F=90=9B=20fix(markdown):=20end=20raw?= =?UTF-8?q?=20HTML=20at=20any=20raw=20closing=20tag=20and=20keep=20tasks,?= =?UTF-8?q?=20tables=20and=20fences=20in=20lazy=20lines=20apart?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - End a `<script>`, `<pre>`, `<style>` or `<textarea>` block at any of their closing tags, as the spec says, and not only at the one that opened it - Read `[ ]` as a task only when the next line goes on as the paragraph, so that a fence, heading, table or setext underline after it stays a block of its own - Decide whether a line is a table header for the lazy line checks the way the table rule does: a header without pipes, and a lone pipe, which has no cells - Keep the blank lines that end an unclosed fence in an item as code, so that the list stays tight --- crates/mq-markdown/src/parser/block.rs | 22 +++++++++++++--- crates/mq-markdown/src/parser/block/leaf.rs | 12 ++++++--- crates/mq-markdown/src/parser/html_flow.rs | 25 ++++++++++--------- crates/mq-markdown/src/parser/table.rs | 25 +++++++++++-------- .../mq-markdown/tests/html_rendering_tests.rs | 24 ++++++++++++++++-- 5 files changed, 77 insertions(+), 31 deletions(-) diff --git a/crates/mq-markdown/src/parser/block.rs b/crates/mq-markdown/src/parser/block.rs index 30ab1b86c..4b9d3f54b 100644 --- a/crates/mq-markdown/src/parser/block.rs +++ b/crates/mq-markdown/src/parser/block.rs @@ -500,10 +500,15 @@ fn list_item( index += 1; } + // Blank lines that end an open fence are code, and not the end of the item. + if blanks.first().is_some_and(|blank| state.fence_holds(blank)) { + inner.append(&mut blanks); + } + let checkbox = inner .iter() .position(|line| !line.is_blank()) - .and_then(|index| task_checkbox(&inner[index], inner.get(index + 1))); + .and_then(|index| task_checkbox(&inner[index..])); // Trailing blank lines belong to whatever follows the item, but still extend its position. let reaches_end = index == lines.len(); @@ -550,7 +555,8 @@ fn list_item( /// Recognises a GFM task marker (`[ ] ` or `[x] `) at the start of an item's content, returning /// the checked state and the bytes to skip. -fn task_checkbox(line: &Line<'_>, next: Option<&Line<'_>>) -> Option<bool> { +fn task_checkbox(lines: &[Line<'_>]) -> Option<bool> { + let line = &lines[0]; if !line.flavor.has_gfm() { return None; } @@ -560,14 +566,22 @@ fn task_checkbox(line: &Line<'_>, next: Option<&Line<'_>>) -> Option<bool> { [b'[', b'x' | b'X', b']'] => true, _ => return None, }; - // Whitespace follows the marker, which a line ending is when the item goes on in the next line. + // Whitespace follows the marker, which a line ending is when the paragraph goes on in the next line. match bytes.get(3) { Some(b' ' | b'\t') => Some(checked), - None if next.is_some_and(|next| !next.is_blank()) => Some(checked), + None if continues_paragraph(lines) => Some(checked), _ => None, } } +/// Whether the paragraph that starts at `lines[0]` goes on in `lines[1]`, as opposed to a blank line, +/// another block, a table or a setext underline. +fn continues_paragraph(lines: &[Line<'_>]) -> bool { + lines.get(1).is_some_and(|next| { + !interrupts_paragraph(next) && setext_depth(next).is_none() && !table::starts_at(lines, 0, interrupts_paragraph) + }) +} + /// Removes the checkbox from the text of the first paragraph of an item, and the paragraph when nothing /// else is in it. Returns whether the item starts with one. fn remove_task_marker(children: &mut Vec<Block>) -> bool { diff --git a/crates/mq-markdown/src/parser/block/leaf.rs b/crates/mq-markdown/src/parser/block/leaf.rs index 2d2ee61bd..e666cd7b6 100644 --- a/crates/mq-markdown/src/parser/block/leaf.rs +++ b/crates/mq-markdown/src/parser/block/leaf.rs @@ -41,8 +41,8 @@ pub(super) struct LeafState<'a> { html: Option<HtmlKind>, /// A table that has not ended yet. Its rows cannot be continued lazily. table: bool, - /// The number of cells of the last line when it was paragraph text with a pipe, which a delimiter - /// row with as many cells can turn into a header. + /// The number of cells of the last line when it was paragraph text, which a delimiter row with as + /// many cells can turn into a header. header: Option<usize>, paragraph: bool, /// The text of the open paragraph when it starts like a definition, which a setext underline does @@ -84,6 +84,12 @@ impl<'a> LeafState<'a> { } } + /// Whether the blank `line` is part of the fence that is open, which a container does not end for + /// the blank lines that come last in it. + pub(super) fn fence_holds(&self, blank: &Line<'a>) -> bool { + self.fence.is_some() && self.strip_prefixes(*blank).0 == self.open.len() + } + /// Removes the prefixes of the open containers from `line`, and returns how many of them it has. fn strip_prefixes(&self, mut line: Line<'a>) -> (usize, Line<'a>) { for (index, open) in self.open.iter().enumerate() { @@ -292,7 +298,7 @@ impl<'a> LeafState<'a> { } return; } - self.header = rest.contains('|').then(|| table::row_cells(&line)); + self.header = table::header_width(&line); if self.paragraph { if let Some(text) = &mut self.definitions { text.push('\n'); diff --git a/crates/mq-markdown/src/parser/html_flow.rs b/crates/mq-markdown/src/parser/html_flow.rs index c15bd4430..524ac5dc3 100644 --- a/crates/mq-markdown/src/parser/html_flow.rs +++ b/crates/mq-markdown/src/parser/html_flow.rs @@ -5,8 +5,8 @@ use super::scan::skip_blanks; /// The kind of an HTML block, which decides how it ends. #[derive(Clone, Copy, PartialEq)] pub(super) enum Kind { - /// `<script`, `<pre`, `<style` or `<textarea`: ends at a line with the matching closing tag. - Raw(&'static str), + /// `<script`, `<pre`, `<style` or `<textarea`: ends at a line with any of their closing tags. + Raw, /// `<!--`: ends at a line with `-->`. Comment, /// `<?`: ends at a line with `?>`. @@ -127,11 +127,8 @@ fn tag(text: &str, closing: bool) -> Option<Kind> { } let name = text[..name_end].to_ascii_lowercase(); - if !closing - && !self_closing - && let Some(raw) = RAW_NAMES.iter().find(|raw| **raw == name) - { - Some(Kind::Raw(raw)) + if !closing && !self_closing && RAW_NAMES.contains(&name.as_str()) { + Some(Kind::Raw) } else if BLOCK_NAMES.contains(&name.as_str()) { // A slash has to be the start of `/>`. (!self_closing || after.starts_with(b"/>")).then_some(Kind::Basic) @@ -243,7 +240,7 @@ pub(super) fn first_line_offset(kind: Kind) -> usize { Kind::Instruction => 1, Kind::Declaration => 3, Kind::Cdata => 9, - Kind::Raw(_) | Kind::Basic | Kind::Complete => 1, + Kind::Raw | Kind::Basic | Kind::Complete => 1, } } @@ -254,19 +251,23 @@ pub(super) fn ends_in(kind: Kind, text: &str) -> bool { Kind::Instruction => text.contains("?>"), Kind::Declaration => text.contains('>'), Kind::Cdata => text.contains("]]>"), - Kind::Raw(name) => has_raw_end(text, name), + Kind::Raw => has_raw_end(text), Kind::Basic | Kind::Complete => false, } } -/// Whether `text` has the closing tag of the raw element `name`, such as `</script>`. -fn has_raw_end(text: &str, name: &str) -> bool { +/// Whether `text` has the closing tag of any raw element, such as `</script>` or `</pre>`. +fn has_raw_end(text: &str) -> bool { let bytes = text.as_bytes(); let mut index = 0; while let Some(found) = text[index..].find("</") { let from = index + found + 2; let letters = bytes[from..].iter().take_while(|b| b.is_ascii_alphabetic()).count(); - if bytes.get(from + letters) == Some(&b'>') && text[from..from + letters].eq_ignore_ascii_case(name) { + if bytes.get(from + letters) == Some(&b'>') + && RAW_NAMES + .iter() + .any(|name| text[from..from + letters].eq_ignore_ascii_case(name)) + { return true; } index = from; diff --git a/crates/mq-markdown/src/parser/table.rs b/crates/mq-markdown/src/parser/table.rs index 73262c11b..f4661f2e9 100644 --- a/crates/mq-markdown/src/parser/table.rs +++ b/crates/mq-markdown/src/parser/table.rs @@ -88,6 +88,20 @@ fn align(content: &str) -> Option<TableAlignKind> { }) } +/// The cells of `line` as the header row of a table. +fn header_cells<'a>(line: &Line<'a>) -> Option<Vec<Cell<'a>>> { + // A lone pipe is both the leading and the trailing one of a row, which then has no cells. + (line.text.trim_matches([' ', '\t']) != "|").then(|| split_row(line)) +} + +/// The number of cells of `line` when a delimiter row with as many cells can make it a header. +pub(super) fn header_width(line: &Line<'_>) -> Option<usize> { + if !line.flavor.has_gfm() { + return None; + } + header_cells(line).map(|cells| cells.len()) +} + /// The alignments of a table that starts with `lines[start]` as its header, if it is one. fn header<'a>( lines: &[Line<'a>], @@ -116,11 +130,7 @@ fn header<'a>( return None; } - // A lone pipe is both the leading and the trailing one of a row, which then has no cells. - if head.text.trim_matches([' ', '\t']) == "|" { - return None; - } - let cells = split_row(head); + let cells = header_cells(head)?; let aligns = split_row(delimiter) .iter() .map(|cell| align(cell.content)) @@ -196,11 +206,6 @@ pub(super) fn parse( Some((items, index)) } -/// The number of cells in the row `line`. -pub(super) fn row_cells(line: &Line<'_>) -> usize { - split_row(line).len() -} - /// The number of cells of `line` when it is a delimiter row such as `|:-|-:|`. pub(super) fn delimiter_cells(line: &Line<'_>) -> Option<usize> { let text = line.text.trim_matches([' ', '\t']); diff --git a/crates/mq-markdown/tests/html_rendering_tests.rs b/crates/mq-markdown/tests/html_rendering_tests.rs index 368a5131d..8a2ea10ca 100644 --- a/crates/mq-markdown/tests/html_rendering_tests.rs +++ b/crates/mq-markdown/tests/html_rendering_tests.rs @@ -75,10 +75,27 @@ use rstest::rstest; "-\n [x] b\n", "<ul>\n<li><input type=\"checkbox\" checked=\"\" disabled=\"\" /> b</li>\n</ul>\n" )] -#[case::raw_html_ignores_other_closing_tags( +#[case::raw_html_ends_at_any_raw_closing_tag( "<script>\n</style>\n*hello*\n</script>\n", - "&lt;script>\n&lt;/style>\n*hello*\n&lt;/script>\n" + "&lt;script>\n&lt;/style>\n<p><em>hello</em>\n&lt;/script></p>\n" )] +#[case::raw_html_ends_at_pre_in_a_script( + "<script>\nfoo\n</pre>\n*a*\n", + "&lt;script>\nfoo\n</pre>\n<p><em>a</em></p>\n" +)] +#[case::blank_lines_in_an_unclosed_fence_keep_the_list_tight( + "- ```\n a\n\n- b\n", + "<ul>\n<li>\n<pre><code>a\n\n</code></pre>\n</li>\n<li>b</li>\n</ul>\n" +)] +#[case::blank_line_in_an_unclosed_fence_is_code( + "- ```\n\n> a\n", + "<ul>\n<li>\n<pre><code>\n</code></pre>\n</li>\n</ul>\n<blockquote>\n<p>a</p>\n</blockquote>\n" +)] +#[case::task_before_a_fence_is_not_a_task( + "- [ ]\n ```\n c\n ```\n", + "<ul>\n<li>[ ]\n<pre><code>c\n</code></pre>\n</li>\n</ul>\n" +)] +#[case::task_before_a_heading_is_not_a_task("- [ ]\n # h\n", "<ul>\n<li>[ ]\n<h1>h</h1>\n</li>\n</ul>\n")] #[case::footnote_with_blank_line_after_marker( "a[^1]\n\n[^1]:\n\n text\n", "<p>a<sup><a href=\"#user-content-fn-1\" id=\"user-content-fnref-1\" data-footnote-ref=\"\" aria-describedby=\"footnote-label\">1</a></sup></p>\n<section data-footnotes=\"\" class=\"footnotes\"><h2 id=\"footnote-label\" class=\"sr-only\">Footnotes</h2>\n<ol>\n<li id=\"user-content-fn-1\">\n<p>text <a href=\"#user-content-fnref-1\" data-footnote-backref=\"\" aria-label=\"Back to content\" class=\"data-footnote-backref\">↩</a></p>\n</li>\n</ol>\n</section>\n" @@ -473,6 +490,9 @@ fn mdx_jsx_flow_children_are_rendered_as_markdown(#[case] input: &str, #[case] e )] #[case::definition_then_underline_in_a_quote("> [r]: /u\n> ===\nlazy\n", "<p>===\nlazy</p>")] #[case::item_content_after_a_blank_line("> - a\n>\n> b\n> ===\nlazy\n", "<h1>b</h1>")] +#[case::line_after_a_lone_pipe_and_a_delimiter_in_a_quote("> a\n|\n> |-|\n| - |\n", "<p>a\n|\n|-|\n| - |</p>")] +#[case::line_after_a_table_without_header_pipe("> a\n> -|\nlazy\n", "</table>\n</blockquote>\n<p>lazy</p>")] +#[case::line_after_a_table_with_delimiter_pipes("> a\n> |-|\nlazy\n", "</table>\n</blockquote>\n<p>lazy</p>")] #[case::line_lazy_for_an_inner_quote("> > q\n> a\n> ===\nlazy\n", "<p>q\na\n===\nlazy</p>")] fn lazy_lines_continue_the_paragraph_of_the_innermost_container(#[case] input: &str, #[case] expected: &str) { let html = to_html(input); From dfadd99812d484966ebddf21cefdd22ad622bca2 Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Tue, 13 Oct 2026 20:21:51 +0900 Subject: [PATCH 67/88] =?UTF-8?q?=F0=9F=90=9B=20fix(markdown):=20keep=20th?= =?UTF-8?q?e=20structure=20of=20emphasis,=20strong=20and=20references=20wh?= =?UTF-8?q?en=20writing=20Markdown?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `Markdown::to_html` and `-F html` write the nodes and parse them again, so these came out as other nodes or lost their text: - Emphasis next to emphasis or strong, and nested at an edge, takes the delimiter that does not fuse with its neighbour into one run, alternating by depth - Strong right after emphasis that ends in `*` uses `__` - A reference followed by a colon escapes it, which would read back as a definition - A list that ends in an unclosed fence is not on the line of the block that follows it --- crates/mq-markdown/src/markdown.rs | 31 ++++++-- crates/mq-markdown/src/node/render.rs | 109 ++++++++++++++++++-------- 2 files changed, 101 insertions(+), 39 deletions(-) diff --git a/crates/mq-markdown/src/markdown.rs b/crates/mq-markdown/src/markdown.rs index 40c5445fa..c427350d6 100644 --- a/crates/mq-markdown/src/markdown.rs +++ b/crates/mq-markdown/src/markdown.rs @@ -60,6 +60,16 @@ impl fmt::Display for Markdown { } } +/// The line where the block before a node ends. An item that ends in a fence without a closing one +/// ends at the start of the line after it, which is not yet a line of its own. +fn last_line(position: &Position, node: Option<&Node>) -> usize { + if matches!(node, Some(Node::List(_))) && position.end.column == 1 && position.end.line > position.start.line { + position.end.line - 1 + } else { + position.end.line + } +} + impl Markdown { pub fn new(nodes: Vec<Node>) -> Self { Self { @@ -104,6 +114,8 @@ impl Markdown { let mut buffer = String::with_capacity(nodes.len() * 50); for (i, node) in nodes.iter().enumerate() { + let prev_node = i.checked_sub(1).and_then(|j| nodes.get(j)); + if let Node::TableCell(TableCell { row, column, values, .. }) = node @@ -133,14 +145,14 @@ impl Markdown { // Insert newlines before the first row of a table let mut new_line_count = pre_position .as_ref() - .map(|p| pos.start.line.saturating_sub(p.end.line)) + .map(|p| pos.start.line.saturating_sub(last_line(p, prev_node))) .unwrap_or_else(|| if is_first { 0 } else { 1 }) .min(2); // A footnote ends where the next line starts, and a single newline after a // block quote reads back as lazy continuation: a table needs a blank line. if new_line_count < 2 && i > 0 - && matches!(nodes[i - 1], Node::Footnote(_) | Node::Blockquote(_)) + && matches!(prev_node, Some(Node::Footnote(_) | Node::Blockquote(_))) { new_line_count = 2; } @@ -190,8 +202,6 @@ impl Markdown { current_table = None; in_table = false; - let prev_node = i.checked_sub(1).and_then(|j| nodes.get(j)); - let value = if let Node::List(list) = node && is_continuation(i) { @@ -221,7 +231,7 @@ impl Markdown { .render_with_theme(&self.options, theme) } else { list_indent_stack.clear(); - render_before(node, nodes.get(i + 1), &self.options, theme) + render_before(node, &nodes[..i], nodes.get(i + 1), &self.options, theme) }; if value.is_empty() || value == "\n" { @@ -232,7 +242,7 @@ impl Markdown { if let Some(pos) = node.position() { let mut new_line_count = pre_position .as_ref() - .map(|p| pos.start.line.saturating_sub(p.end.line)) + .map(|p| pos.start.line.saturating_sub(last_line(p, prev_node))) .unwrap_or_else(|| if is_first { 0 } else { 1 }) .min(2); @@ -585,6 +595,15 @@ mod tests { // an empty task item must not take the paragraph after it #[case::empty_task_item_then_paragraph("- [ ] \n\npara", 2, "- [ ] \n\npara\n")] #[case::empty_checked_ordered_task_item_then_paragraph("1. [x] \n\npara", 2, "1. [x] \n\npara\n")] + // a fence that is not closed ends with its item, and what follows needs a line of its own + #[case::quote_after_an_unclosed_fence_in_an_item("- ```\n a\n> q\n", 2, "- ```\n a\n ```\n> q\n")] + // emphasis next to emphasis, or nested at its edge, keeps its own run of delimiters + #[case::adjacent_emphasis("_*_*] x*_", 3, "*\\**_\\] x_\\_\n")] + #[case::emphasis_in_emphasis_at_an_edge("***a*b*", 2, "\\*_*a*b_\n")] + #[case::emphasis_after_emphasis_after_a_nested_one("\\**_*a***b**)", 4, "\\*_\\_*a*_*b*\\*)\n")] + #[case::strong_after_emphasis("*a*__b__", 2, "*a*__b__\n")] + #[case::emphasis_nested_three_deep("***a***", 1, "***a***\n")] + #[case::reference_then_colon_is_not_a_definition("[x] a\n\n[x]: /u", 3, "[x] a\n\n[x]: /u\n")] // an indented code block right after a list must not read back as list continuation #[case::code_after_list_uses_fence(" - one\n\n two", 2, "- one\n\n```\n two\n```\n")] // a block quote nested in a list item needs its continuation line's own indent diff --git a/crates/mq-markdown/src/node/render.rs b/crates/mq-markdown/src/node/render.rs index 6a61d1cfa..01f077a0b 100644 --- a/crates/mq-markdown/src/node/render.rs +++ b/crates/mq-markdown/src/node/render.rs @@ -174,14 +174,7 @@ impl Node { format!("{}~~{}~~{}", ds, render_values(values, options, theme), de) } Self::Emphasis(Emphasis { values, .. }) => { - // A lone nested Emphasis child needs the other delimiter, or adjacent - // `*` `*` pairs fuse into `**` and reparse as Strong instead. - let delim = if matches!(values.as_slice(), [Self::Emphasis(_)]) { - "_" - } else { - "*" - }; - render_emphasis(values, delim, options, theme) + render_emphasis(values, emphasis_delimiter(values), options, theme) } Self::Footnote(Footnote { values, ident, .. }) => { format!( @@ -439,29 +432,7 @@ impl Node { format!("{{{}}}", indent_expression(&mdx_text_expression.value)) } Self::MdxJsEsm(mdxjs_esm) => mdxjs_esm.value.to_string(), - Self::Strong(Strong { values, .. }) => { - let (ss, se) = &theme.strong; - let last = values.len().saturating_sub(1); - let content = values - .iter() - .enumerate() - .map(|(index, value)| match value { - // Emphasis at an end uses the other delimiter, or `*` and `**` fuse into `***`. - // An underscore does not open or close next to a letter or digit, so those keep `*`. - Self::Emphasis(Emphasis { values: inner, .. }) - if (index == 0 || index == last) - && !index - .checked_sub(1) - .is_some_and(|before| ends_with_word_char(&values[before])) - && !values.get(index + 1).is_some_and(starts_with_word_char) => - { - render_emphasis(inner, "_", options, theme) - } - value => value.render_with_theme(options, theme), - }) - .collect::<String>(); - format!("{}**{}**{}", ss, content, se) - } + Self::Strong(Strong { values, .. }) => render_strong(values, "**", options, theme), Self::Yaml(Yaml { value, .. }) => { let (fs, fe) = &theme.frontmatter; format!("{}---\n{}\n---{}", fs, value, fe) @@ -604,6 +575,48 @@ fn indent_continuation(content: &str, width: usize) -> String { result } +/// The delimiter of emphasis around `values`. Emphasis nested at an edge takes the other one, or the +/// two runs fuse into `**` and read back as strong emphasis. +fn emphasis_delimiter(values: &[Node]) -> &'static str { + let edge = [values.first(), values.last()] + .into_iter() + .flatten() + .find_map(|node| match node { + Node::Emphasis(Emphasis { values, .. }) => Some(values), + _ => None, + }); + match edge.map(|inner| emphasis_delimiter(inner)) { + Some("_") => "*", + Some(_) => "_", + None => "*", + } +} + +/// Renders strong emphasis around `values` with the delimiter `delim`. +fn render_strong(values: &[Node], delim: &str, options: &RenderOptions, theme: &ColorTheme<'_>) -> String { + let (ss, se) = &theme.strong; + let last = values.len().saturating_sub(1); + let content = values + .iter() + .enumerate() + .map(|(index, value)| match value { + // Emphasis at an end uses the other delimiter, or `*` and `**` fuse into `***`. + // An underscore does not open or close next to a letter or digit, so those keep `*`. + Node::Emphasis(Emphasis { values: inner, .. }) + if (index == 0 || index == last) + && !index + .checked_sub(1) + .is_some_and(|before| ends_with_word_char(&values[before])) + && !values.get(index + 1).is_some_and(starts_with_word_char) => + { + render_emphasis(inner, "_", options, theme) + } + value => value.render_with_theme(options, theme), + }) + .collect::<String>(); + format!("{ss}{delim}{content}{delim}{se}") +} + fn render_emphasis(values: &[Node], delim: &str, options: &RenderOptions, theme: &ColorTheme<'_>) -> String { let (es, ee) = &theme.emphasis; format!("{es}{delim}{}{delim}{ee}", render_values(values, options, theme)) @@ -638,10 +651,37 @@ fn starts_with_word_char(node: &Node) -> bool { /// Renders `value`, escaping a `!` that ends its text when `next` starts a link, which would make an image. pub(crate) fn render_before( value: &Node, + before: &[Node], next: Option<&Node>, options: &RenderOptions, theme: &ColorTheme<'_>, ) -> String { + // Emphasis right after other emphasis takes the other delimiter, or the two fuse into one run. + if let Node::Emphasis(Emphasis { values, .. }) = value + && let Some((prev, earlier)) = before.split_last() + && matches!(prev, Node::Emphasis(_) | Node::Strong(_)) + && !matches!(values.first(), Some(Node::Emphasis(_))) + && !matches!(values.last(), Some(Node::Emphasis(_))) + && !next.is_some_and(starts_with_word_char) + { + let delim = if render_before(prev, earlier, Some(value), options, theme).ends_with('_') { + "*" + } else { + "_" + }; + return render_emphasis(values, delim, options, theme); + } + // Strong right after emphasis that ends in `*` takes `__`, or the runs of `*` fuse. + if let Node::Strong(Strong { values, .. }) = value + && let Some((prev, earlier)) = before.split_last() + && matches!(prev, Node::Emphasis(_) | Node::Strong(_)) + && !next.is_some_and(starts_with_word_char) + && render_before(prev, earlier, Some(value), options, theme).ends_with('*') + { + return render_strong(values, "__", options, theme); + } + // A reference followed by a colon at the start of a line would read back as a definition. + let after_reference = matches!(before.last(), Some(Node::LinkRef(_) | Node::FootnoteRef(_))); let mut rendered = value.render_with_theme(options, theme); let starts_link = match next { Some(Node::Link(_) | Node::LinkRef(_)) => true, @@ -649,6 +689,9 @@ pub(crate) fn render_before( Some(Node::WikiLink(_)) => true, _ => false, }; + if after_reference && matches!(value, Node::Text(_)) && rendered.starts_with(':') { + rendered.insert(0, '\\'); + } if starts_link && matches!(value, Node::Text(Text { position: Some(_), .. })) && rendered.ends_with('!') { rendered.insert(rendered.len() - 1, '\\'); } @@ -661,7 +704,7 @@ pub(crate) fn render_values(values: &[Node], options: &RenderOptions, theme: &Co .iter() .enumerate() .map(|(index, value)| { - let rendered = || render_before(value, values.get(index + 1), options, theme); + let rendered = || render_before(value, &values[..index], values.get(index + 1), options, theme); if let Some(pos) = value.position() { let new_line_count = pre_position .as_ref() @@ -717,7 +760,7 @@ pub(crate) fn render_values_block(values: &[Node], options: &RenderOptions, them render_table_run(values, index, options, theme) } else { ( - render_before(first, values.get(index + 1), options, theme), + render_before(first, &values[..index], values.get(index + 1), options, theme), first.position(), index + 1, ) From 409f331145defb61803d5d5b73ae89b4bb0f3ad2 Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Tue, 13 Oct 2026 20:21:57 +0900 Subject: [PATCH 68/88] =?UTF-8?q?=F0=9F=92=A5=20feat(markdown)!:=20count?= =?UTF-8?q?=20a=20tab=20as=20one=20column=20and=20columns=20in=20UTF-16=20?= =?UTF-8?q?code=20units?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Columns of positions were bytes, except that a tab advanced to the next multiple of four. A tab is now one column and a character is as many columns as it has UTF-16 code units, which is what micromark and the LSP use. The byte order mark is not a column. The indentation that decides the structure still expands tabs. Columns are counted where they are asked for, so that a document of ASCII pays nothing and no index of the characters is built. On the parse benchmarks, ASCII prose is unchanged, Japanese prose is about 5% slower and one very long Japanese line about 10%. BREAKING CHANGE: the `column` of a position that follows a tab or a multibyte character is lower than before. --- crates/mq-markdown/benches/benchmark.rs | 38 +++ crates/mq-markdown/src/node.rs | 2 + crates/mq-markdown/src/parser/block.rs | 17 +- crates/mq-markdown/src/parser/code.rs | 2 +- crates/mq-markdown/src/parser/error.rs | 15 +- crates/mq-markdown/src/parser/line.rs | 37 ++- crates/mq-markdown/src/parser/tree.rs | 66 ++-- .../tests/parser_property_tests.rs | 14 +- .../tests/parser_structure_tests.rs | 26 +- crates/mq-markdown/tests/parser_tree_tests.rs | 290 +++++++++--------- 10 files changed, 285 insertions(+), 222 deletions(-) diff --git a/crates/mq-markdown/benches/benchmark.rs b/crates/mq-markdown/benches/benchmark.rs index 5f5577fa4..a1429328a 100644 --- a/crates/mq-markdown/benches/benchmark.rs +++ b/crates/mq-markdown/benches/benchmark.rs @@ -45,6 +45,44 @@ fn embed_doc() -> String { .collect() } +fn ascii_prose_doc() -> String { + (0..500) + .map(|i| format!("# Heading {i}\n\nSome *emphasis*, **strong** text with `code` and a [link](https://example.com/{i}) in paragraph {i}.\n\n- item *a*\n- item **b**\n\n")) + .collect() +} + +fn japanese_prose_doc() -> String { + (0..500) + .map(|i| format!("# 見出し {i}\n\n日本語の *強調* や **太字** を含む文章と `コード`、[リンク](https://example.com/{i}) を段落 {i} に入れます。\n\n- 項目 *あ*\n- 項目 **い**\n\n")) + .collect() +} + +/// One very long line, so that a position near its end is far from the start of the line. +fn japanese_long_line_doc() -> String { + (0..3000) + .map(|i| format!("語{i}は *強調* と `コード` です。")) + .collect::<String>() + + "\n" +} + +#[divan::bench(name = "parse/ascii_prose")] +fn parse_ascii_prose(bencher: divan::Bencher) { + let doc = ascii_prose_doc(); + bencher.bench(|| mq_markdown::Markdown::from_markdown_str(&doc).unwrap()); +} + +#[divan::bench(name = "parse/japanese_prose")] +fn parse_japanese_prose(bencher: divan::Bencher) { + let doc = japanese_prose_doc(); + bencher.bench(|| mq_markdown::Markdown::from_markdown_str(&doc).unwrap()); +} + +#[divan::bench(name = "parse/japanese_long_line")] +fn parse_japanese_long_line(bencher: divan::Bencher) { + let doc = japanese_long_line_doc(); + bencher.bench(|| mq_markdown::Markdown::from_markdown_str(&doc).unwrap()); +} + fn table_doc() -> String { let rows = (0..100) .map(|i| format!("| Item {i} | {} | テーブル {i} |\n", i * 10)) diff --git a/crates/mq-markdown/src/node.rs b/crates/mq-markdown/src/node.rs index 7404d3394..5b9961f0c 100644 --- a/crates/mq-markdown/src/node.rs +++ b/crates/mq-markdown/src/node.rs @@ -1076,7 +1076,9 @@ pub struct Position { serde(rename_all = "camelCase") )] pub struct Point { + /// The one-based line. pub line: usize, + /// The one-based column, counted in UTF-16 code units. A tab is one column. pub column: usize, } diff --git a/crates/mq-markdown/src/parser/block.rs b/crates/mq-markdown/src/parser/block.rs index 4b9d3f54b..b512cfdb3 100644 --- a/crates/mq-markdown/src/parser/block.rs +++ b/crates/mq-markdown/src/parser/block.rs @@ -43,7 +43,14 @@ pub(super) fn parse(src: &str, flavor: Flavor, read_frontmatter: bool) -> Vec<Bl let mut lines = split_lines(src, flavor, &stops); // A byte order mark at the start of the document is not content. if let Some(first) = lines.first_mut().filter(|line| line.text.starts_with('\u{feff}')) { - *first = first.skip('\u{feff}'.len_utf8()); + let mark = '\u{feff}'.len_utf8(); + // Columns start after it. + *first = Line { + text: &first.text[mark..], + whole: &first.whole[mark..], + base: first.base + mark, + ..*first + }; } match read_frontmatter.then(|| frontmatter(&lines)).flatten() { Some((node, next)) => { @@ -818,13 +825,7 @@ fn paragraph_source(lines: &[Line<'_>]) -> InlineSource { } else { text }; - let offset = offset_in(line.text, text); - let point = if index == 0 { - line.content_point(offset) - } else { - line.point(offset) - }; - (text, line.eol, point) + (text, line.eol, line.point(offset_in(line.text, text))) })) } diff --git a/crates/mq-markdown/src/parser/code.rs b/crates/mq-markdown/src/parser/code.rs index 47ab39075..b21fc0fb7 100644 --- a/crates/mq-markdown/src/parser/code.rs +++ b/crates/mq-markdown/src/parser/code.rs @@ -87,7 +87,7 @@ pub(super) fn fenced_code( }); let position = Some(Position { - start: lines[start].content_point(indent), + start: lines[start].point(indent), end, }); let mut value = String::with_capacity(body.iter().map(|(pad, text, eol)| pad + text.len() + eol.len()).sum()); diff --git a/crates/mq-markdown/src/parser/error.rs b/crates/mq-markdown/src/parser/error.rs index bc909078e..5525291e9 100644 --- a/crates/mq-markdown/src/parser/error.rs +++ b/crates/mq-markdown/src/parser/error.rs @@ -240,8 +240,8 @@ impl fmt::Display for MdxDiagnostic { impl std::error::Error for MdxDiagnostic {} -/// The byte offset of `point` in `content`, where columns count bytes and a tab goes to the next -/// multiple of four, as the positions of nodes do. +/// The byte offset of `point` in `content`, where columns count UTF-16 code units, as the positions of +/// nodes do. fn offset_of(content: &str, point: &Point) -> Option<usize> { let mut start = 0; for _ in 1..point.line { @@ -256,11 +256,7 @@ fn offset_of(content: &str, point: &Point) -> Option<usize> { if column + 1 >= point.column { return Some(start + index); } - column = if char == '\t' { - (column / 4 + 1) * 4 - } else { - column + char.len_utf8() - }; + column += char.len_utf16(); } Some(start + line.len()) } @@ -274,8 +270,9 @@ mod tests { #[case::first("abc", 1, 2, Some(1))] #[case::second_line("a\nbc", 2, 2, Some(3))] #[case::crlf("a\r\nbc", 2, 1, Some(3))] - #[case::tab("\tx", 1, 5, Some(1))] - #[case::multibyte("éx", 1, 3, Some(2))] + #[case::tab("\tx", 1, 2, Some(1))] + #[case::multibyte("éx", 1, 2, Some(2))] + #[case::astral("😀x", 1, 3, Some(4))] #[case::end_of_line("ab\ncd", 1, 3, Some(2))] #[case::missing_line("ab", 3, 1, None)] fn offset_of_point( diff --git a/crates/mq-markdown/src/parser/line.rs b/crates/mq-markdown/src/parser/line.rs index f5c847454..92058f0c1 100644 --- a/crates/mq-markdown/src/parser/line.rs +++ b/crates/mq-markdown/src/parser/line.rs @@ -28,6 +28,9 @@ pub(super) struct Line<'a> { pub(super) lazy: bool, /// The tabs of the line, to compute columns without scanning the line again. pub(super) stops: &'a [TabStop], + /// The whole line without its terminator, whatever a container removed from `text`, to turn byte + /// offsets into columns. + pub(super) whole: &'a str, /// Byte offset of the line in the document, to compare with `stops`. pub(super) base: usize, /// Columns left over from a tab that a container consumed only in part. They count as leading @@ -94,30 +97,13 @@ impl<'a> Line<'a> { self.text.bytes().all(|b| matches!(b, b' ' | b'\t')) } - /// The position of the byte at `byte` in `text`. Columns are in bytes, except that a tab advances - /// to the next multiple of four, as mdast positions count them. + /// The position of the byte at `byte` in `text`. Columns are in UTF-16 code units, and a tab is one. pub(super) fn point(&self, byte: usize) -> Point { let offset = self.column + byte; - let mut column = self.visual_column(offset) + 1; - // Whitespace at the start of a line inside a tab starts before the end of that tab, content - // starts where that tab ends. - if byte == 0 { - column -= self.pad; - } Point { line: self.number, - column, - } - } - - /// Like [`Line::point`] for content that mdast places where the rest of a tab ends, instead - /// of before it: paragraphs and fences. - pub(super) fn content_point(&self, byte: usize) -> Point { - let mut point = self.point(byte); - if byte == 0 { - point.column += self.pad; + column: offset - extra_bytes(&self.whole.as_bytes()[..offset.min(self.whole.len())]) + 1, } - point } /// The same line with the first `bytes` bytes removed. @@ -204,6 +190,17 @@ pub(super) fn tab_stops(src: &str) -> Vec<TabStop> { stops } +/// How many of `bytes`, which are whole characters, are more than UTF-16 code units: each continuation +/// byte is one, except that a character of four bytes is two code units. +pub(super) fn extra_bytes(bytes: &[u8]) -> usize { + let (mut continuation, mut astral) = (0, 0); + for &byte in bytes { + continuation += usize::from(byte & 0xC0 == 0x80); + astral += usize::from(byte >= 0xF0); + } + continuation - astral +} + /// The `Line::others` of `text`. fn last_others(text: &str) -> [usize; 3] { let mut last = [0; 3]; @@ -253,6 +250,7 @@ pub(super) fn split_lines<'a>(src: &'a str, flavor: Flavor, stops: &'a [TabStop] item_end: false, lazy: false, stops: take_stops(end), + whole: &src[start..end], base: start, pad: 0, flavor, @@ -271,6 +269,7 @@ pub(super) fn split_lines<'a>(src: &'a str, flavor: Flavor, stops: &'a [TabStop] item_end: false, lazy: false, stops: take_stops(src.len()), + whole: &src[start..], base: start, pad: 0, flavor, diff --git a/crates/mq-markdown/src/parser/tree.rs b/crates/mq-markdown/src/parser/tree.rs index 43ec22353..af0935468 100644 --- a/crates/mq-markdown/src/parser/tree.rs +++ b/crates/mq-markdown/src/parser/tree.rs @@ -4,6 +4,7 @@ //! it runs in [`super::resolve`] once all blocks are known. use super::error::MdxError; +use super::line::extra_bytes; use super::mdx::TagKind; use super::scan::eol_len; use crate::node::{HeadingDepth, ListMarker, MdxAttributeContent, Node, Point, Position, TableAlignKind}; @@ -23,9 +24,11 @@ pub(super) struct InlineSource { pub(super) lines: Vec<LineStart>, /// Whether this is the content of a table cell, where `\|` in code stands for `|`. pub(super) table: bool, - /// The tabs in `text` and the document column right after each, which make columns differ from - /// byte offsets. - tabs: Vec<(usize, usize)>, + /// For each line, how many bytes of `text` before it are more than code units. Empty for ASCII. + line_extra: Vec<usize>, + /// An offset in `text` and how many bytes before it are more than code units: positions are mostly + /// asked for in order, or near each other. + cursor: std::cell::Cell<(usize, usize)>, } impl InlineSource { @@ -45,34 +48,45 @@ impl InlineSource { } } let mut source = Self { - tabs: Vec::new(), text, lines: starts, table: false, + line_extra: Vec::new(), + cursor: std::cell::Cell::new((0, 0)), }; - source.index_tabs(); + source.index_lines(); source } - /// Finds the tabs of every line and the column after them. - fn index_tabs(&mut self) { - self.tabs.clear(); - if !self.text.contains('\t') { + /// Finds how far each line is in code units from its offset in bytes. + fn index_lines(&mut self) { + self.line_extra.clear(); + self.cursor.set((0, 0)); + if self.text.is_ascii() { return; } - for (index, line) in self.lines.iter().enumerate() { - let end = self.lines.get(index + 1).map_or(self.text.len(), |next| next.offset); - // The offset and the zero-based column right after the last tab or line start. - let (mut anchor, mut anchor_column) = (line.offset, line.point.column - 1); - for (at, _) in self.text[line.offset..end].match_indices('\t') { - let at = line.offset + at; - let after = ((anchor_column + (at - anchor)) / 4 + 1) * 4; - self.tabs.push((at, after)); - (anchor, anchor_column) = (at + 1, after); - } + let mut extra = 0; + let mut from = 0; + for line in &self.lines { + extra += extra_bytes(&self.text.as_bytes()[from..line.offset]); + from = line.offset; + self.line_extra.push(extra); } } + /// How many bytes of `text` before `offset` are more than UTF-16 code units. + fn extra_before(&self, offset: usize) -> usize { + let bytes = self.text.as_bytes(); + let (at, extra) = self.cursor.get(); + let extra = if offset >= at { + extra + extra_bytes(&bytes[at..offset]) + } else { + extra - extra_bytes(&bytes[offset..at]) + }; + self.cursor.set((offset, extra)); + extra + } + /// Removes the first `marker` bytes and the one space, tab or line ending that follows them, and the /// line ending after a space or a tab. pub(super) fn remove_prefix_and_one(&mut self, marker: usize) { @@ -116,7 +130,7 @@ impl InlineSource { point: line.point.clone(), }); self.lines = std::iter::once(first).chain(rest).collect(); - self.index_tabs(); + self.index_lines(); } /// The index of the line that `offset` is on. @@ -141,15 +155,15 @@ impl InlineSource { /// The position of the byte at `offset` in `text`. pub(super) fn point(&self, offset: usize) -> Point { - let LineStart { offset: start, point } = &self.lines[self.line_index(offset)]; - let passed = self.tabs.partition_point(|&(at, _)| at < offset); - let column = match passed.checked_sub(1).map(|index| self.tabs[index]) { - Some((at, after)) if at >= *start => after + (offset - at - 1) + 1, - _ => point.column + (offset - start), + let index = self.line_index(offset); + let LineStart { offset: start, point } = &self.lines[index]; + let extra = match self.line_extra.get(index) { + Some(before) => self.extra_before(offset) - before, + None => 0, }; Point { line: point.line, - column, + column: point.column + (offset - start) - extra, } } } diff --git a/crates/mq-markdown/tests/parser_property_tests.rs b/crates/mq-markdown/tests/parser_property_tests.rs index 79980e5c1..21bc79180 100644 --- a/crates/mq-markdown/tests/parser_property_tests.rs +++ b/crates/mq-markdown/tests/parser_property_tests.rs @@ -99,7 +99,19 @@ fn byte_offset(input: &str, line: usize, column: usize) -> Option<usize> { current += 1; start = index; } - Some(start + column - 1) + // Columns count UTF-16 code units. An offset inside a character is returned as such. + let target = column - 1; + let mut units = 0; + for (offset, char) in input[start..].char_indices() { + if matches!(char, '\n' | '\r') || units >= target { + return Some(start + offset + target.saturating_sub(units).min(usize::from(units < target))); + } + units += char.len_utf16(); + if units > target { + return Some(start + offset + 1); + } + } + Some(input.len()) } fn walk(nodes: Vec<Node>) -> Vec<Node> { diff --git a/crates/mq-markdown/tests/parser_structure_tests.rs b/crates/mq-markdown/tests/parser_structure_tests.rs index 705df23db..03ef2bedd 100644 --- a/crates/mq-markdown/tests/parser_structure_tests.rs +++ b/crates/mq-markdown/tests/parser_structure_tests.rs @@ -136,24 +136,24 @@ fn attributes(#[case] input: &str, #[case] name: &str, #[case] expected: &str) { // The position of an item is the one of its content. #[case::list_item("- a", 0, (1, 3), (1, 4))] #[case::emphasis_position("a *b*", 1, (1, 3), (1, 6))] -#[case::task_after_tab("- [ ]\tfoo", 0, (1, 9), (1, 12))] +#[case::task_after_tab("- [ ]\tfoo", 0, (1, 7), (1, 10))] #[case::crlf("a\r\nb", 0, (1, 1), (2, 2))] #[case::cr("a\rb", 0, (1, 1), (2, 2))] #[case::second_line_after_crlf("a\r\n\r\nb", 1, (3, 1), (3, 2))] -#[case::multibyte_paragraph("あ", 0, (1, 1), (1, 4))] -#[case::multibyte_second_node("あ\n\nい", 1, (3, 1), (3, 4))] -#[case::multibyte_emphasis("あ*い*う", 1, (1, 4), (1, 9))] -#[case::multibyte_link("あ[い](u)", 1, (1, 4), (1, 12))] -#[case::emoji("🎉 *a*", 1, (1, 6), (1, 9))] -#[case::combining_mark("e\u{301}*a*", 1, (1, 4), (1, 7))] -#[case::multibyte_heading("# 見出し", 0, (1, 1), (1, 12))] -#[case::multibyte_quote("> あ", 0, (1, 1), (1, 6))] -#[case::multibyte_list("- あ", 0, (1, 3), (1, 6))] +#[case::multibyte_paragraph("あ", 0, (1, 1), (1, 2))] +#[case::multibyte_second_node("あ\n\nい", 1, (3, 1), (3, 2))] +#[case::multibyte_emphasis("あ*い*う", 1, (1, 2), (1, 5))] +#[case::multibyte_link("あ[い](u)", 1, (1, 2), (1, 8))] +#[case::emoji("🎉 *a*", 1, (1, 4), (1, 7))] +#[case::combining_mark("e\u{301}*a*", 1, (1, 3), (1, 6))] +#[case::multibyte_heading("# 見出し", 0, (1, 1), (1, 6))] +#[case::multibyte_quote("> あ", 0, (1, 1), (1, 4))] +#[case::multibyte_list("- あ", 0, (1, 3), (1, 4))] #[case::multibyte_code("```\nあ\n```", 0, (1, 1), (3, 4))] #[case::multibyte_fence_info("```あ\nx\n```", 0, (1, 1), (3, 4))] -#[case::ideographic_space("あ\u{3000}い", 0, (1, 1), (1, 10))] -#[case::table_cell("| あ | い |\n|---|---|", 0, (1, 1), (1, 7))] -#[case::bom("\u{feff}# a", 0, (1, 4), (1, 7))] +#[case::ideographic_space("あ\u{3000}い", 0, (1, 1), (1, 4))] +#[case::table_cell("| あ | い |\n|---|---|", 0, (1, 1), (1, 5))] +#[case::bom("\u{feff}# a", 0, (1, 1), (1, 4))] #[case::after_footnote("a[^1]\n\n[^1]: b", 1, (1, 2), (1, 6))] fn positions(#[case] input: &str, #[case] index: usize, #[case] start: (usize, usize), #[case] end: (usize, usize)) { let nodes = parse(input); diff --git a/crates/mq-markdown/tests/parser_tree_tests.rs b/crates/mq-markdown/tests/parser_tree_tests.rs index f06473bb7..82a83858e 100644 --- a/crates/mq-markdown/tests/parser_tree_tests.rs +++ b/crates/mq-markdown/tests/parser_tree_tests.rs @@ -238,7 +238,7 @@ fn tree(nodes: &[Node]) -> String { #[case::paragraph_no_eol("hello", "<p>hello</p>", r#"text "hello" @1:1-1:6"#)] #[case::paragraph_multiline("a\nb\nc\n", "<p>a\nb\nc</p>\n", r#"text "a\nb\nc" @1:1-3:2"#)] #[case::paragraph_indented_continuation("a\n b\n", "<p>a\nb</p>\n", r#"text "a\nb" @1:1-2:6"#)] -#[case::paragraph_trailing_space("あい\n うえ \n", "<p>あい\nうえ</p>\n", r#"text "あい\nうえ" @1:1-2:9"#)] +#[case::paragraph_trailing_space("あい\n うえ \n", "<p>あい\nうえ</p>\n", r#"text "あい\nうえ" @1:1-2:5"#)] #[case::two_paragraphs( "a\n\nb\n", "<p>a</p>\n<p>b</p>\n", @@ -312,8 +312,8 @@ heading 2 @1:1-1:16 "# あ h #\n", "<h1>あ h</h1>\n", r#" -heading 1 @1:1-1:10 - text "あ h" @1:3-1:8 +heading 1 @1:1-1:8 + text "あ h" @1:3-1:6 "# )] #[case::atx_interrupts_paragraph( @@ -503,8 +503,8 @@ hr marker='-' @2:1-2:4 "> あ\n> い\n", "<blockquote>\n<p>あ\nい</p>\n</blockquote>\n", r#" -blockquote @1:1-2:6 - text "あ\nい" @1:3-2:6 +blockquote @1:1-2:4 + text "あ\nい" @1:3-2:4 "# )] #[case::bullet( @@ -804,10 +804,10 @@ list level=0 index=0 marker=- @1:3-3:6 "- あ\n- い\n", "<ul>\n<li>あ</li>\n<li>い</li>\n</ul>\n", r#" -list level=0 index=0 marker=- @1:3-1:6 - text "あ" @1:3-1:6 -list level=0 index=1 marker=- @2:3-2:6 - text "い" @2:3-2:6 +list level=0 index=0 marker=- @1:3-1:4 + text "あ" @1:3-1:4 +list level=0 index=1 marker=- @2:3-2:4 + text "い" @2:3-2:4 "# )] #[case::list_after_paragraph_blank( @@ -1150,15 +1150,15 @@ cell 1:1 @3:5-3:8 "| あ | い |\n|---|---|\n| う | え |\n", "<table>\n<thead>\n<tr>\n<th>あ</th>\n<th>い</th>\n</tr>\n</thead>\n<tbody>\n<tr>\n<td>う</td>\n<td>え</td>\n</tr>\n</tbody>\n</table>\n", r#" -cell 0:0 @1:1-1:7 - text "あ" @1:3-1:6 -cell 0:1 @1:7-1:14 - text "い" @1:9-1:12 +cell 0:0 @1:1-1:5 + text "あ" @1:3-1:4 +cell 0:1 @1:5-1:10 + text "い" @1:7-1:8 align none,none @2:1-2:1 -cell 1:0 @3:1-3:7 - text "う" @3:3-3:6 -cell 1:1 @3:7-3:14 - text "え" @3:9-3:12 +cell 1:0 @3:1-3:5 + text "う" @3:3-3:4 +cell 1:1 @3:5-3:10 + text "え" @3:7-3:8 "# )] #[case::not_table_setext( @@ -1508,14 +1508,14 @@ align none,none @2:1-2:1 "あ*い*う `え` [お](か)", "<p>あ<em>い</em>う <code>え</code> <a href=\"%E3%81%8B\">お</a></p>", r#" -text "あ" @1:1-1:4 -emphasis @1:4-1:9 - text "い" @1:5-1:8 -text "う " @1:9-1:13 -code_inline "え" @1:13-1:18 -text " " @1:18-1:19 -link "か" @1:19-1:29 - text "お" @1:20-1:23 +text "あ" @1:1-1:2 +emphasis @1:2-1:5 + text "い" @1:3-1:4 +text "う " @1:5-1:7 +code_inline "え" @1:7-1:10 +text " " @1:10-1:11 +link "か" @1:11-1:17 + text "お" @1:12-1:13 "# )] #[case::inline_crlf_break( @@ -2428,10 +2428,10 @@ align none,center @2:1-2:1 r#" list level=0 index=0 marker=- @1:3-1:4 text "a" @1:3-1:4 -list level=1 index=0 marker=- @2:7-2:8 - text "b" @2:7-2:8 -list level=2 index=0 marker=- @3:11-3:12 - text "c" @3:11-3:12 +list level=1 index=0 marker=- @2:4-2:5 + text "b" @2:4-2:5 +list level=2 index=0 marker=- @3:5-3:6 + text "c" @3:5-3:6 "# )] #[case::tab_nested_ordered( @@ -2440,41 +2440,41 @@ list level=2 index=0 marker=- @3:11-3:12 r#" list level=0 index=0 ordered start=1 marker=. @1:4-1:5 text "a" @1:4-1:5 -list level=1 index=0 ordered start=1 marker=. @2:8-2:9 - text "b" @2:8-2:9 -list level=2 index=0 ordered start=1 marker=. @3:12-3:13 - text "c" @3:12-3:13 +list level=1 index=0 ordered start=1 marker=. @2:5-2:6 + text "b" @2:5-2:6 +list level=2 index=0 ordered start=1 marker=. @3:6-3:7 + text "c" @3:6-3:7 "# )] #[case::tab_list_code( "- a\n\n\t\tcode\n", "<ul>\n<li>\n<p>a</p>\n<pre><code> code\n</code></pre>\n</li>\n</ul>\n", r#" -list level=0 index=0 marker=- @1:3-3:13 +list level=0 index=0 marker=- @1:3-3:7 text "a" @1:3-1:4 - code " code" @3:3-3:13 + code " code" @3:2-3:7 "# )] #[case::tab_in_text( "a\tb\t*c*\n", "<p>a\tb\t<em>c</em></p>\n", r#" -text "a\tb\t" @1:1-1:9 -emphasis @1:9-1:12 - text "c" @1:10-1:11 +text "a\tb\t" @1:1-1:5 +emphasis @1:5-1:8 + text "c" @1:6-1:7 "# )] #[case::tab_code_block( "\tcode\n\t\tmore\n", "<pre><code>code\n\tmore\n</code></pre>\n", - r#"code "code\n\tmore" @1:1-2:13"# + r#"code "code\n\tmore" @1:1-2:7"# )] #[case::tab_after_marker( "-\ta\n", "<ul>\n<li>a</li>\n</ul>\n", r#" -list level=0 index=0 marker=- @1:5-1:6 - text "a" @1:5-1:6 +list level=0 index=0 marker=- @1:3-1:4 + text "a" @1:3-1:4 "# )] #[case::atx_leading_sequences( @@ -2521,8 +2521,8 @@ heading 1 @1:1-1:7 "#\t# a", "<h1># a</h1>", r##" -heading 1 @1:1-1:8 - text "# a" @1:5-1:8 +heading 1 @1:1-1:6 + text "# a" @1:3-1:6 "## )] #[case::atx_closing_and_leading( @@ -2557,34 +2557,34 @@ list level=0 index=0 marker=- @1:3-3:3 "\u{feff}# a\n", "<h1>a</h1>\n", r#" -heading 1 @1:4-1:7 - text "a" @1:6-1:7 +heading 1 @1:1-1:4 + text "a" @1:3-1:4 "# )] -#[case::bom_frontmatter("\u{feff}---\na: b\n---\n", "", r#"yaml "a: b" @1:4-3:4"#)] -#[case::bom_text("\u{feff}a\n", "<p>a</p>\n", r#"text "a" @1:4-1:5"#)] -#[case::www_multibyte_after_ww("wwß x", "<p>wwß x</p>", r#"text "wwß x" @1:1-1:7"#)] -#[case::www_combining("ww\u{301}.a.com", "<p>ww\u{301}.a.com</p>", r#"text "ww\u{301}.a.com" @1:1-1:11"#)] +#[case::bom_frontmatter("\u{feff}---\na: b\n---\n", "", r#"yaml "a: b" @1:1-3:4"#)] +#[case::bom_text("\u{feff}a\n", "<p>a</p>\n", r#"text "a" @1:1-1:2"#)] +#[case::www_multibyte_after_ww("wwß x", "<p>wwß x</p>", r#"text "wwß x" @1:1-1:6"#)] +#[case::www_combining("ww\u{301}.a.com", "<p>ww\u{301}.a.com</p>", r#"text "ww\u{301}.a.com" @1:1-1:10"#)] #[case::www_cjk( "www.あ.com/パス x", "<p><a href=\"http://www.%E3%81%82.com/%E3%83%91%E3%82%B9\">www.あ.com/パス</a> x</p>", r#" -link "http://www.あ.com/パス" @1:1-1:19 - text "www.あ.com/パス" @1:1-1:19 -text " x" @1:19-1:21 +link "http://www.あ.com/パス" @1:1-1:13 + text "www.あ.com/パス" @1:1-1:13 +text " x" @1:13-1:15 "# )] #[case::http_cjk( "http://あ.jp/パス ok", "<p><a href=\"http://%E3%81%82.jp/%E3%83%91%E3%82%B9\">http://あ.jp/パス</a> ok</p>", r#" -link "http://あ.jp/パス" @1:1-1:21 - text "http://あ.jp/パス" @1:1-1:21 -text " ok" @1:21-1:24 +link "http://あ.jp/パス" @1:1-1:15 + text "http://あ.jp/パス" @1:1-1:15 +text " ok" @1:15-1:18 "# )] -#[case::email_cjk_domain("a@あ.com", "<p>a@あ.com</p>", r#"text "a@あ.com" @1:1-1:10"#)] -#[case::email_cjk_local("あ@b.com", "<p>あ@b.com</p>", r#"text "あ@b.com" @1:1-1:10"#)] +#[case::email_cjk_domain("a@あ.com", "<p>a@あ.com</p>", r#"text "a@あ.com" @1:1-1:8"#)] +#[case::email_cjk_local("あ@b.com", "<p>あ@b.com</p>", r#"text "あ@b.com" @1:1-1:8"#)] #[case::html_inline_continuation( "<img a\n b=\"c\">", "<p><img a\nb=\"c\"></p>", @@ -2593,7 +2593,7 @@ text " ok" @1:21-1:24 #[case::html_inline_continuation_tab( "<img a\n\t\tb=\"c\">", "<p><img a\nb=\"c\"></p>", - r#"html "<img a\nb=\"c\">" @1:1-2:15"# + r#"html "<img a\nb=\"c\">" @1:1-2:9"# )] #[case::table_trailing_space( "| a | b | \n|-|-| \n| c | d | \n", @@ -2631,7 +2631,7 @@ cell 1:1 @3:3-3:7 r#" cell 0:0 @1:1-1:5 text "a" @1:3-1:4 -cell 0:1 @1:5-1:13 +cell 0:1 @1:5-1:11 text "b" @1:7-1:8 align none,none @2:1-2:1 cell 1:0 @3:1-3:5 @@ -2644,15 +2644,15 @@ cell 1:1 @3:5-3:13 "| あ | 🎉 |\n|:-|-:|\n| é | ß |\n", "<table>\n<thead>\n<tr>\n<th align=\"left\">あ</th>\n<th align=\"right\">🎉</th>\n</tr>\n</thead>\n<tbody>\n<tr>\n<td align=\"left\">é</td>\n<td align=\"right\">ß</td>\n</tr>\n</tbody>\n</table>\n", r#" -cell 0:0 @1:1-1:7 - text "あ" @1:3-1:6 -cell 0:1 @1:7-1:15 - text "🎉" @1:9-1:13 +cell 0:0 @1:1-1:5 + text "あ" @1:3-1:4 +cell 0:1 @1:5-1:11 + text "🎉" @1:7-1:9 align left,right @2:1-2:1 -cell 1:0 @3:1-3:6 - text "é" @3:3-3:5 -cell 1:1 @3:6-3:12 - text "ß" @3:8-3:10 +cell 1:0 @3:1-3:5 + text "é" @3:3-3:4 +cell 1:1 @3:5-3:10 + text "ß" @3:7-3:8 "# )] #[case::table_single_column_colon( @@ -2721,49 +2721,49 @@ list level=0 index=0 ordered start=1 marker=. @3:1-3:3 "- a\n\n\t[P](h) x\n", "<ul>\n<li>\n<p>a</p>\n<p><a href=\"h\">P</a> x</p>\n</li>\n</ul>\n", r#" -list level=0 index=0 marker=- @1:3-3:13 +list level=0 index=0 marker=- @1:3-3:10 text "a" @1:3-1:4 - link "h" @3:5-3:11 - text "P" @3:6-3:7 - text " x" @3:11-3:13 + link "h" @3:2-3:8 + text "P" @3:3-3:4 + text " x" @3:8-3:10 "# )] #[case::fence_after_partial_tab( "- a\n\n\t```ts\n\t\tx\n\t```\n", "<ul>\n<li>\n<p>a</p>\n<pre><code class=\"language-ts\">\tx\n</code></pre>\n</li>\n</ul>\n", r#" -list level=0 index=0 marker=- @1:3-5:8 +list level=0 index=0 marker=- @1:3-5:5 text "a" @1:3-1:4 - code "\tx" lang="ts" fence @3:5-5:8 + code "\tx" lang="ts" fence @3:2-5:5 "# )] #[case::math_after_partial_tab( "- a\n\n\t$$\n\tm\n\t$$\n", "<ul>\n<li>\n<p>a</p>\n<pre><code class=\"language-math math-display\">m\n</code></pre>\n</li>\n</ul>\n", r#" -list level=0 index=0 marker=- @1:3-5:7 +list level=0 index=0 marker=- @1:3-5:4 text "a" @1:3-1:4 - math "m" @3:5-5:7 + math "m" @3:2-5:4 "# )] #[case::quote_after_partial_tab( "- a\n\n\t> q\n", "<ul>\n<li>\n<p>a</p>\n<blockquote>\n<p>q</p>\n</blockquote>\n</li>\n</ul>\n", r#" -list level=0 index=0 marker=- @1:3-3:8 +list level=0 index=0 marker=- @1:3-3:5 text "a" @1:3-1:4 - blockquote @3:3-3:8 - text "q" @3:7-3:8 + blockquote @3:2-3:5 + text "q" @3:4-3:5 "# )] #[case::heading_after_partial_tab( "- a\n\n\t# h\n", "<ul>\n<li>\n<p>a</p>\n<h1>h</h1>\n</li>\n</ul>\n", r#" -list level=0 index=0 marker=- @1:3-3:8 +list level=0 index=0 marker=- @1:3-3:5 text "a" @1:3-1:4 - heading 1 @3:3-3:8 - text "h" @3:7-3:8 + heading 1 @3:2-3:5 + text "h" @3:4-3:5 "# )] #[case::control_character_references( @@ -2845,62 +2845,62 @@ text "1." @2:1-2:3 "*あ*。*い*", "<p><em>あ</em>。<em>い</em></p>", r#" -emphasis @1:1-1:6 - text "あ" @1:2-1:5 -text "。" @1:6-1:9 -emphasis @1:9-1:14 - text "い" @1:10-1:13 +emphasis @1:1-1:4 + text "あ" @1:2-1:3 +text "。" @1:4-1:5 +emphasis @1:5-1:8 + text "い" @1:6-1:7 "# )] #[case::strong_cjk( "**日本語**です", "<p><strong>日本語</strong>です</p>", r#" -strong @1:1-1:14 - text "日本語" @1:3-1:12 -text "です" @1:14-1:20 +strong @1:1-1:8 + text "日本語" @1:3-1:6 +text "です" @1:8-1:10 "# )] #[case::underscore_accent( "_é_ _ß_", "<p><em>é</em> <em>ß</em></p>", r#" -emphasis @1:1-1:5 - text "é" @1:2-1:4 -text " " @1:5-1:6 -emphasis @1:6-1:10 - text "ß" @1:7-1:9 +emphasis @1:1-1:4 + text "é" @1:2-1:3 +text " " @1:4-1:5 +emphasis @1:5-1:8 + text "ß" @1:6-1:7 "# )] -#[case::fullwidth_markers("*a* _b_", "<p>*a* _b_</p>", r#"text "*a* _b_" @1:1-1:16"#)] +#[case::fullwidth_markers("*a* _b_", "<p>*a* _b_</p>", r#"text "*a* _b_" @1:1-1:8"#)] #[case::emphasis_after_cjk_punctuation( "。*a*、**b**", "<p>。<em>a</em>、<strong>b</strong></p>", r#" -text "。" @1:1-1:4 -emphasis @1:4-1:7 - text "a" @1:5-1:6 -text "、" @1:7-1:10 -strong @1:10-1:15 - text "b" @1:12-1:13 +text "。" @1:1-1:2 +emphasis @1:2-1:5 + text "a" @1:3-1:4 +text "、" @1:5-1:6 +strong @1:6-1:11 + text "b" @1:8-1:9 "# )] #[case::combining_mark( "e\u{301}*a*", "<p>e\u{301}<em>a</em></p>", r#" -text "e\u{301}" @1:1-1:4 -emphasis @1:4-1:7 - text "a" @1:5-1:6 +text "e\u{301}" @1:1-1:3 +emphasis @1:3-1:6 + text "a" @1:4-1:5 "# )] #[case::emoji_zwj( "👨\u{200d}👩\u{200d}👧 *a*", "<p>👨\u{200d}👩\u{200d}👧 <em>a</em></p>", r#" -text "👨\u{200d}👩\u{200d}👧 " @1:1-1:20 -emphasis @1:20-1:23 - text "a" @1:21-1:22 +text "👨\u{200d}👩\u{200d}👧 " @1:1-1:10 +emphasis @1:10-1:13 + text "a" @1:11-1:12 "# )] #[case::fence_cjk_info( @@ -2929,92 +2929,92 @@ list level=0 index=0 marker=- @1:3-3:6 "<pre><code class=\"language-math math-display\">あ\n</code></pre>\n<p><code class=\"language-math math-inline\">い</code></p>\n", r#" math "あ" @1:1-3:3 -math_inline "い" @5:1-5:6 +math_inline "い" @5:1-5:4 "# )] #[case::heading_cjk( "# 見出し\n\n見出し2\n===\n", "<h1>見出し</h1>\n<h1>見出し2</h1>\n", r#" -heading 1 @1:1-1:12 - text "見出し" @1:3-1:12 +heading 1 @1:1-1:6 + text "見出し" @1:3-1:6 heading 1 @3:1-4:4 - text "見出し2" @3:1-3:11 + text "見出し2" @3:1-3:5 "# )] #[case::link_cjk( "[あ](http://い.jp/う \"え\")", "<p><a href=\"http://%E3%81%84.jp/%E3%81%86\" title=\"え\">あ</a></p>", r#" -link "http://い.jp/う" title="え" @1:1-1:31 - text "あ" @1:2-1:5 +link "http://い.jp/う" title="え" @1:1-1:23 + text "あ" @1:2-1:3 "# )] #[case::reference_cjk( "[あ]\n\n[あ]: /u\n", "<p><a href=\"/u\">あ</a></p>\n", r#" -link_ref "あ" label="あ" @1:1-1:6 - text "あ" @1:2-1:5 -definition "あ" "/u" label="あ" @3:1-3:10 +link_ref "あ" label="あ" @1:1-1:4 + text "あ" @1:2-1:3 +definition "あ" "/u" label="あ" @3:1-3:8 "# )] #[case::footnote_cjk( "あ[^い]\n\n[^い]: う\n", "<p>あ<sup><a href=\"#user-content-fn-%E3%81%84\" id=\"user-content-fnref-%E3%81%84\" data-footnote-ref=\"\" aria-describedby=\"footnote-label\">1</a></sup></p>\n<section data-footnotes=\"\" class=\"footnotes\"><h2 id=\"footnote-label\" class=\"sr-only\">Footnotes</h2>\n<ol>\n<li id=\"user-content-fn-%E3%81%84\">\n<p>う <a href=\"#user-content-fnref-%E3%81%84\" data-footnote-backref=\"\" aria-label=\"Back to content\" class=\"data-footnote-backref\">↩</a></p>\n</li>\n</ol>\n</section>\n", r#" -text "あ" @1:1-1:4 -footnote_ref "い" label="い" @1:4-1:10 -footnote "い" @3:1-3:12 - text "う" @3:9-3:12 +text "あ" @1:1-1:2 +footnote_ref "い" label="い" @1:2-1:6 +footnote "い" @3:1-3:8 + text "う" @3:7-3:8 "# )] #[case::ideographic_space( "a\u{3000}b\n\u{3000}c", "<p>a\u{3000}b\n\u{3000}c</p>", - r#"text "a\u{3000}b\n\u{3000}c" @1:1-2:5"# + r#"text "a\u{3000}b\n\u{3000}c" @1:1-2:3"# )] #[case::nbsp_around_emphasis( "a\u{a0}*b*\u{a0}c", "<p>a\u{a0}<em>b</em>\u{a0}c</p>", r#" -text "a\u{a0}" @1:1-1:4 -emphasis @1:4-1:7 - text "b" @1:5-1:6 -text "\u{a0}c" @1:7-1:10 +text "a\u{a0}" @1:1-1:3 +emphasis @1:3-1:6 + text "b" @1:4-1:5 +text "\u{a0}c" @1:6-1:8 "# )] #[case::entity_cjk_context( "あ&amp;い&#12354;&#x1F389;", "<p>あ&amp;いあ🎉</p>", - r#"text "あ&いあ🎉" @1:1-1:29"# + r#"text "あ&いあ🎉" @1:1-1:25"# )] #[case::autolink_cjk( "<http://あ.jp/パス>", "<p><a href=\"http://%E3%81%82.jp/%E3%83%91%E3%82%B9\">http://あ.jp/パス</a></p>", r#" -link "http://あ.jp/パス" @1:1-1:23 - text "http://あ.jp/パス" @1:2-1:22 +link "http://あ.jp/パス" @1:1-1:17 + text "http://あ.jp/パス" @1:2-1:16 "# )] #[case::code_span_cjk( "`あ`と``い`う``", "<p><code>あ</code>と<code>い`う</code></p>", r#" -code_inline "あ" @1:1-1:6 -text "と" @1:6-1:9 -code_inline "い`う" @1:9-1:20 +code_inline "あ" @1:1-1:4 +text "と" @1:4-1:5 +code_inline "い`う" @1:5-1:12 "# )] #[case::hard_break_cjk( "あ \nい\\\nう", "<p>あ<br />\nい<br />\nう</p>", r#" -text "あ" @1:1-1:4 -break @1:4-2:1 -text "い" @2:1-2:4 -break @2:4-3:1 -text "う" @3:1-3:4 +text "あ" @1:1-1:2 +break @1:2-2:1 +text "い" @2:1-2:2 +break @2:2-3:1 +text "う" @3:1-3:2 "# )] #[case::nul_in_text("a\0b", "<p>a�b</p>", r#"text "a\0b" @1:1-1:4"#)] @@ -3408,25 +3408,25 @@ list level=0 index=0 marker=- @1:8-1:9 #[case::expression_lone_cr("{\r }", r#"expression_flow "\r" @1:1-2:3"#)] #[case::expression_cr_indent("{a\r b\r\n c\n d}", r#"expression_flow "a\r b\r\nc\nd" @1:1-4:5"#)] #[case::tilde_not_delimiter("*~*a", r#"text "*~*a" @1:1-1:5"#)] -#[case::tilde_strong_not_delimiter("**~**あ", r#"text "**~**あ" @1:1-1:9"#)] +#[case::tilde_strong_not_delimiter("**~**あ", r#"text "**~**あ" @1:1-1:7"#)] #[case::dollar_after_name("<a$/>", "error: Unexpected character `$` (U+0024) after name")] #[case::dollar_after_space("<a $b/>", "jsx_flow <a $b> @1:1-1:8")] -#[case::cjk_tag_name("<あ b=\"c\" />", r#"jsx_flow <あ b="c"> @1:1-1:14"#)] -#[case::fullwidth_underscore_name("<a_b />", "jsx_flow <a_b> @1:1-1:10")] +#[case::cjk_tag_name("<あ b=\"c\" />", r#"jsx_flow <あ b="c"> @1:1-1:12"#)] +#[case::fullwidth_underscore_name("<a_b />", "jsx_flow <a_b> @1:1-1:8")] #[case::jsx_text_cjk( "あ <b>い</b> う", r#" -text "あ " @1:1-1:5 -jsx_text <b> @1:5-1:15 - text "い" @1:8-1:11 -text " う" @1:15-1:19 +text "あ " @1:1-1:3 +jsx_text <b> @1:3-1:11 + text "い" @1:6-1:7 +text " う" @1:11-1:13 "# )] #[case::expression_cjk( "{あ} い", r#" -expression_text "あ" @1:1-1:6 -text " い" @1:6-1:10 +expression_text "あ" @1:1-1:4 +text " い" @1:4-1:6 "# )] fn mdx(#[case] input: &str, #[case] expected: &str) { From c67e71cae25b8d835c98ade3909cf54615c3e9c6 Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Tue, 13 Oct 2026 21:09:27 +0900 Subject: [PATCH 69/88] =?UTF-8?q?=F0=9F=92=A5=20feat(markdown)!:=20include?= =?UTF-8?q?=20the=20backslash=20of=20an=20escape=20in=20the=20position=20o?= =?UTF-8?q?f=20the=20text=20it=20starts?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Text that starts with an escape began after its backslash, while the same text after other characters began at them, so the position of `\*a` was not the source it was read from. It now starts at the backslash, as it does in micromark. BREAKING CHANGE: the start column of a text that starts with an escape is one lower than before. --- crates/mq-markdown/src/parser/inline.rs | 3 +-- crates/mq-markdown/tests/parser_structure_tests.rs | 2 ++ crates/mq-markdown/tests/parser_tree_tests.rs | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/crates/mq-markdown/src/parser/inline.rs b/crates/mq-markdown/src/parser/inline.rs index 8fc166c33..efadac51f 100644 --- a/crates/mq-markdown/src/parser/inline.rs +++ b/crates/mq-markdown/src/parser/inline.rs @@ -207,8 +207,7 @@ pub(super) fn to_nodes(items: Vec<Item>, context: &Context<'_>) -> Vec<Node> { for item in items { let (start, end, text): (usize, usize, Cow<'_, str>) = match item { Item::Text { start, end, value } => match value { - // The backslash of an escape is not part of the position of the text it starts. - Value::Slice(span) => (span.start, end, Cow::Borrowed(span.of(src))), + Value::Slice(span) => (start, end, Cow::Borrowed(span.of(src))), Value::Owned(value) => (start, end, Cow::Owned(value)), }, Item::Delim(delim) => { diff --git a/crates/mq-markdown/tests/parser_structure_tests.rs b/crates/mq-markdown/tests/parser_structure_tests.rs index 03ef2bedd..b019c56a4 100644 --- a/crates/mq-markdown/tests/parser_structure_tests.rs +++ b/crates/mq-markdown/tests/parser_structure_tests.rs @@ -154,6 +154,8 @@ fn attributes(#[case] input: &str, #[case] name: &str, #[case] expected: &str) { #[case::ideographic_space("あ\u{3000}い", 0, (1, 1), (1, 4))] #[case::table_cell("| あ | い |\n|---|---|", 0, (1, 1), (1, 5))] #[case::bom("\u{feff}# a", 0, (1, 1), (1, 4))] +#[case::escape_at_the_start("\\*a", 0, (1, 1), (1, 4))] +#[case::escape_after_text("a\\*b", 0, (1, 1), (1, 5))] #[case::after_footnote("a[^1]\n\n[^1]: b", 1, (1, 2), (1, 6))] fn positions(#[case] input: &str, #[case] index: usize, #[case] start: (usize, usize), #[case] end: (usize, usize)) { let nodes = parse(input); diff --git a/crates/mq-markdown/tests/parser_tree_tests.rs b/crates/mq-markdown/tests/parser_tree_tests.rs index 82a83858e..9ceec0d82 100644 --- a/crates/mq-markdown/tests/parser_tree_tests.rs +++ b/crates/mq-markdown/tests/parser_tree_tests.rs @@ -3267,7 +3267,7 @@ text " " @1:7-1:8 expression_text "b" @1:8-1:11 "# )] -#[case::escaped("\\<a/> \\{b}", r#"text "<a/> {b}" @1:2-1:11"#)] +#[case::escaped("\\<a/> \\{b}", r#"text "<a/> {b}" @1:1-1:11"#)] #[case::attr_entities("<a b=\"&lt;&#x41;\" />", r#"jsx_flow <a b="<A"> @1:1-1:21"#)] #[case::attr_multiline_literal("<a b='x\ny' />", r#"jsx_flow <a b="x\ny"> @1:1-2:6"#)] #[case::closing_spaces( From 79790ccec4f01c1652dbec0997429b08f7e7ba0d Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Tue, 13 Oct 2026 21:13:32 +0900 Subject: [PATCH 70/88] =?UTF-8?q?=E2=9A=A1=20perf(markdown):=20size=20the?= =?UTF-8?q?=20buffers=20of=20the=20inline=20parser=20and=20the=20resolver?= =?UTF-8?q?=20up=20front?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The vectors of the items, the slots of the delimiters, the nodes of a paragraph and the nodes of a block were grown one push at a time, which was about a fifth of the time of parsing. They are now sized from the text and the delimiters, and the nodes of a block are moved in one step. On the parse benchmarks that is about 17% faster for ASCII prose and 15 to 20% for Japanese. --- crates/mq-markdown/src/parser/inline.rs | 4 ++-- crates/mq-markdown/src/parser/inline/emphasis.rs | 4 +++- crates/mq-markdown/src/parser/resolve.rs | 11 +++++++++-- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/crates/mq-markdown/src/parser/inline.rs b/crates/mq-markdown/src/parser/inline.rs index efadac51f..ce9c5c530 100644 --- a/crates/mq-markdown/src/parser/inline.rs +++ b/crates/mq-markdown/src/parser/inline.rs @@ -159,7 +159,7 @@ pub(super) fn parse(source: &InlineSource, doc: Document<'_>) -> Result<Vec<Node }; let mut scanner = Scanner { context: &context, - items: Vec::new(), + items: Vec::with_capacity(source.text.len() / 12 + 4), openers: Vec::new(), depths: Vec::new(), inactive_below: 0, @@ -190,7 +190,7 @@ pub(super) fn to_nodes(items: Vec<Item>, context: &Context<'_>) -> Vec<Node> { } else { items }; - let mut nodes = Vec::new(); + let mut nodes = Vec::with_capacity(items.len()); // Elements that are open, with the nodes that came before them. let mut open: Vec<(TextTag, Vec<Node>)> = Vec::new(); let mut current: Option<(usize, usize, String)> = None; diff --git a/crates/mq-markdown/src/parser/inline/emphasis.rs b/crates/mq-markdown/src/parser/inline/emphasis.rs index 14ebdee73..abdaca6f2 100644 --- a/crates/mq-markdown/src/parser/inline/emphasis.rs +++ b/crates/mq-markdown/src/parser/inline/emphasis.rs @@ -108,7 +108,9 @@ impl Slots { /// Moves `items` into a list. fn new(items: &mut Vec<Item>) -> Self { let total = items.len(); - let mut slots = Vec::with_capacity(total + 1); + // Every pair of delimiters that match adds the slot of its node. + let delimiters = items.iter().filter(|item| matches!(item, Item::Delim(_))).count(); + let mut slots = Vec::with_capacity(total + delimiters / 2 + 1); slots.push(Slot { item: None, prev: Self::HEAD, diff --git a/crates/mq-markdown/src/parser/resolve.rs b/crates/mq-markdown/src/parser/resolve.rs index 1d5cd5374..f460e58d4 100644 --- a/crates/mq-markdown/src/parser/resolve.rs +++ b/crates/mq-markdown/src/parser/resolve.rs @@ -68,6 +68,13 @@ impl Frames { } } + fn extend(&mut self, nodes: Vec<Node>) { + match self.open.last_mut() { + Some((_, children)) => children.extend(nodes), + None => self.root.extend(nodes), + } + } + fn tag(&mut self, tag: JsxTag) -> Result<(), MdxError> { match tag.kind { TagKind::Open if self.open.len() >= MAX_NESTING => { @@ -128,13 +135,13 @@ fn flatten(blocks: Vec<Block>, doc: Document<'_>, nodes: &mut Vec<Node>) -> Resu Block::Inline(block) => { let mut out = Vec::new(); inline_block(block, doc, &mut out)?; - out.into_iter().for_each(|node| frames.push(node)); + frames.extend(out); } Block::Quote(quote) => frames.push(quote_node(quote, doc)?), Block::List(list) => { let mut out = Vec::new(); list_nodes(list, 0, doc, &mut out)?; - out.into_iter().for_each(|node| frames.push(node)); + frames.extend(out); } Block::Footnote(footnote) => { let mut values = Vec::new(); From 2be601dd7ebbda13d6d7e5e80562945c6aea6da8 Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Tue, 13 Oct 2026 21:18:27 +0900 Subject: [PATCH 71/88] =?UTF-8?q?=E2=9A=A1=20perf(markdown):=20count=20the?= =?UTF-8?q?=20cells=20of=20a=20header=20only=20when=20a=20delimiter=20row?= =?UTF-8?q?=20follows=20it?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The line of a paragraph was split into cells for every line of text to remember how many it had. It is kept as it is now, and counted when a line that can be a delimiter row comes after it. --- crates/mq-markdown/src/parser/block/leaf.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/mq-markdown/src/parser/block/leaf.rs b/crates/mq-markdown/src/parser/block/leaf.rs index e666cd7b6..8999dcdbd 100644 --- a/crates/mq-markdown/src/parser/block/leaf.rs +++ b/crates/mq-markdown/src/parser/block/leaf.rs @@ -41,9 +41,9 @@ pub(super) struct LeafState<'a> { html: Option<HtmlKind>, /// A table that has not ended yet. Its rows cannot be continued lazily. table: bool, - /// The number of cells of the last line when it was paragraph text, which a delimiter row with as - /// many cells can turn into a header. - header: Option<usize>, + /// The last line when it was paragraph text, which a delimiter row with as many cells can turn + /// into a header. + header: Option<Line<'a>>, paragraph: bool, /// The text of the open paragraph when it starts like a definition, which a setext underline does /// not turn into a heading when there is nothing else in it. @@ -212,8 +212,8 @@ impl<'a> LeafState<'a> { } } else if self.paragraph && !line.lazy - && header.is_some() - && table::delimiter_cells(&line) == header + && let Some(cells) = table::delimiter_cells(&line) + && header.as_ref().and_then(table::header_width) == Some(cells) && !interrupts_paragraph(&line) { self.table = true; @@ -298,7 +298,7 @@ impl<'a> LeafState<'a> { } return; } - self.header = table::header_width(&line); + self.header = Some(line); if self.paragraph { if let Some(text) = &mut self.definitions { text.push('\n'); From 75cf5c9a0450324d6f730b49e350e0e5a34aef75 Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Tue, 13 Oct 2026 21:18:27 +0900 Subject: [PATCH 72/88] =?UTF-8?q?=E2=9A=A1=20perf(markdown):=20box=20the?= =?UTF-8?q?=20nodes=20among=20the=20inline=20items=20to=20keep=20the=20ite?= =?UTF-8?q?ms=20small?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An item was as large as a node, 144 bytes, and was moved into and out of the list of slots while the delimiters were matched. Boxing the finished nodes makes an item half of that, 72 bytes. The parse benchmarks are 3 to 10% faster. --- crates/mq-markdown/src/parser/inline.rs | 20 ++++++++++++++----- .../mq-markdown/src/parser/inline/emphasis.rs | 9 ++++++++- crates/mq-markdown/src/parser/inline/link.rs | 5 ++++- 3 files changed, 27 insertions(+), 7 deletions(-) diff --git a/crates/mq-markdown/src/parser/inline.rs b/crates/mq-markdown/src/parser/inline.rs index ce9c5c530..a9e5014df 100644 --- a/crates/mq-markdown/src/parser/inline.rs +++ b/crates/mq-markdown/src/parser/inline.rs @@ -65,9 +65,10 @@ pub(super) enum Item { end: usize, value: Value, }, - /// A finished node and how deeply it nests: a leaf is 1. + /// A finished node and how deeply it nests: a leaf is 1. It is boxed to keep the items small, which + /// are moved around while delimiters are matched. Node { - node: Node, + node: Box<Node>, depth: usize, }, Delim(Delim), @@ -220,7 +221,7 @@ pub(super) fn to_nodes(items: Vec<Item>, context: &Context<'_>) -> Vec<Node> { } Item::Node { node, .. } => { finish(&mut current, &mut nodes); - nodes.push(node); + nodes.push(*node); continue; } Item::Jsx(tag) => { @@ -519,7 +520,13 @@ impl Scanner<'_> { } fn push_node(&mut self, node: Node, end: usize) { - self.push(Item::Node { node, depth: 1 }, end); + self.push( + Item::Node { + node: Box::new(node), + depth: 1, + }, + end, + ); } fn escape(&mut self) { @@ -590,7 +597,10 @@ impl Scanner<'_> { position: Some(self.context.position(start, pos + eol)), }); self.note_depth(1); - self.items.push(Item::Node { node, depth: 1 }); + self.items.push(Item::Node { + node: Box::new(node), + depth: 1, + }); } else { self.items.push(Item::Text { start: pos, diff --git a/crates/mq-markdown/src/parser/inline/emphasis.rs b/crates/mq-markdown/src/parser/inline/emphasis.rs index abdaca6f2..ba1b7e547 100644 --- a/crates/mq-markdown/src/parser/inline/emphasis.rs +++ b/crates/mq-markdown/src/parser/inline/emphasis.rs @@ -327,7 +327,14 @@ pub(super) fn process(items: &mut Vec<Item>, context: &Context<'_>) { (_, 2) => Node::Strong(Strong { values, position }), _ => Node::Emphasis(Emphasis { values, position }), }; - slots.insert_between(opener, closer, Item::Node { node, depth: depth + 1 }); + slots.insert_between( + opener, + closer, + Item::Node { + node: Box::new(node), + depth: depth + 1, + }, + ); slots.delim_mut(opener).count -= used; let close = slots.delim_mut(closer); diff --git a/crates/mq-markdown/src/parser/inline/link.rs b/crates/mq-markdown/src/parser/inline/link.rs index 4fc579136..9aa491634 100644 --- a/crates/mq-markdown/src/parser/inline/link.rs +++ b/crates/mq-markdown/src/parser/inline/link.rs @@ -126,7 +126,10 @@ pub(super) fn close(scanner: &mut Scanner<'_>) -> bool { } scanner.note_depth(depth); - scanner.items.push(Item::Node { node, depth }); + scanner.items.push(Item::Node { + node: Box::new(node), + depth, + }); scanner.pos = end; scanner.run = end; true From 40562a4a380ae782afb267a0a9721ab4fef1fa83 Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Tue, 13 Oct 2026 21:24:22 +0900 Subject: [PATCH 73/88] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor(markdown):?= =?UTF-8?q?=20share=20the=20collection=20of=20the=20lines=20of=20a=20conta?= =?UTF-8?q?iner=20between=20quote,=20item=20and=20footnote?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The three loops took lines for the container, held blank lines back and took lazy continuation lines with the same steps. They are one `Collector` now. --- crates/mq-markdown/src/parser/block.rs | 83 +++++++------------ .../mq-markdown/src/parser/block/collect.rs | 59 +++++++++++++ 2 files changed, 88 insertions(+), 54 deletions(-) create mode 100644 crates/mq-markdown/src/parser/block/collect.rs diff --git a/crates/mq-markdown/src/parser/block.rs b/crates/mq-markdown/src/parser/block.rs index b512cfdb3..fa788dc04 100644 --- a/crates/mq-markdown/src/parser/block.rs +++ b/crates/mq-markdown/src/parser/block.rs @@ -12,12 +12,13 @@ use super::inline; use super::line::{Indent, Line, split_lines, tab_stops}; use super::mdx_flow::{FlowOutcome, absorbed_until, blocks_lazy_continuation, probe_mdx_flow}; use super::table; +mod collect; mod leaf; mod rule; use super::tree::{Block, FootnoteBlock, InlineBlock, InlineKind, InlineSource, Item, ListBlock, QuoteBlock}; use crate::node::{HeadingDepth, Html, ListMarker, Node, Point, Position, Toml, Yaml}; -use leaf::LeafState; +use collect::Collector; use rule::{BlockRule, Cx, Rules, fallback}; /// Lines indented by this many columns or more are code, not other blocks. @@ -206,23 +207,19 @@ fn blockquote( interrupting: Interrupt, blocks: &mut Vec<Block>, ) -> usize { - let mut inner = Vec::new(); - let mut state = LeafState::new(interrupting, depth); + let mut collector = Collector::new(depth, interrupting); let mut index = start; while let Some(line) = lines.get(index) { - let stripped = match strip_blockquote(*line) { - Some(stripped) => stripped, + match strip_blockquote(*line) { + Some(stripped) => collector.push(stripped), // Lazy continuation of a paragraph. - None if state.continues_with(line) && !blocks_lazy_continuation(lines, index) => { - Line { lazy: true, ..*line } - } + None if collector.lazy(lines, index) => {} None => break, - }; - state.feed(&stripped); - inner.push(stripped); + } index += 1; } + let inner = collector.lines; let children = parse_blocks(&inner, depth + 1, interrupting); let mut end = inner.last().map_or_else(|| lines[start].end(), Line::end); @@ -471,46 +468,34 @@ fn list_item( interrupting: Interrupt, ) -> (Item, usize) { let first = marker.content(lines[start]); - let mut inner = vec![first]; - let mut state = LeafState::new(interrupting, depth); - state.feed(&first); - let mut blanks = Vec::new(); + let mut collector = Collector::new(depth, interrupting); + collector.push(first); let mut last_blank_has_whitespace = false; let mut index = start + 1; while let Some(line) = lines.get(index) { if line.is_blank() { // An item that starts with a blank line cannot continue after another one. - if first.is_blank() && inner.len() == 1 { + if first.is_blank() && collector.lines.len() == 1 { break; } last_blank_has_whitespace = !line.text.is_empty(); - blanks.push(line.skip_columns(marker.width)); + collector.blank(line.skip_columns(marker.width)); } else if line.lazy { - state.feed(line); - inner.push(*line); + collector.push(*line); } else if line.indent().columns >= marker.width { - for blank in &blanks { - state.feed(blank); - } - inner.append(&mut blanks); - let stripped = line.skip_columns(marker.width); - state.feed(&stripped); - inner.push(stripped); - } else if blanks.is_empty() && state.continues_with(line) && !blocks_lazy_continuation(lines, index) { - let lazy = Line { lazy: true, ..*line }; - state.feed(&lazy); - inner.push(lazy); - } else { + collector.push(line.skip_columns(marker.width)); + } else if !collector.lazy(lines, index) { break; } index += 1; } - - // Blank lines that end an open fence are code, and not the end of the item. - if blanks.first().is_some_and(|blank| state.fence_holds(blank)) { - inner.append(&mut blanks); - } + collector.keep_fence_blanks(); + let Collector { + lines: mut inner, + blanks, + .. + } = collector; let checkbox = inner .iter() @@ -882,33 +867,23 @@ fn footnote( interrupting: Interrupt, blocks: &mut Vec<Block>, ) -> usize { - let first = lines[start].skip(marker.content); - let mut inner = vec![first]; - let mut state = LeafState::new(interrupting, depth); - state.feed(&first); - let mut blanks = Vec::new(); + let mut collector = Collector::new(depth, interrupting); + collector.push(lines[start].skip(marker.content)); let mut index = start + 1; while let Some(line) = lines.get(index) { if line.is_blank() { - blanks.push(line.skip(line.text.len())); + collector.blank(line.skip(line.text.len())); } else if line.indent().columns >= FOOTNOTE_INDENT { - for blank in &blanks { - state.feed(blank); - } - inner.append(&mut blanks); - let stripped = line.skip_columns(FOOTNOTE_INDENT); - state.feed(&stripped); - inner.push(stripped); - } else if blanks.is_empty() && state.continues_with(line) && !blocks_lazy_continuation(lines, index) { - let lazy = Line { lazy: true, ..*line }; - state.feed(&lazy); - inner.push(lazy); - } else { + collector.push(line.skip_columns(FOOTNOTE_INDENT)); + } else if !collector.lazy(lines, index) { break; } index += 1; } + let Collector { + lines: inner, blanks, .. + } = collector; // Trailing blank lines belong to whatever follows, but still extend the position. index -= blanks.len(); diff --git a/crates/mq-markdown/src/parser/block/collect.rs b/crates/mq-markdown/src/parser/block/collect.rs new file mode 100644 index 000000000..a6c0f99e1 --- /dev/null +++ b/crates/mq-markdown/src/parser/block/collect.rs @@ -0,0 +1,59 @@ +//! The lines of a container, collected for as long as they belong to it. + +use super::leaf::LeafState; +use super::{Interrupt, blocks_lazy_continuation}; +use crate::parser::line::Line; + +/// The lines of a container without its prefix, which a quote, an item or a footnote takes one by +/// one, and the blank lines that wait after them. +pub(super) struct Collector<'a> { + /// The lines that belong to the container. + pub(super) lines: Vec<Line<'a>>, + /// The blank lines after the last one. They end the container unless a line that belongs to it + /// follows. + pub(super) blanks: Vec<Line<'a>>, + state: LeafState<'a>, +} + +impl<'a> Collector<'a> { + /// `depth` is the depth of the container. + pub(super) fn new(depth: usize, interrupting: Interrupt) -> Self { + Self { + lines: Vec::new(), + blanks: Vec::new(), + state: LeafState::new(interrupting, depth), + } + } + + /// Takes `line`, which belongs to the container, together with the blank lines before it. + pub(super) fn push(&mut self, line: Line<'a>) { + for blank in &self.blanks { + self.state.feed(blank); + } + self.lines.append(&mut self.blanks); + self.state.feed(&line); + self.lines.push(line); + } + + /// Holds a blank line back until a line that belongs to the container follows it. + pub(super) fn blank(&mut self, line: Line<'a>) { + self.blanks.push(line); + } + + /// Takes `lines[index]` as a lazy continuation of the paragraph that is open, if it can be one. + pub(super) fn lazy(&mut self, lines: &[Line<'a>], index: usize) -> bool { + let line = &lines[index]; + if !self.blanks.is_empty() || !self.state.continues_with(line) || blocks_lazy_continuation(lines, index) { + return false; + } + self.push(Line { lazy: true, ..*line }); + true + } + + /// Keeps the blank lines that end an open fence as code, and not as the end of the container. + pub(super) fn keep_fence_blanks(&mut self) { + if self.blanks.first().is_some_and(|blank| self.state.fence_holds(blank)) { + self.lines.append(&mut self.blanks); + } + } +} From ad08112b6bdd5fdaf90f56b1ffe46cdd8ed8da7e Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Tue, 13 Oct 2026 21:31:09 +0900 Subject: [PATCH 74/88] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor(markdown):?= =?UTF-8?q?=20move=20lists,=20footnotes=20and=20HTML=20blocks=20out=20of?= =?UTF-8?q?=20the=20block=20rules?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `block.rs` held the line rules, the quote, the list and its items, the footnote, the HTML block and the paragraph. The lists, footnotes and HTML blocks are in files of their own next to `leaf.rs` and `rule.rs`, with what each of them needs named at the top. The behavior is the same. --- crates/mq-markdown/src/parser/block.rs | 453 +----------------- .../mq-markdown/src/parser/block/footnote.rs | 96 ++++ crates/mq-markdown/src/parser/block/html.rs | 70 +++ crates/mq-markdown/src/parser/block/list.rs | 291 +++++++++++ 4 files changed, 464 insertions(+), 446 deletions(-) create mode 100644 crates/mq-markdown/src/parser/block/footnote.rs create mode 100644 crates/mq-markdown/src/parser/block/html.rs create mode 100644 crates/mq-markdown/src/parser/block/list.rs diff --git a/crates/mq-markdown/src/parser/block.rs b/crates/mq-markdown/src/parser/block.rs index fa788dc04..66e008bd4 100644 --- a/crates/mq-markdown/src/parser/block.rs +++ b/crates/mq-markdown/src/parser/block.rs @@ -13,20 +13,23 @@ use super::line::{Indent, Line, split_lines, tab_stops}; use super::mdx_flow::{FlowOutcome, absorbed_until, blocks_lazy_continuation, probe_mdx_flow}; use super::table; mod collect; +mod footnote; +mod html; mod leaf; +mod list; mod rule; use super::tree::{Block, FootnoteBlock, InlineBlock, InlineKind, InlineSource, Item, ListBlock, QuoteBlock}; use crate::node::{HeadingDepth, Html, ListMarker, Node, Point, Position, Toml, Yaml}; use collect::Collector; +use footnote::{FOOTNOTE_INDENT, FootnoteMarker, footnote, footnote_marker}; +use html::{html_block, html_start}; +use list::{ItemMarker, list}; use rule::{BlockRule, Cx, Rules, fallback}; /// Lines indented by this many columns or more are code, not other blocks. pub(super) const CODE_INDENT: usize = 4; -/// The indentation of the lines that continue a footnote definition. -const FOOTNOTE_INDENT: usize = 4; - /// Containers nested deeper than this are parsed as plain paragraph text, which bounds recursion. const MAX_DEPTH: usize = 128; @@ -142,7 +145,7 @@ fn parse_blocks(lines: &[Line<'_>], depth: usize, interrupting: Interrupt) -> Ve /// /// The constructs are told apart by their first character, so a line starts at most one of them. Block /// rules, lazy continuation and the end of paragraphs and tables all decide from this one result. -pub(super) enum LineStart<'a> { +enum LineStart<'a> { Fence(Fence<'a>), /// An ATX heading of this depth. Atx(HeadingDepth), @@ -238,156 +241,6 @@ fn blockquote( index } -#[derive(Clone)] -pub(super) struct ItemMarker { - ordered: bool, - kind: ListMarker, - start: u32, - /// Bytes from the start of the line to the end of the marker. - after_marker: usize, - /// Columns of whitespace between the marker and the content that belong to the marker. - gap: usize, - /// Columns that continuation lines must be indented by to belong to the item. - width: usize, - /// Whether the first line has no content after the marker. - empty: bool, -} - -impl ItemMarker { - fn parse(line: &Line<'_>) -> Option<Self> { - let Indent { columns, bytes: indent } = line.indent(); - if columns >= line.code_indent() { - return None; - } - let rest = &line.text[indent..]; - let bytes = rest.as_bytes(); - if is_thematic_break(line, indent) { - return None; - } - - let (ordered, start, marker_len) = match *bytes.first()? { - b'-' | b'+' | b'*' => (false, 0, 1), - b'0'..=b'9' => { - let digits = bytes.iter().take_while(|b| b.is_ascii_digit()).count(); - if digits > 9 || !matches!(bytes.get(digits), Some(b'.' | b')')) { - return None; - } - (true, rest[..digits].parse().ok()?, digits + 1) - } - _ => return None, - }; - let kind = match bytes[marker_len - 1] { - b'-' => ListMarker::Dash, - b'+' => ListMarker::Plus, - b'*' => ListMarker::Star, - b'.' => ListMarker::Period, - _ => ListMarker::Paren, - }; - - let after = &rest[marker_len..]; - let Indent { - columns: spaces, - bytes: blanks, - } = line.skip(indent + marker_len).indent(); - if blanks == 0 && !after.is_empty() { - return None; - } - - let empty = blanks == after.len(); - // Five or more columns means the content is indented code, so only one belongs to the marker. - let gap = if empty || spaces > line.code_indent() { - 1 - } else { - spaces - }; - - Some(Self { - ordered, - kind, - start, - after_marker: indent + marker_len, - gap, - width: columns + marker_len + gap, - empty, - }) - } - - /// The line without the marker and the whitespace that belongs to it. - fn content<'a>(&self, line: Line<'a>) -> Line<'a> { - if self.empty { - line.skip(line.text.len()) - } else { - line.skip(self.after_marker).skip_columns(self.gap) - } - } - - fn same_list(&self, other: &Self) -> bool { - self.ordered == other.ordered && self.kind == other.kind - } - - /// Whether this marker may interrupt a paragraph. - fn interrupts_paragraph(&self) -> bool { - !self.empty && (!self.ordered || self.start == 1) - } -} - -fn list( - lines: &[Line<'_>], - start: usize, - first: &ItemMarker, - depth: usize, - interrupting: Interrupt, - blocks: &mut Vec<Block>, -) -> usize { - let mut items = Vec::new(); - let mut spread = false; - let mut index = start; - let mut marker = first.clone(); - - loop { - let (item, end) = list_item( - lines, - index, - &marker, - depth, - if items.is_empty() { - interrupting - } else { - Interrupt::default() - }, - ); - items.push(item); - - // Blank lines between items make the list loose. - let next = (end..lines.len()).find(|&i| !lines[i].is_blank()).and_then(|i| { - ItemMarker::parse(&lines[i]) - .filter(|m| m.same_list(first)) - .map(|m| (i, m)) - }); - match next { - Some((next, next_marker)) => { - spread |= next > end; - index = next; - marker = next_marker; - } - None => { - index = end; - break; - } - } - } - - blocks.push(Block::List(ListBlock { - ordered: first.ordered, - start: first.ordered.then_some(first.start), - marker: first.kind, - spread, - items, - })); - - index -} - /// The end of the last node that `block` produces. fn last_end(block: &Block) -> Option<Point> { match block { @@ -459,141 +312,6 @@ fn extend_quote(quote: &mut QuoteBlock, end: &Point) { } } -/// Collects one item starting at `start`, returning it and the index of the first line after it. -fn list_item( - lines: &[Line<'_>], - start: usize, - marker: &ItemMarker, - depth: usize, - interrupting: Interrupt, -) -> (Item, usize) { - let first = marker.content(lines[start]); - let mut collector = Collector::new(depth, interrupting); - collector.push(first); - let mut last_blank_has_whitespace = false; - let mut index = start + 1; - - while let Some(line) = lines.get(index) { - if line.is_blank() { - // An item that starts with a blank line cannot continue after another one. - if first.is_blank() && collector.lines.len() == 1 { - break; - } - last_blank_has_whitespace = !line.text.is_empty(); - collector.blank(line.skip_columns(marker.width)); - } else if line.lazy { - collector.push(*line); - } else if line.indent().columns >= marker.width { - collector.push(line.skip_columns(marker.width)); - } else if !collector.lazy(lines, index) { - break; - } - index += 1; - } - collector.keep_fence_blanks(); - let Collector { - lines: mut inner, - blanks, - .. - } = collector; - - let checkbox = inner - .iter() - .position(|line| !line.is_blank()) - .and_then(|index| task_checkbox(&inner[index..])); - - // Trailing blank lines belong to whatever follows the item, but still extend its position. - let reaches_end = index == lines.len(); - index -= blanks.len(); - - // An unclosed fence in an item includes its line terminator when the next line is blank or starts - // another container, but not when a plain paragraph or thematic break follows. - let next = lines.get(index); - let item_end = next.is_none_or(|line| { - line.is_blank() - || matches!( - LineStart::of(line, line.indent()), - Some(LineStart::Blockquote | LineStart::Item(_)) - ) - }); - if let Some(last) = inner.last_mut() { - last.item_end = item_end; - } - - let content_end = inner.last().map_or_else(|| lines[start].end(), Line::end); - let position = Position { - start: lines[start].point(0), - end: blanks.last().map_or_else(|| content_end.clone(), Line::end), - }; - let mut children = parse_blocks(&inner, depth + 1, interrupting); - // The checkbox is the start of the text of the first paragraph. - let checked = checkbox.filter(|_| remove_task_marker(&mut children)); - if let Some(blank) = blanks.last() { - extend_last_items( - &mut children, - &blank.end(), - last_blank_has_whitespace && reaches_end && blanks.len() == 1 && blank.eol.is_empty(), - ); - } - let item = Item { - checked, - children, - position, - content_end, - }; - - (item, index) -} - -/// Recognises a GFM task marker (`[ ] ` or `[x] `) at the start of an item's content, returning -/// the checked state and the bytes to skip. -fn task_checkbox(lines: &[Line<'_>]) -> Option<bool> { - let line = &lines[0]; - if !line.flavor.has_gfm() { - return None; - } - let bytes = line.text.as_bytes(); - let checked = match bytes.get(..3)? { - [b'[', b' ', b']'] => false, - [b'[', b'x' | b'X', b']'] => true, - _ => return None, - }; - // Whitespace follows the marker, which a line ending is when the paragraph goes on in the next line. - match bytes.get(3) { - Some(b' ' | b'\t') => Some(checked), - None if continues_paragraph(lines) => Some(checked), - _ => None, - } -} - -/// Whether the paragraph that starts at `lines[0]` goes on in `lines[1]`, as opposed to a blank line, -/// another block, a table or a setext underline. -fn continues_paragraph(lines: &[Line<'_>]) -> bool { - lines.get(1).is_some_and(|next| { - !interrupts_paragraph(next) && setext_depth(next).is_none() && !table::starts_at(lines, 0, interrupts_paragraph) - }) -} - -/// Removes the checkbox from the text of the first paragraph of an item, and the paragraph when nothing -/// else is in it. Returns whether the item starts with one. -fn remove_task_marker(children: &mut Vec<Block>) -> bool { - let Some(Block::Inline(InlineBlock { - source, - kind: InlineKind::Paragraph, - })) = children.first_mut() - else { - return false; - }; - if !matches!(source.text.as_bytes().get(..3), Some([b'[', b' ' | b'x' | b'X', b']'])) { - return false; - } - source.remove_prefix_and_one(3); - if source.text.is_empty() { - children.remove(0); - } - true -} - /// Returns the heading depth when `rest` (indent already removed) is an ATX heading line. fn atx_depth(rest: &str) -> Option<HeadingDepth> { let hashes = rest.bytes().take_while(|&b| b == b'#').count(); @@ -813,160 +531,3 @@ fn paragraph_source(lines: &[Line<'_>]) -> InlineSource { (text, line.eol, line.point(offset_in(line.text, text))) })) } - -/// The start of a footnote definition: `[^label]:`. -pub(super) struct FootnoteMarker<'a> { - label: &'a str, - /// Bytes from the start of the line to the content. - content: usize, -} - -fn footnote_marker<'a>(line: &Line<'a>) -> Option<FootnoteMarker<'a>> { - if !line.flavor.has_gfm() { - return None; - } - let Indent { columns, bytes: indent } = line.indent(); - let rest = line.text[indent..].strip_prefix("[^")?; - if columns >= line.code_indent() { - return None; - } - // A backslash escapes the bracket that follows it, as well as another backslash. - let bytes = rest.as_bytes(); - let mut close = 0; - while close < bytes.len() && bytes[close] != b']' { - if bytes[close] == b'[' { - return None; - } - close += if bytes[close] == b'\\' && matches!(bytes.get(close + 1), Some(b'[' | b'\\' | b']')) { - 2 - } else { - 1 - }; - } - if close >= bytes.len() { - return None; - } - let label = &rest[..close]; - if label.is_empty() || label.bytes().any(|b| matches!(b, b' ' | b'\t' | b'\r' | b'\n')) { - return None; - } - let after = rest[close + 1..].strip_prefix(':')?; - let blanks = after.len() - after.trim_start_matches([' ', '\t']).len(); - Some(FootnoteMarker { - label, - content: indent + 2 + close + 2 + blanks, - }) -} - -/// Collects a footnote definition, which continues on lines indented by four columns. -fn footnote( - lines: &[Line<'_>], - start: usize, - marker: &FootnoteMarker<'_>, - depth: usize, - interrupting: Interrupt, - blocks: &mut Vec<Block>, -) -> usize { - let mut collector = Collector::new(depth, interrupting); - collector.push(lines[start].skip(marker.content)); - let mut index = start + 1; - - while let Some(line) = lines.get(index) { - if line.is_blank() { - collector.blank(line.skip(line.text.len())); - } else if line.indent().columns >= FOOTNOTE_INDENT { - collector.push(line.skip_columns(FOOTNOTE_INDENT)); - } else if !collector.lazy(lines, index) { - break; - } - index += 1; - } - let Collector { - lines: inner, blanks, .. - } = collector; - - // Trailing blank lines belong to whatever follows, but still extend the position. - index -= blanks.len(); - let end = blanks - .last() - .or(inner.last()) - .map_or_else(|| lines[start].end(), Line::end); - let position = Position { - start: lines[start].point(0), - end, - }; - blocks.push(Block::Footnote(FootnoteBlock { - ident: inline::normalize(marker.label), - children: parse_blocks(&inner, depth + 1, interrupting), - position, - })); - - index -} - -/// Collects an HTML block of `kind` starting at `lines[start]`, which is indented less than four columns. -fn html_block(lines: &[Line<'_>], start: usize, kind: HtmlKind, blocks: &mut Vec<Block>) -> usize { - let first = &lines[start]; - let mut last = start; - let mut unterminated = false; - - match kind { - HtmlKind::Basic | HtmlKind::Complete => { - // Ends before a blank line, and lines cannot continue a paragraph lazily. - for (index, line) in lines.iter().enumerate().skip(start + 1) { - if line.is_blank() || line.lazy { - break; - } - last = index; - } - } - _ => { - let indent = first.indent().bytes; - let from = indent + html_flow::first_line_offset(kind); - if !html_flow::ends_in(kind, &first.text[from..]) { - let mut ended = false; - for (index, line) in lines.iter().enumerate().skip(start + 1) { - if line.lazy { - break; - } - last = index; - if html_flow::ends_in(kind, line.text) { - ended = true; - break; - } - } - // A block that never ends runs to the end of the document, line ending included. - unterminated = !ended && lines[last].eof && !lines[last].eol.is_empty(); - } - } - } - - let parts = lines[start..=last] - .iter() - .map(|line| (line.text, line.eol)) - .collect::<Vec<_>>(); - let mut value = join_lines(&parts); - let end = if unterminated { - value.push_str(lines[last].eol); - Point { - line: lines[last].number + 1, - column: 1, - } - } else { - lines[last].end() - }; - blocks.push(Block::Node(Node::Html(Html { - value, - position: Some(Position { - start: first.point(0), - end, - }), - }))); - - last + 1 -} - -/// The kind of HTML block that starts at `rest`, in a flavor that has HTML. -fn html_start(line: &Line<'_>, rest: &str) -> Option<HtmlKind> { - line.flavor.has_html().then(|| html_flow::start(rest)).flatten() -} diff --git a/crates/mq-markdown/src/parser/block/footnote.rs b/crates/mq-markdown/src/parser/block/footnote.rs new file mode 100644 index 000000000..d9b4a4269 --- /dev/null +++ b/crates/mq-markdown/src/parser/block/footnote.rs @@ -0,0 +1,96 @@ +//! Footnote definitions. + +use super::{Block, Collector, FootnoteBlock, Indent, Interrupt, Line, Position, inline, parse_blocks}; + +/// The indentation of the lines that continue a footnote definition. +pub(super) const FOOTNOTE_INDENT: usize = 4; + +/// The start of a footnote definition: `[^label]:`. +pub(super) struct FootnoteMarker<'a> { + label: &'a str, + /// Bytes from the start of the line to the content. + pub(super) content: usize, +} + +pub(super) fn footnote_marker<'a>(line: &Line<'a>) -> Option<FootnoteMarker<'a>> { + if !line.flavor.has_gfm() { + return None; + } + let Indent { columns, bytes: indent } = line.indent(); + let rest = line.text[indent..].strip_prefix("[^")?; + if columns >= line.code_indent() { + return None; + } + // A backslash escapes the bracket that follows it, as well as another backslash. + let bytes = rest.as_bytes(); + let mut close = 0; + while close < bytes.len() && bytes[close] != b']' { + if bytes[close] == b'[' { + return None; + } + close += if bytes[close] == b'\\' && matches!(bytes.get(close + 1), Some(b'[' | b'\\' | b']')) { + 2 + } else { + 1 + }; + } + if close >= bytes.len() { + return None; + } + let label = &rest[..close]; + if label.is_empty() || label.bytes().any(|b| matches!(b, b' ' | b'\t' | b'\r' | b'\n')) { + return None; + } + let after = rest[close + 1..].strip_prefix(':')?; + let blanks = after.len() - after.trim_start_matches([' ', '\t']).len(); + Some(FootnoteMarker { + label, + content: indent + 2 + close + 2 + blanks, + }) +} + +/// Collects a footnote definition, which continues on lines indented by four columns. +pub(super) fn footnote( + lines: &[Line<'_>], + start: usize, + marker: &FootnoteMarker<'_>, + depth: usize, + interrupting: Interrupt, + blocks: &mut Vec<Block>, +) -> usize { + let mut collector = Collector::new(depth, interrupting); + collector.push(lines[start].skip(marker.content)); + let mut index = start + 1; + + while let Some(line) = lines.get(index) { + if line.is_blank() { + collector.blank(line.skip(line.text.len())); + } else if line.indent().columns >= FOOTNOTE_INDENT { + collector.push(line.skip_columns(FOOTNOTE_INDENT)); + } else if !collector.lazy(lines, index) { + break; + } + index += 1; + } + let Collector { + lines: inner, blanks, .. + } = collector; + + // Trailing blank lines belong to whatever follows, but still extend the position. + index -= blanks.len(); + let end = blanks + .last() + .or(inner.last()) + .map_or_else(|| lines[start].end(), Line::end); + let position = Position { + start: lines[start].point(0), + end, + }; + blocks.push(Block::Footnote(FootnoteBlock { + ident: inline::normalize(marker.label), + children: parse_blocks(&inner, depth + 1, interrupting), + position, + })); + + index +} diff --git a/crates/mq-markdown/src/parser/block/html.rs b/crates/mq-markdown/src/parser/block/html.rs new file mode 100644 index 000000000..27a810593 --- /dev/null +++ b/crates/mq-markdown/src/parser/block/html.rs @@ -0,0 +1,70 @@ +//! HTML blocks. + +use super::{Block, Html, HtmlKind, Line, Node, Point, Position, html_flow, join_lines}; + +/// Collects an HTML block of `kind` starting at `lines[start]`, which is indented less than four columns. +pub(super) fn html_block(lines: &[Line<'_>], start: usize, kind: HtmlKind, blocks: &mut Vec<Block>) -> usize { + let first = &lines[start]; + let mut last = start; + let mut unterminated = false; + + match kind { + HtmlKind::Basic | HtmlKind::Complete => { + // Ends before a blank line, and lines cannot continue a paragraph lazily. + for (index, line) in lines.iter().enumerate().skip(start + 1) { + if line.is_blank() || line.lazy { + break; + } + last = index; + } + } + _ => { + let indent = first.indent().bytes; + let from = indent + html_flow::first_line_offset(kind); + if !html_flow::ends_in(kind, &first.text[from..]) { + let mut ended = false; + for (index, line) in lines.iter().enumerate().skip(start + 1) { + if line.lazy { + break; + } + last = index; + if html_flow::ends_in(kind, line.text) { + ended = true; + break; + } + } + // A block that never ends runs to the end of the document, line ending included. + unterminated = !ended && lines[last].eof && !lines[last].eol.is_empty(); + } + } + } + + let parts = lines[start..=last] + .iter() + .map(|line| (line.text, line.eol)) + .collect::<Vec<_>>(); + let mut value = join_lines(&parts); + let end = if unterminated { + value.push_str(lines[last].eol); + Point { + line: lines[last].number + 1, + column: 1, + } + } else { + lines[last].end() + }; + blocks.push(Block::Node(Node::Html(Html { + value, + position: Some(Position { + start: first.point(0), + end, + }), + }))); + + last + 1 +} + +/// The kind of HTML block that starts at `rest`, in a flavor that has HTML. +pub(super) fn html_start(line: &Line<'_>, rest: &str) -> Option<HtmlKind> { + line.flavor.has_html().then(|| html_flow::start(rest)).flatten() +} diff --git a/crates/mq-markdown/src/parser/block/list.rs b/crates/mq-markdown/src/parser/block/list.rs new file mode 100644 index 000000000..e5dfd866a --- /dev/null +++ b/crates/mq-markdown/src/parser/block/list.rs @@ -0,0 +1,291 @@ +//! Lists and their items. + +use super::{ + Block, Collector, Indent, InlineBlock, InlineKind, Interrupt, Item, Line, LineStart, ListBlock, ListMarker, + Position, extend_last_items, interrupts_paragraph, is_thematic_break, parse_blocks, setext_depth, table, +}; + +#[derive(Clone)] +pub(super) struct ItemMarker { + ordered: bool, + kind: ListMarker, + start: u32, + /// Bytes from the start of the line to the end of the marker. + after_marker: usize, + /// Columns of whitespace between the marker and the content that belong to the marker. + gap: usize, + /// Columns that continuation lines must be indented by to belong to the item. + pub(super) width: usize, + /// Whether the first line has no content after the marker. + pub(super) empty: bool, +} + +impl ItemMarker { + pub(super) fn parse(line: &Line<'_>) -> Option<Self> { + let Indent { columns, bytes: indent } = line.indent(); + if columns >= line.code_indent() { + return None; + } + let rest = &line.text[indent..]; + let bytes = rest.as_bytes(); + if is_thematic_break(line, indent) { + return None; + } + + let (ordered, start, marker_len) = match *bytes.first()? { + b'-' | b'+' | b'*' => (false, 0, 1), + b'0'..=b'9' => { + let digits = bytes.iter().take_while(|b| b.is_ascii_digit()).count(); + if digits > 9 || !matches!(bytes.get(digits), Some(b'.' | b')')) { + return None; + } + (true, rest[..digits].parse().ok()?, digits + 1) + } + _ => return None, + }; + let kind = match bytes[marker_len - 1] { + b'-' => ListMarker::Dash, + b'+' => ListMarker::Plus, + b'*' => ListMarker::Star, + b'.' => ListMarker::Period, + _ => ListMarker::Paren, + }; + + let after = &rest[marker_len..]; + let Indent { + columns: spaces, + bytes: blanks, + } = line.skip(indent + marker_len).indent(); + if blanks == 0 && !after.is_empty() { + return None; + } + + let empty = blanks == after.len(); + // Five or more columns means the content is indented code, so only one belongs to the marker. + let gap = if empty || spaces > line.code_indent() { + 1 + } else { + spaces + }; + + Some(Self { + ordered, + kind, + start, + after_marker: indent + marker_len, + gap, + width: columns + marker_len + gap, + empty, + }) + } + + /// The line without the marker and the whitespace that belongs to it. + pub(super) fn content<'a>(&self, line: Line<'a>) -> Line<'a> { + if self.empty { + line.skip(line.text.len()) + } else { + line.skip(self.after_marker).skip_columns(self.gap) + } + } + + fn same_list(&self, other: &Self) -> bool { + self.ordered == other.ordered && self.kind == other.kind + } + + /// Whether this marker may interrupt a paragraph. + pub(super) fn interrupts_paragraph(&self) -> bool { + !self.empty && (!self.ordered || self.start == 1) + } +} + +pub(super) fn list( + lines: &[Line<'_>], + start: usize, + first: &ItemMarker, + depth: usize, + interrupting: Interrupt, + blocks: &mut Vec<Block>, +) -> usize { + let mut items = Vec::new(); + let mut spread = false; + let mut index = start; + let mut marker = first.clone(); + + loop { + let (item, end) = list_item( + lines, + index, + &marker, + depth, + if items.is_empty() { + interrupting + } else { + Interrupt::default() + }, + ); + items.push(item); + + // Blank lines between items make the list loose. + let next = (end..lines.len()).find(|&i| !lines[i].is_blank()).and_then(|i| { + ItemMarker::parse(&lines[i]) + .filter(|m| m.same_list(first)) + .map(|m| (i, m)) + }); + match next { + Some((next, next_marker)) => { + spread |= next > end; + index = next; + marker = next_marker; + } + None => { + index = end; + break; + } + } + } + + blocks.push(Block::List(ListBlock { + ordered: first.ordered, + start: first.ordered.then_some(first.start), + marker: first.kind, + spread, + items, + })); + + index +} + +/// Collects one item starting at `start`, returning it and the index of the first line after it. +pub(super) fn list_item( + lines: &[Line<'_>], + start: usize, + marker: &ItemMarker, + depth: usize, + interrupting: Interrupt, +) -> (Item, usize) { + let first = marker.content(lines[start]); + let mut collector = Collector::new(depth, interrupting); + collector.push(first); + let mut last_blank_has_whitespace = false; + let mut index = start + 1; + + while let Some(line) = lines.get(index) { + if line.is_blank() { + // An item that starts with a blank line cannot continue after another one. + if first.is_blank() && collector.lines.len() == 1 { + break; + } + last_blank_has_whitespace = !line.text.is_empty(); + collector.blank(line.skip_columns(marker.width)); + } else if line.lazy { + collector.push(*line); + } else if line.indent().columns >= marker.width { + collector.push(line.skip_columns(marker.width)); + } else if !collector.lazy(lines, index) { + break; + } + index += 1; + } + collector.keep_fence_blanks(); + let Collector { + lines: mut inner, + blanks, + .. + } = collector; + + let checkbox = inner + .iter() + .position(|line| !line.is_blank()) + .and_then(|index| task_checkbox(&inner[index..])); + + // Trailing blank lines belong to whatever follows the item, but still extend its position. + let reaches_end = index == lines.len(); + index -= blanks.len(); + + // An unclosed fence in an item includes its line terminator when the next line is blank or starts + // another container, but not when a plain paragraph or thematic break follows. + let next = lines.get(index); + let item_end = next.is_none_or(|line| { + line.is_blank() + || matches!( + LineStart::of(line, line.indent()), + Some(LineStart::Blockquote | LineStart::Item(_)) + ) + }); + if let Some(last) = inner.last_mut() { + last.item_end = item_end; + } + + let content_end = inner.last().map_or_else(|| lines[start].end(), Line::end); + let position = Position { + start: lines[start].point(0), + end: blanks.last().map_or_else(|| content_end.clone(), Line::end), + }; + let mut children = parse_blocks(&inner, depth + 1, interrupting); + // The checkbox is the start of the text of the first paragraph. + let checked = checkbox.filter(|_| remove_task_marker(&mut children)); + if let Some(blank) = blanks.last() { + extend_last_items( + &mut children, + &blank.end(), + last_blank_has_whitespace && reaches_end && blanks.len() == 1 && blank.eol.is_empty(), + ); + } + let item = Item { + checked, + children, + position, + content_end, + }; + + (item, index) +} + +/// Recognises a GFM task marker (`[ ] ` or `[x] `) at the start of an item's content, returning +/// the checked state and the bytes to skip. +fn task_checkbox(lines: &[Line<'_>]) -> Option<bool> { + let line = &lines[0]; + if !line.flavor.has_gfm() { + return None; + } + let bytes = line.text.as_bytes(); + let checked = match bytes.get(..3)? { + [b'[', b' ', b']'] => false, + [b'[', b'x' | b'X', b']'] => true, + _ => return None, + }; + // Whitespace follows the marker, which a line ending is when the paragraph goes on in the next line. + match bytes.get(3) { + Some(b' ' | b'\t') => Some(checked), + None if continues_paragraph(lines) => Some(checked), + _ => None, + } +} + +/// Whether the paragraph that starts at `lines[0]` goes on in `lines[1]`, as opposed to a blank line, +/// another block, a table or a setext underline. +fn continues_paragraph(lines: &[Line<'_>]) -> bool { + lines.get(1).is_some_and(|next| { + !interrupts_paragraph(next) && setext_depth(next).is_none() && !table::starts_at(lines, 0, interrupts_paragraph) + }) +} + +/// Removes the checkbox from the text of the first paragraph of an item, and the paragraph when nothing +/// else is in it. Returns whether the item starts with one. +fn remove_task_marker(children: &mut Vec<Block>) -> bool { + let Some(Block::Inline(InlineBlock { + source, + kind: InlineKind::Paragraph, + })) = children.first_mut() + else { + return false; + }; + if !matches!(source.text.as_bytes().get(..3), Some([b'[', b' ' | b'x' | b'X', b']'])) { + return false; + } + source.remove_prefix_and_one(3); + if source.text.is_empty() { + children.remove(0); + } + true +} From be51fa4d0898ec48e5801d41c6386ded055a905d Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Tue, 13 Oct 2026 23:34:22 +0900 Subject: [PATCH 75/88] =?UTF-8?q?=F0=9F=90=9B=20fix(markdown):=20separate?= =?UTF-8?q?=20the=20lines=20of=20pending=20MDX=20flow=20with=20the=20termi?= =?UTF-8?q?nator=20of=20the=20line=20before?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The terminator that follows a line belongs between it and the next line, not before the line it ends. --- crates/mq-markdown/src/parser/block/leaf.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/crates/mq-markdown/src/parser/block/leaf.rs b/crates/mq-markdown/src/parser/block/leaf.rs index 8999dcdbd..950026f67 100644 --- a/crates/mq-markdown/src/parser/block/leaf.rs +++ b/crates/mq-markdown/src/parser/block/leaf.rs @@ -28,8 +28,10 @@ enum Open { } /// MDX flow content that goes on in the next line. -struct Pending { +struct Pending<'a> { text: String, + /// Terminator of the last line in `text`, which separates it from the next line. + eol: &'a str, lines: usize, } @@ -48,7 +50,7 @@ pub(super) struct LeafState<'a> { /// The text of the open paragraph when it starts like a definition, which a setext underline does /// not turn into a heading when there is nothing else in it. definitions: Option<String>, - flow: Option<Pending>, + flow: Option<Pending<'a>>, /// Restrictions on the next fed line, the first line of a container that interrupted a paragraph. restricted: Interrupt, /// How many containers can still be open before the depth limit, which no longer open any. @@ -180,8 +182,9 @@ impl<'a> LeafState<'a> { } if let Some(pending) = &mut self.flow { - pending.text.push_str(line.eol); + pending.text.push_str(pending.eol); pending.text.push_str(line.text); + pending.eol = line.eol; pending.lines += 1; let end = if pending.lines < MAX_FLOW_LINES { flow_end(&pending.text) @@ -293,6 +296,7 @@ impl<'a> LeafState<'a> { if matches!(start, FlowStart::Pending) { self.flow = Some(Pending { text: line.text.to_string(), + eol: line.eol, lines: 1, }); } From 2ce99249bd0712a09c497030493444ada7d7eacc Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Tue, 13 Oct 2026 23:34:22 +0900 Subject: [PATCH 76/88] =?UTF-8?q?=F0=9F=90=9B=20fix(markdown):=20keep=20a?= =?UTF-8?q?=20blank=20line=20before=20a=20dash=20rule=20that=20follows=20i?= =?UTF-8?q?nline=20HTML=20or=20MDX=20text?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rendered without it, the rule reads back as a setext heading underline. --- crates/mq-markdown/src/markdown.rs | 11 +++++++++++ crates/mq-markdown/src/node.rs | 3 +++ 2 files changed, 14 insertions(+) diff --git a/crates/mq-markdown/src/markdown.rs b/crates/mq-markdown/src/markdown.rs index c427350d6..7b7bb0430 100644 --- a/crates/mq-markdown/src/markdown.rs +++ b/crates/mq-markdown/src/markdown.rs @@ -590,6 +590,15 @@ mod tests { #[case::dash_rule_under_text_stays_a_rule("Foo\n--- -\n", 2, "Foo\n\n---\n")] #[case::dash_rule_under_emphasis_stays_a_rule("*Foo*\n- - -\n", 2, "*Foo*\n\n---\n")] #[case::star_rule_under_text_needs_no_blank_line("Foo\n***\n", 2, "Foo\n***\n")] + #[cfg_attr( + feature = "wikilink", + case::dash_rule_under_wikilink_stays_a_rule("[[note]]\n- - -\n", 2, "[[note]]\n\n---\n") + )] + #[cfg_attr( + feature = "embed", + case::dash_rule_under_embed_stays_a_rule("![[note]]\n- - -\n", 2, "![[note]]\n\n---\n") + )] + #[case::dash_rule_under_inline_html_stays_a_rule("<b>x</b>\n- - -\n", 4, "<b>x</b>\n\n---\n")] // items whose marker sits alone on its own line must stay tight, not go loose #[case::list_tight_despite_marker_on_own_line("-\n foo\n-\n bar", 2, "- foo\n- bar\n")] // an empty task item must not take the paragraph after it @@ -642,6 +651,8 @@ mod tests { 1, "<Chart color=\"#fcb32c\" year={year} />\n" )] + #[case::dash_rule_under_text_expression("a {b}\n- - -\n", 3, "a {b}\n\n---\n")] + #[case::dash_rule_under_jsx_text("a <b />\n- - -\n", 3, "a <b />\n\n---\n")] fn test_markdown_from_mdx_str(#[case] input: &str, #[case] expected_nodes: usize, #[case] expected_output: &str) { let md = Markdown::from_mdx_str(input).unwrap(); assert_eq!(md.nodes.len(), expected_nodes); diff --git a/crates/mq-markdown/src/node.rs b/crates/mq-markdown/src/node.rs index 5b9961f0c..a2628a65b 100644 --- a/crates/mq-markdown/src/node.rs +++ b/crates/mq-markdown/src/node.rs @@ -1685,6 +1685,9 @@ impl Node { | Self::Image(_) | Self::ImageRef(_) | Self::FootnoteRef(_) + | Self::Html(_) + | Self::MdxJsxTextElement(_) + | Self::MdxTextExpression(_) ) } From 4e90471d0044d821d31db1daa555f1852334e18b Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Wed, 14 Oct 2026 00:22:03 +0900 Subject: [PATCH 77/88] =?UTF-8?q?=E2=9C=85=20test(markdown):=20add=20fuzz?= =?UTF-8?q?=20targets=20for=20reading=20and=20rendering=20Markdown?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `markdown` reads generated Markdown and MDX and renders it as Markdown, HTML, text and JSON. `markdown_scaling` fails when the time for a document repeated to four times its size grows by more than 12 times. --- Cargo.lock | 1 + fuzz/Cargo.toml | 15 ++ fuzz/README.md | 11 ++ fuzz/fuzz_targets/markdown.rs | 8 + fuzz/fuzz_targets/markdown_scaling.rs | 8 + fuzz/src/lib.rs | 207 +++++++++++++++++++++++++- 6 files changed, 249 insertions(+), 1 deletion(-) create mode 100644 fuzz/fuzz_targets/markdown.rs create mode 100644 fuzz/fuzz_targets/markdown_scaling.rs diff --git a/Cargo.lock b/Cargo.lock index d057b3438..533836d39 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2762,6 +2762,7 @@ dependencies = [ "itertools 0.15.0", "libfuzzer-sys", "mq-lang", + "mq-markdown", "sha2", ] diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index 1ea106ec3..2e8e73605 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -13,6 +13,7 @@ arbitrary = { workspace = true } itertools = { workspace = true } libfuzzer-sys = { workspace = true } mq-lang = { workspace = true, features = ["mqc"] } +mq-markdown = { workspace = true, features = ["json", "obsidian"] } sha2 = { workspace = true } [lib] @@ -31,3 +32,17 @@ doc = false name = "mqc" path = "fuzz_targets/mqc.rs" test = false + +[[bin]] +bench = false +doc = false +name = "markdown" +path = "fuzz_targets/markdown.rs" +test = false + +[[bin]] +bench = false +doc = false +name = "markdown_scaling" +path = "fuzz_targets/markdown_scaling.rs" +test = false diff --git a/fuzz/README.md b/fuzz/README.md index 0408d9eea..47112e98d 100644 --- a/fuzz/README.md +++ b/fuzz/README.md @@ -15,3 +15,14 @@ just test-fuzz mqc ``` The `mqc` target compiles a known program, overwrites bytes of the file, and recomputes its checksum so the input reaches the decoder and the bytecode verifier. A file that loads must then run without panicking, since the VM trusts verified bytecode. + +Run the Markdown parser targets: + +```sh +just test-fuzz markdown +just test-fuzz markdown_scaling +``` + +`markdown` reads generated Markdown and MDX and renders it as Markdown, HTML, text and JSON, so a panic, an abort or a stack overflow is reported. A document is put together from markup tokens, repeats of one token and free text, which reaches nesting and repeated constructs that random bytes seldom do. + +`markdown_scaling` repeats the document to 16 KiB and to four times that, and fails when the second takes more than 12 times as long as the first. Linear work takes four times as long and quadratic work sixteen times. A reported input is slow in itself, so check it with `cargo fuzz run markdown_scaling <artifact>`. diff --git a/fuzz/fuzz_targets/markdown.rs b/fuzz/fuzz_targets/markdown.rs new file mode 100644 index 000000000..394eb2ffa --- /dev/null +++ b/fuzz/fuzz_targets/markdown.rs @@ -0,0 +1,8 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; +use mq_fuzz::MarkdownInput; + +fuzz_target!(|input: MarkdownInput| { + mq_fuzz::read_markdown(&input); +}); diff --git a/fuzz/fuzz_targets/markdown_scaling.rs b/fuzz/fuzz_targets/markdown_scaling.rs new file mode 100644 index 000000000..b4aba9d7c --- /dev/null +++ b/fuzz/fuzz_targets/markdown_scaling.rs @@ -0,0 +1,8 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; +use mq_fuzz::MarkdownInput; + +fuzz_target!(|input: MarkdownInput| { + mq_fuzz::check_markdown_scaling(&input); +}); diff --git a/fuzz/src/lib.rs b/fuzz/src/lib.rs index e2ad227e8..6fe6869c5 100644 --- a/fuzz/src/lib.rs +++ b/fuzz/src/lib.rs @@ -1,5 +1,6 @@ //! Shared input generation for the fuzz targets: `interpreter` feeds generated scripts -//! through [`mq_lang::DefaultEngine`], and `mqc` loads corrupted `.mqc` files. +//! through [`mq_lang::DefaultEngine`], `mqc` loads corrupted `.mqc` files, and `markdown` and +//! `markdown_scaling` read and render generated Markdown and MDX. use arbitrary::Arbitrary; use itertools::Itertools; @@ -176,3 +177,207 @@ pub fn load_and_run_mqc(input: &MqcInput) { let _ = engine.eval_compiled(&program, input.into_iter()); } } + +/// Markup that the `markdown` targets join into documents, so that they reach the block and inline +/// rules that random bytes seldom do. +const MARKDOWN_TOKENS: &[&str] = &[ + "\n", + "\n\n", + "\r\n", + "\r", + " ", + " ", + " ", + "\t", + " \n", + "\\", + "\0", + "\u{feff}", + "# ", + "###### ", + "> ", + ">", + "- ", + "* ", + "+ ", + "1. ", + "1) ", + "- [ ] ", + "- [x] ", + "---", + "***", + "___", + "===", + "+++\n", + "---\n", + "```", + "~~~", + "``", + "`", + "*", + "**", + "_", + "__", + "~", + "~~", + "[", + "]", + "](", + ")", + "![", + "[^1]", + "[^1]: ", + "[a]: ", + "<", + ">", + "</", + "/>", + "<div>", + "</div>", + "<!--", + "-->", + "<?", + "<![CDATA[", + "<script>", + "<a href=\"", + "\"", + "'", + "<https://a.b>", + "https://a.b", + "www.a.b", + "a@b.c", + "|", + "| --- |", + "| :-: |", + "&amp;", + "&#35;", + "&#x0;", + "&bogus;", + "$", + "$$", + "{", + "}", + "{...a}", + "<Foo bar={", + "import a from \"b\"\n", + "export const a = 1\n", + "[[", + "]]", + "![[", + "|", + "> [!NOTE]\n", + "> [!", + "#tag", + "^id", + "%%", + "a", + "b", + "0", + "é", + "日本語", +]; + +/// The longest source a generated input makes, so that repeats do not slow each run down. +const MARKDOWN_MAX_LEN: usize = 1 << 20; + +#[derive(Debug, Clone, Arbitrary)] +enum MarkdownPart { + Token(u8), + Repeat(u8, u8), + Text(String), +} + +/// A Markdown or MDX document put together from markup tokens, repeats of one token and free text. +#[derive(Debug, Clone, Arbitrary)] +pub struct MarkdownInput { + mdx: bool, + frontmatter: bool, + parts: Vec<MarkdownPart>, +} + +impl MarkdownInput { + fn source(&self) -> String { + let token = |index: u8| MARKDOWN_TOKENS[usize::from(index) % MARKDOWN_TOKENS.len()]; + let mut source = String::new(); + for part in &self.parts { + match part { + MarkdownPart::Token(index) => source.push_str(token(*index)), + MarkdownPart::Repeat(index, count) => { + for _ in 0..*count { + source.push_str(token(*index)); + } + } + MarkdownPart::Text(text) => source.push_str(text), + } + if source.len() >= MARKDOWN_MAX_LEN { + break; + } + } + source + } +} + +/// Reads `source` and writes it as Markdown, HTML, text and JSON. A `Result::Err` from the reader is +/// fine (MDX reports unclosed expressions and elements); a panic, an abort or a stack overflow is not. +fn read_and_render(source: &str, mdx: bool, frontmatter: bool) { + use mq_markdown::{Markdown, ParseOptions}; + + let options = ParseOptions::default().with_frontmatter(frontmatter); + let _ = mq_markdown::to_html(source); + let parsed = if mdx { + Markdown::from_mdx_str_with(source, options) + } else { + Markdown::from_markdown_str_with(source, options) + }; + if let Ok(markdown) = parsed { + let _ = markdown.to_string(); + let _ = markdown.to_html(); + let _ = markdown.to_text(); + let _ = markdown.to_json(); + } +} + +/// Reads and renders the generated document. +pub fn read_markdown(input: &MarkdownInput) { + read_and_render(&input.source(), input.mdx, input.frontmatter); +} + +/// Fails when the time to read and render a document grows faster than its size. +/// +/// The document, cut at `BASE_LEN` bytes, is repeated up to that length and again to four times that. Work that is linear +/// takes four times as long, quadratic work sixteen times; the check allows `MAX_RATIO`. A run that +/// is over is measured once more, as a single run can hit a pause. +pub fn check_markdown_scaling(input: &MarkdownInput) { + use std::time::{Duration, Instant}; + + const BASE_LEN: usize = 16 * 1024; + const MAX_RATIO: u32 = 12; + const FLOOR: Duration = Duration::from_millis(1); + + let source = input.source(); + let end = (0..=BASE_LEN.min(source.len())) + .rev() + .find(|&end| source.is_char_boundary(end)) + .unwrap_or(0); + let source = &source[..end]; + if source.is_empty() { + return; + } + let base = source.repeat(BASE_LEN.div_ceil(source.len())); + let large = base.repeat(4); + let time = |text: &str| { + let start = Instant::now(); + read_and_render(text, input.mdx, input.frontmatter); + start.elapsed() + }; + + let base_time = time(&base).min(time(&base)).max(FLOOR); + let allowed = base_time * MAX_RATIO; + if time(&large) > allowed && time(&large) > allowed { + panic!( + "time grows faster than the size: {base_time:?} for {} bytes, more than {allowed:?} for {} bytes", + base.len(), + large.len() + ); + } +} From ee378dbce8fd4172cd80efb56535511022038bcd Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Wed, 14 Oct 2026 09:57:20 +0900 Subject: [PATCH 78/88] =?UTF-8?q?=E2=9A=A1=20perf(fuzz):=20cap=20the=20gen?= =?UTF-8?q?erated=20Markdown=20source=20at=2064=20KiB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit to_json writes about 100 times the source, so a 1 MiB source made a run build hundreds of megabytes of output and pushed the resident size of the fuzzer up to the libFuzzer limit. --- fuzz/src/lib.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/fuzz/src/lib.rs b/fuzz/src/lib.rs index 6fe6869c5..0f799a09d 100644 --- a/fuzz/src/lib.rs +++ b/fuzz/src/lib.rs @@ -277,8 +277,9 @@ const MARKDOWN_TOKENS: &[&str] = &[ "日本語", ]; -/// The longest source a generated input makes, so that repeats do not slow each run down. -const MARKDOWN_MAX_LEN: usize = 1 << 20; +/// The longest source a generated input makes. `to_json` writes about 100 times the source, so a larger +/// source makes a run slow and its memory use high. +const MARKDOWN_MAX_LEN: usize = 64 * 1024; #[derive(Debug, Clone, Arbitrary)] enum MarkdownPart { From d500a7033bc801cb4d2c4e08351eeaa7061b5ab5 Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Wed, 14 Oct 2026 09:57:20 +0900 Subject: [PATCH 79/88] =?UTF-8?q?=E2=9C=A8=20feat(markdown):=20compare=20H?= =?UTF-8?q?eadingDepth=20with=20u8=20and=20convert=20it=20to=20usize=20and?= =?UTF-8?q?=20i64?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Code that used the depth of a heading as a u8 keeps working for comparisons, and `usize::from(depth)` replaces `depth as usize`. --- crates/mq-markdown/src/node.rs | 36 ++++++++++++++++++++++++++++ crates/mq-markdown/src/node/tests.rs | 10 ++++++++ 2 files changed, 46 insertions(+) diff --git a/crates/mq-markdown/src/node.rs b/crates/mq-markdown/src/node.rs index a2628a65b..0c6efdfd1 100644 --- a/crates/mq-markdown/src/node.rs +++ b/crates/mq-markdown/src/node.rs @@ -683,6 +683,42 @@ impl From<HeadingDepth> for u8 { } } +impl PartialEq<u8> for HeadingDepth { + fn eq(&self, other: &u8) -> bool { + self.0 == *other + } +} + +impl PartialEq<HeadingDepth> for u8 { + fn eq(&self, other: &HeadingDepth) -> bool { + *self == other.0 + } +} + +impl PartialOrd<u8> for HeadingDepth { + fn partial_cmp(&self, other: &u8) -> Option<std::cmp::Ordering> { + self.0.partial_cmp(other) + } +} + +impl PartialOrd<HeadingDepth> for u8 { + fn partial_cmp(&self, other: &HeadingDepth) -> Option<std::cmp::Ordering> { + self.partial_cmp(&other.0) + } +} + +impl From<HeadingDepth> for usize { + fn from(depth: HeadingDepth) -> Self { + usize::from(depth.0) + } +} + +impl From<HeadingDepth> for i64 { + fn from(depth: HeadingDepth) -> Self { + i64::from(depth.0) + } +} + impl Display for HeadingDepth { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { self.0.fmt(f) diff --git a/crates/mq-markdown/src/node/tests.rs b/crates/mq-markdown/src/node/tests.rs index a30ed62bf..f248337a2 100644 --- a/crates/mq-markdown/src/node/tests.rs +++ b/crates/mq-markdown/src/node/tests.rs @@ -3093,3 +3093,13 @@ fn heading_depth_is_a_number_in_json() { assert_eq!(serde_json::from_str::<Heading>(&json).unwrap(), heading); assert!(serde_json::from_str::<Heading>(&json.replace("\"depth\":3", "\"depth\":7")).is_err()); } + +#[test] +fn heading_depth_compares_with_u8() { + assert!(HeadingDepth::H2 == 2u8); + assert!(2u8 == HeadingDepth::H2); + assert!(HeadingDepth::H2 < 3u8); + assert!(3u8 > HeadingDepth::H2); + assert_eq!(usize::from(HeadingDepth::H3), 3); + assert_eq!(i64::from(HeadingDepth::H3), 3); +} From acb5f7ea5979f917aabe8401801ea89164c6de89 Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Wed, 14 Oct 2026 12:25:00 +0900 Subject: [PATCH 80/88] =?UTF-8?q?=E2=9A=A1=20perf(markdown):=20write=20adj?= =?UTF-8?q?acent=20emphasis=20in=20linear=20time?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The delimiter of emphasis or strong after other emphasis is picked from how the node before it was written, which rendered every node before it again. A run of 16 thousand `*_` took 3 seconds to write as Markdown and as HTML. The caller now passes what it already wrote for the node before. --- crates/mq-markdown/src/markdown.rs | 15 +++++- crates/mq-markdown/src/node/render.rs | 46 ++++++++++++++----- .../mq-markdown/tests/parser_limits_tests.rs | 36 +++++++++++++++ 3 files changed, 85 insertions(+), 12 deletions(-) diff --git a/crates/mq-markdown/src/markdown.rs b/crates/mq-markdown/src/markdown.rs index 7b7bb0430..5f66a0177 100644 --- a/crates/mq-markdown/src/markdown.rs +++ b/crates/mq-markdown/src/markdown.rs @@ -103,6 +103,8 @@ impl Markdown { // Sibling list items only carry a `level`, not a parent link; reconstructs // nested-list indentation from the nearest open ancestor at each level. let mut list_indent_stack: Vec<usize> = Vec::new(); + // What `render_before` returned for the node before, to pick a delimiter that does not fuse with it. + let mut previous_rendered: Option<String> = None; let reordered = list_order::reorder(&self.nodes); let nodes: &[Node] = reordered.as_ref().map_or(&self.nodes, |reordered| &reordered.nodes); @@ -202,6 +204,8 @@ impl Markdown { current_table = None; in_table = false; + let previous = previous_rendered.take(); + let mut plain = false; let value = if let Node::List(list) = node && is_continuation(i) { @@ -231,8 +235,17 @@ impl Markdown { .render_with_theme(&self.options, theme) } else { list_indent_stack.clear(); - render_before(node, &nodes[..i], nodes.get(i + 1), &self.options, theme) + plain = true; + render_before( + node, + &nodes[..i], + previous.as_deref(), + nodes.get(i + 1), + &self.options, + theme, + ) }; + previous_rendered = plain.then(|| value.clone()); if value.is_empty() || value == "\n" { pre_position = None; diff --git a/crates/mq-markdown/src/node/render.rs b/crates/mq-markdown/src/node/render.rs index 01f077a0b..2cb23088e 100644 --- a/crates/mq-markdown/src/node/render.rs +++ b/crates/mq-markdown/src/node/render.rs @@ -649,22 +649,24 @@ fn starts_with_word_char(node: &Node) -> bool { } /// Renders `value`, escaping a `!` that ends its text when `next` starts a link, which would make an image. +/// +/// `previous` is what this function returned for the last of `before`, with `value` as `next`. pub(crate) fn render_before( value: &Node, before: &[Node], + previous: Option<&str>, next: Option<&Node>, options: &RenderOptions, theme: &ColorTheme<'_>, ) -> String { // Emphasis right after other emphasis takes the other delimiter, or the two fuse into one run. if let Node::Emphasis(Emphasis { values, .. }) = value - && let Some((prev, earlier)) = before.split_last() - && matches!(prev, Node::Emphasis(_) | Node::Strong(_)) + && matches!(before.last(), Some(Node::Emphasis(_) | Node::Strong(_))) && !matches!(values.first(), Some(Node::Emphasis(_))) && !matches!(values.last(), Some(Node::Emphasis(_))) && !next.is_some_and(starts_with_word_char) { - let delim = if render_before(prev, earlier, Some(value), options, theme).ends_with('_') { + let delim = if previous.is_some_and(|previous| previous.ends_with('_')) { "*" } else { "_" @@ -673,10 +675,9 @@ pub(crate) fn render_before( } // Strong right after emphasis that ends in `*` takes `__`, or the runs of `*` fuse. if let Node::Strong(Strong { values, .. }) = value - && let Some((prev, earlier)) = before.split_last() - && matches!(prev, Node::Emphasis(_) | Node::Strong(_)) + && matches!(before.last(), Some(Node::Emphasis(_) | Node::Strong(_))) && !next.is_some_and(starts_with_word_char) - && render_before(prev, earlier, Some(value), options, theme).ends_with('*') + && previous.is_some_and(|previous| previous.ends_with('*')) { return render_strong(values, "__", options, theme); } @@ -700,11 +701,20 @@ pub(crate) fn render_before( pub(crate) fn render_values(values: &[Node], options: &RenderOptions, theme: &ColorTheme<'_>) -> String { let mut pre_position: Option<Position> = None; + let mut previous: Option<String> = None; values .iter() .enumerate() .map(|(index, value)| { - let rendered = || render_before(value, &values[..index], values.get(index + 1), options, theme); + let raw = render_before( + value, + &values[..index], + previous.as_deref(), + values.get(index + 1), + options, + theme, + ); + let rendered = || raw.as_str(); if let Some(pos) = value.position() { let new_line_count = pre_position .as_ref() @@ -724,7 +734,7 @@ pub(crate) fn render_values(values: &[Node], options: &RenderOptions, theme: &Co pre_position = Some(pos); - if space.is_empty() { + let piece = if space.is_empty() { format!("{}{}", "\n".repeat(new_line_count), rendered()) } else { format!( @@ -732,10 +742,14 @@ pub(crate) fn render_values(values: &[Node], options: &RenderOptions, theme: &Co "\n".repeat(new_line_count), rendered().lines().map(|line| format!("{}{}", space, line)).join("\n") ) - } + }; + previous = Some(raw); + piece } else { pre_position = None; - rendered() + let piece = raw.clone(); + previous = Some(raw); + piece } }) .collect::<String>() @@ -752,6 +766,8 @@ pub(crate) fn render_values_block(values: &[Node], options: &RenderOptions, them let mut index = 0; // The last node written, to tell paragraphs from the inline nodes of one when there are no positions. let mut previous: Option<&Node> = None; + // What `render_before` returned for `previous`, or `None` when it is part of a table. + let mut previous_rendered: Option<String> = None; while index < values.len() { let first = &values[index]; @@ -760,7 +776,14 @@ pub(crate) fn render_values_block(values: &[Node], options: &RenderOptions, them render_table_run(values, index, options, theme) } else { ( - render_before(first, &values[..index], values.get(index + 1), options, theme), + render_before( + first, + &values[..index], + previous_rendered.as_deref(), + values.get(index + 1), + options, + theme, + ), first.position(), index + 1, ) @@ -783,6 +806,7 @@ pub(crate) fn render_values_block(values: &[Node], options: &RenderOptions, them } previous = Some(&values[index - 1]); result.push_str(&rendered); + previous_rendered = (!is_table_part(first)).then_some(rendered); } result } diff --git a/crates/mq-markdown/tests/parser_limits_tests.rs b/crates/mq-markdown/tests/parser_limits_tests.rs index e24376667..f488b02dd 100644 --- a/crates/mq-markdown/tests/parser_limits_tests.rs +++ b/crates/mq-markdown/tests/parser_limits_tests.rs @@ -134,3 +134,39 @@ fn emphasis_matches_as_before(#[case] input: &str, #[case] expected: &str) { fn thematic_breaks_are_told_from_other_lines(#[case] input: &str, #[case] expected: &str) { assert_eq!(mq_markdown::to_html(input), expected); } + +/// A run of emphasis nodes that follow each other is written with alternating delimiters, which must +/// not rewrite every node before it: that took seconds for a few thousand nodes. +#[test] +fn adjacent_emphasis_renders_in_linear_time() { + let markdown = Markdown::from_markdown_str(&"*_".repeat(20_000)).unwrap(); + let start = std::time::Instant::now(); + let _ = markdown.to_string(); + let _ = markdown.to_html(); + assert!( + start.elapsed() < std::time::Duration::from_secs(5), + "{:?}", + start.elapsed() + ); +} + +#[test] +fn adjacent_emphasis_alternates_its_delimiters() { + use mq_markdown::{Emphasis, Heading, HeadingDepth, Node}; + + let emphasis = |text: &str| { + Node::Emphasis(Emphasis { + values: vec![text.to_string().into()], + position: None, + }) + }; + let nodes = vec![emphasis("a"), emphasis("b"), emphasis("c")]; + assert_eq!(Markdown::new(nodes.clone()).to_string().trim_end(), "*a*\n_b_\n*c*"); + + let heading = Node::Heading(Heading { + depth: HeadingDepth::H1, + values: nodes, + position: None, + }); + assert_eq!(Markdown::new(vec![heading]).to_string().trim_end(), "# *a*_b_*c*"); +} From 138ab5aff00d1408ed3ecf2d48844425d6c76995 Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Wed, 14 Oct 2026 12:25:00 +0900 Subject: [PATCH 81/88] =?UTF-8?q?=E2=9A=A1=20perf(fuzz):=20write=20only=20?= =?UTF-8?q?small=20sources=20as=20JSON=20in=20the=20markdown=20targets?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The JSON is pretty printed, so nested nodes make it a few thousand times the source: 165 MB for 64 KiB of `[^1]: `. --- fuzz/src/lib.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/fuzz/src/lib.rs b/fuzz/src/lib.rs index 0f799a09d..cb685fb94 100644 --- a/fuzz/src/lib.rs +++ b/fuzz/src/lib.rs @@ -277,10 +277,13 @@ const MARKDOWN_TOKENS: &[&str] = &[ "日本語", ]; -/// The longest source a generated input makes. `to_json` writes about 100 times the source, so a larger -/// source makes a run slow and its memory use high. +/// The longest source a generated input makes, so that repeats do not slow each run down. const MARKDOWN_MAX_LEN: usize = 64 * 1024; +/// The longest source that is written as JSON. The output is pretty printed, so nested nodes make it a +/// few thousand times the source (165 MB for 64 KiB of `[^1]: `), which fills the memory of a run. +const JSON_MAX_LEN: usize = 8 * 1024; + #[derive(Debug, Clone, Arbitrary)] enum MarkdownPart { Token(u8), @@ -334,7 +337,9 @@ fn read_and_render(source: &str, mdx: bool, frontmatter: bool) { let _ = markdown.to_string(); let _ = markdown.to_html(); let _ = markdown.to_text(); - let _ = markdown.to_json(); + if source.len() <= JSON_MAX_LEN { + let _ = markdown.to_json(); + } } } From cae8f663bc9d52de639d91851c1d97cf940b115c Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Wed, 14 Oct 2026 21:17:50 +0900 Subject: [PATCH 82/88] =?UTF-8?q?=F0=9F=90=9B=20fix(markdown):=20keep=20an?= =?UTF-8?q?=20indented=20code=20block=20with=20trailing=20blank=20lines=20?= =?UTF-8?q?unfenced?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The parser knows an indented block is not fenced, so set fence to false instead of inferring it from the line count. A trailing whitespace-only line no longer turns the block into a fenced one when written back. --- crates/mq-markdown/src/parser/code.rs | 4 +--- crates/mq-markdown/tests/parser_tree_tests.rs | 5 +++++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/crates/mq-markdown/src/parser/code.rs b/crates/mq-markdown/src/parser/code.rs index b21fc0fb7..016e978b5 100644 --- a/crates/mq-markdown/src/parser/code.rs +++ b/crates/mq-markdown/src/parser/code.rs @@ -170,14 +170,12 @@ pub(super) fn indented_code(lines: &[Line<'_>], start: usize, blocks: &mut Vec<B start: lines[start].point(0), end: lines[last_code].end(), }; - // Mirrors the `mdast` conversion: a value whose line count differs from the span is reported as fenced. - let fence = value.lines().count() != position.end.line - position.start.line + 1; blocks.push(Block::Node(Node::Code(Code { value, lang: None, meta: None, - fence, + fence: false, position: Some(position), }))); diff --git a/crates/mq-markdown/tests/parser_tree_tests.rs b/crates/mq-markdown/tests/parser_tree_tests.rs index 9ceec0d82..00566f662 100644 --- a/crates/mq-markdown/tests/parser_tree_tests.rs +++ b/crates/mq-markdown/tests/parser_tree_tests.rs @@ -2553,6 +2553,11 @@ list level=0 index=0 marker=- @1:3-3:3 "# )] #[case::indented_code_trailing_blank(" code\n\n\n", "<pre><code>code\n</code></pre>\n", r#"code "code" @1:1-1:9"#)] +#[case::indented_code_trailing_spaces( + " code\n \n", + "<pre><code>code\n</code></pre>\n", + r#"code "code" @1:1-2:9"# +)] #[case::bom_heading( "\u{feff}# a\n", "<h1>a</h1>\n", From ff521d5aa1d5890a765beea316da7bc0c7df6c5c Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Wed, 14 Oct 2026 21:27:23 +0900 Subject: [PATCH 83/88] =?UTF-8?q?=F0=9F=90=9B=20fix(markdown):=20keep=20th?= =?UTF-8?q?e=20delimiter=20of=20emphasis=20that=20starts=20a=20new=20parag?= =?UTF-8?q?raph?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Emphasis or strong after other emphasis switched delimiters even when a blank line separated them, so a second paragraph of `*b*` was written as `_b_`. Switch only when the two nodes touch. --- crates/mq-markdown/src/markdown.rs | 2 ++ crates/mq-markdown/src/node/render.rs | 11 +++++++++++ 2 files changed, 13 insertions(+) diff --git a/crates/mq-markdown/src/markdown.rs b/crates/mq-markdown/src/markdown.rs index 5f66a0177..3c021ca2d 100644 --- a/crates/mq-markdown/src/markdown.rs +++ b/crates/mq-markdown/src/markdown.rs @@ -624,6 +624,8 @@ mod tests { #[case::emphasis_in_emphasis_at_an_edge("***a*b*", 2, "\\*_*a*b_\n")] #[case::emphasis_after_emphasis_after_a_nested_one("\\**_*a***b**)", 4, "\\*_\\_*a*_*b*\\*)\n")] #[case::strong_after_emphasis("*a*__b__", 2, "*a*__b__\n")] + #[case::emphasis_in_the_next_paragraph("*a*\n\n*b*", 2, "*a*\n\n*b*\n")] + #[case::strong_in_the_next_paragraph("*a*\n\n**b**", 2, "*a*\n\n**b**\n")] #[case::emphasis_nested_three_deep("***a***", 1, "***a***\n")] #[case::reference_then_colon_is_not_a_definition("[x] a\n\n[x]: /u", 3, "[x] a\n\n[x]: /u\n")] // an indented code block right after a list must not read back as list continuation diff --git a/crates/mq-markdown/src/node/render.rs b/crates/mq-markdown/src/node/render.rs index 2cb23088e..73ef9a831 100644 --- a/crates/mq-markdown/src/node/render.rs +++ b/crates/mq-markdown/src/node/render.rs @@ -648,6 +648,15 @@ fn starts_with_word_char(node: &Node) -> bool { matches!(node, Node::Text(Text { value, .. }) if value.starts_with(|c: char| c.is_alphanumeric())) } +/// Whether `value` starts where the last of `before` ends, so the two share a run of delimiters. +/// Nodes without positions are taken to touch. +fn touches_previous(before: &[Node], value: &Node) -> bool { + match (before.last().and_then(Node::position), value.position()) { + (Some(previous), Some(position)) => previous.end == position.start, + _ => true, + } +} + /// Renders `value`, escaping a `!` that ends its text when `next` starts a link, which would make an image. /// /// `previous` is what this function returned for the last of `before`, with `value` as `next`. @@ -662,6 +671,7 @@ pub(crate) fn render_before( // Emphasis right after other emphasis takes the other delimiter, or the two fuse into one run. if let Node::Emphasis(Emphasis { values, .. }) = value && matches!(before.last(), Some(Node::Emphasis(_) | Node::Strong(_))) + && touches_previous(before, value) && !matches!(values.first(), Some(Node::Emphasis(_))) && !matches!(values.last(), Some(Node::Emphasis(_))) && !next.is_some_and(starts_with_word_char) @@ -676,6 +686,7 @@ pub(crate) fn render_before( // Strong right after emphasis that ends in `*` takes `__`, or the runs of `*` fuse. if let Node::Strong(Strong { values, .. }) = value && matches!(before.last(), Some(Node::Emphasis(_) | Node::Strong(_))) + && touches_previous(before, value) && !next.is_some_and(starts_with_word_char) && previous.is_some_and(|previous| previous.ends_with('*')) { From 0bb49d720a521cd76000def11bbabf2635e43c60 Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Wed, 14 Oct 2026 21:27:23 +0900 Subject: [PATCH 84/88] =?UTF-8?q?=F0=9F=90=9B=20fix(markdown):=20read=20an?= =?UTF-8?q?=20escaped=20pipe=20in=20a=20table=20cell=20as=20a=20pipe=20ins?= =?UTF-8?q?ide=20wikilinks=20and=20embeds?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `[[x\|y]]` in a cell had the target `x\` and an href ending in `%5C`. Unescape `\|` as code spans already do. --- crates/mq-markdown/src/parser/inline.rs | 8 ++++++++ crates/mq-markdown/tests/obsidian_tests.rs | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/crates/mq-markdown/src/parser/inline.rs b/crates/mq-markdown/src/parser/inline.rs index a9e5014df..217e0c7c6 100644 --- a/crates/mq-markdown/src/parser/inline.rs +++ b/crates/mq-markdown/src/parser/inline.rs @@ -448,6 +448,14 @@ impl Scanner<'_> { return None; } let content = &src[open..close]; + // In a table cell `\|` stands for the pipe, as in a code span. + let unescaped; + let content = if self.context.source.table && content.contains("\\|") { + unescaped = content.replace("\\|", "|"); + unescaped.as_str() + } else { + content + }; let (target, label) = match content.split_once('|') { Some((target, label)) => (target.trim(), Some(label.trim().to_string())), None => (content.trim(), None), diff --git a/crates/mq-markdown/tests/obsidian_tests.rs b/crates/mq-markdown/tests/obsidian_tests.rs index 6d6abf633..a9951e288 100644 --- a/crates/mq-markdown/tests/obsidian_tests.rs +++ b/crates/mq-markdown/tests/obsidian_tests.rs @@ -49,6 +49,14 @@ fn parse(input: &str) -> String { .join(" ") } +#[rstest] +#[case::wikilink_in_a_cell("| a |\n| - |\n| [[x\\|y]] |", "wikilink(x|y)")] +#[case::embed_in_a_cell("| a |\n| - |\n| ![[x\\|400]] |", "embed(x|400)")] +#[case::outside_a_table("[[x\\|y]]", "wikilink(x\\|y)")] +fn escaped_pipe_in_a_table_cell(#[case] input: &str, #[case] expected: &str) { + assert!(parse(input).contains(expected), "{}", parse(input)); +} + #[rstest] #[case::plain("[[Page]]", "wikilink(Page|)")] #[case::alias("[[Page|Alias]]", "wikilink(Page|Alias)")] From abd80a1dfd819c5b630ca6e03dd9abc80d59e89b Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Wed, 14 Oct 2026 22:01:37 +0900 Subject: [PATCH 85/88] =?UTF-8?q?=F0=9F=90=9B=20fix(markdown):=20escape=20?= =?UTF-8?q?for=20a=20neighbor=20only=20when=20it=20touches=20the=20node?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A `!` at the end of a paragraph was escaped when the next paragraph began with a link, and a `:` that began a paragraph was escaped after a paragraph that ended in a reference. Apply the touching check that emphasis uses to these rules and to the word-character check too. --- crates/mq-markdown/src/markdown.rs | 2 ++ crates/mq-markdown/src/node/render.rs | 32 ++++++++++++++------------- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/crates/mq-markdown/src/markdown.rs b/crates/mq-markdown/src/markdown.rs index 3c021ca2d..3def3ef83 100644 --- a/crates/mq-markdown/src/markdown.rs +++ b/crates/mq-markdown/src/markdown.rs @@ -628,6 +628,8 @@ mod tests { #[case::strong_in_the_next_paragraph("*a*\n\n**b**", 2, "*a*\n\n**b**\n")] #[case::emphasis_nested_three_deep("***a***", 1, "***a***\n")] #[case::reference_then_colon_is_not_a_definition("[x] a\n\n[x]: /u", 3, "[x] a\n\n[x]: /u\n")] + #[case::bang_before_a_link_in_the_next_paragraph("a!\n\n[b](u)", 2, "a!\n\n[b](u)\n")] + #[case::colon_after_a_reference_in_the_next_paragraph("[x]\n\n:a\n\n[x]: /u", 3, "[x]\n\n:a\n\n[x]: /u\n")] // an indented code block right after a list must not read back as list continuation #[case::code_after_list_uses_fence(" - one\n\n two", 2, "- one\n\n```\n two\n```\n")] // a block quote nested in a list item needs its continuation line's own indent diff --git a/crates/mq-markdown/src/node/render.rs b/crates/mq-markdown/src/node/render.rs index 73ef9a831..cc463f228 100644 --- a/crates/mq-markdown/src/node/render.rs +++ b/crates/mq-markdown/src/node/render.rs @@ -648,11 +648,11 @@ fn starts_with_word_char(node: &Node) -> bool { matches!(node, Node::Text(Text { value, .. }) if value.starts_with(|c: char| c.is_alphanumeric())) } -/// Whether `value` starts where the last of `before` ends, so the two share a run of delimiters. +/// Whether `second` starts where `first` ends, so the two are written next to each other. /// Nodes without positions are taken to touch. -fn touches_previous(before: &[Node], value: &Node) -> bool { - match (before.last().and_then(Node::position), value.position()) { - (Some(previous), Some(position)) => previous.end == position.start, +fn touches(first: &Node, second: &Node) -> bool { + match (first.position(), second.position()) { + (Some(first), Some(second)) => first.end == second.start, _ => true, } } @@ -671,10 +671,10 @@ pub(crate) fn render_before( // Emphasis right after other emphasis takes the other delimiter, or the two fuse into one run. if let Node::Emphasis(Emphasis { values, .. }) = value && matches!(before.last(), Some(Node::Emphasis(_) | Node::Strong(_))) - && touches_previous(before, value) + && before.last().is_some_and(|last| touches(last, value)) && !matches!(values.first(), Some(Node::Emphasis(_))) && !matches!(values.last(), Some(Node::Emphasis(_))) - && !next.is_some_and(starts_with_word_char) + && !next.is_some_and(|next| touches(value, next) && starts_with_word_char(next)) { let delim = if previous.is_some_and(|previous| previous.ends_with('_')) { "*" @@ -686,21 +686,23 @@ pub(crate) fn render_before( // Strong right after emphasis that ends in `*` takes `__`, or the runs of `*` fuse. if let Node::Strong(Strong { values, .. }) = value && matches!(before.last(), Some(Node::Emphasis(_) | Node::Strong(_))) - && touches_previous(before, value) - && !next.is_some_and(starts_with_word_char) + && before.last().is_some_and(|last| touches(last, value)) + && !next.is_some_and(|next| touches(value, next) && starts_with_word_char(next)) && previous.is_some_and(|previous| previous.ends_with('*')) { return render_strong(values, "__", options, theme); } // A reference followed by a colon at the start of a line would read back as a definition. - let after_reference = matches!(before.last(), Some(Node::LinkRef(_) | Node::FootnoteRef(_))); + let after_reference = matches!(before.last(), Some(Node::LinkRef(_) | Node::FootnoteRef(_))) + && before.last().is_some_and(|last| touches(last, value)); let mut rendered = value.render_with_theme(options, theme); - let starts_link = match next { - Some(Node::Link(_) | Node::LinkRef(_)) => true, - #[cfg(feature = "wikilink")] - Some(Node::WikiLink(_)) => true, - _ => false, - }; + let starts_link = next.is_some_and(|next| touches(value, next)) + && match next { + Some(Node::Link(_) | Node::LinkRef(_)) => true, + #[cfg(feature = "wikilink")] + Some(Node::WikiLink(_)) => true, + _ => false, + }; if after_reference && matches!(value, Node::Text(_)) && rendered.starts_with(':') { rendered.insert(0, '\\'); } From 01a15e00c6f17c92adf374055a34cdeadd6c7c84 Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Wed, 14 Oct 2026 22:09:31 +0900 Subject: [PATCH 86/88] =?UTF-8?q?=F0=9F=90=9B=20fix(markdown):=20write=20o?= =?UTF-8?q?ne=20blank=20line=20after=20a=20table?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The last row of a table ends with its own line ending, and the gap of two lines to the next block added two more, so a blank line in the source became two in the output. A block that followed a table without a gap had one blank line, so the output changed again when read back. --- crates/mq-markdown/src/markdown.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/crates/mq-markdown/src/markdown.rs b/crates/mq-markdown/src/markdown.rs index 3def3ef83..0b4807b37 100644 --- a/crates/mq-markdown/src/markdown.rs +++ b/crates/mq-markdown/src/markdown.rs @@ -300,6 +300,11 @@ impl Markdown { new_line_count = if cur_list.spread { 2 } else { 1 }; } + // The last row of a table ends with its own line ending, so one more makes the blank line. + if matches!(prev_node, Some(Node::TableCell(_) | Node::TableAlign(_))) { + new_line_count = 1; + } + pre_position = Some(pos.clone()); for _ in 0..new_line_count { @@ -527,6 +532,9 @@ mod tests { 10, "| A | B |\n| - | - |\n| 1 | 2 |\n\n| C | D |\n| - | - |\n| 3 | 4 |\n" )] + // the line ending of the last row is part of the blank line before what follows + #[case::table_then_paragraph("| A |\n|---|\n| 1 |\n\npara\n", 4, "| A |\n| - |\n| 1 |\n\npara\n")] + #[case::table_then_rule("| A |\n|---|\n| 1 |\n---\n", 4, "| A |\n| - |\n| 1 |\n\n---\n")] // adjacent tables of different widths must not share column layout. #[case::tables_adjacent_different_widths( "| A | B |\n|---|---|\n| 1 | 2 |\n\n| C |\n|---|\n| 3 |\n", From 3000d06b7f8c3f9852db06affa8f0ffa13cc4e4e Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Wed, 14 Oct 2026 22:56:49 +0900 Subject: [PATCH 87/88] =?UTF-8?q?=F0=9F=90=9B=20fix(markdown):=20write=20l?= =?UTF-8?q?ists=20in=20a=20block=20quote=20as=20at=20the=20top=20level?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Inside a quote or callout, an item that held a nested item before the rest of its content was written out of order, so `para` after a nested `- b` came before it. Items on one line took a padded marker, which made the third level an indented code block, and a bullet under an ordered item was indented by two columns and left the item. Share the ordering and the indent of the content with the top level. --- crates/mq-markdown/src/markdown.rs | 6 ++- crates/mq-markdown/src/markdown/list_order.rs | 8 +-- crates/mq-markdown/src/node/render.rs | 50 ++++++++++++++++++- 3 files changed, 58 insertions(+), 6 deletions(-) diff --git a/crates/mq-markdown/src/markdown.rs b/crates/mq-markdown/src/markdown.rs index 0b4807b37..e62149e86 100644 --- a/crates/mq-markdown/src/markdown.rs +++ b/crates/mq-markdown/src/markdown.rs @@ -13,7 +13,7 @@ use miette::miette; use std::{fmt, str::FromStr}; use table_layout::{TableLayout, write_padded_cell}; -mod list_order; +pub(crate) mod list_order; mod table_layout; /// What reading Markdown or MDX turns on. @@ -638,6 +638,10 @@ mod tests { #[case::reference_then_colon_is_not_a_definition("[x] a\n\n[x]: /u", 3, "[x] a\n\n[x]: /u\n")] #[case::bang_before_a_link_in_the_next_paragraph("a!\n\n[b](u)", 2, "a!\n\n[b](u)\n")] #[case::colon_after_a_reference_in_the_next_paragraph("[x]\n\n:a\n\n[x]: /u", 3, "[x]\n\n:a\n\n[x]: /u\n")] + // lists in a block quote: items on one line, the rest of an item after nested ones, nested under an ordered one + #[case::quote_list_in_list_on_one_line("> - - - x", 1, "> - - - x\n")] + #[case::quote_paragraph_after_nested_item("> - a\n> - b\n>\n> para", 1, "> - a\n> - b\n> \n> para\n")] + #[case::quote_bullet_nested_under_ordered("> 1. a\n> - b", 1, "> 1. a\n> - b\n")] // an indented code block right after a list must not read back as list continuation #[case::code_after_list_uses_fence(" - one\n\n two", 2, "- one\n\n```\n two\n```\n")] // a block quote nested in a list item needs its continuation line's own indent diff --git a/crates/mq-markdown/src/markdown/list_order.rs b/crates/mq-markdown/src/markdown/list_order.rs index 189608224..5a74a4751 100644 --- a/crates/mq-markdown/src/markdown/list_order.rs +++ b/crates/mq-markdown/src/markdown/list_order.rs @@ -7,11 +7,11 @@ use crate::node::{List, Node, Position}; /// Nodes in source order. -pub(super) struct Reordered { - pub(super) nodes: Vec<Node>, +pub(crate) struct Reordered { + pub(crate) nodes: Vec<Node>, /// For each node, whether it holds the rest of an item that follows the items nested in it. It is /// rendered as its content, indented to the content of the item, without a marker. - pub(super) continuation: Vec<bool>, + pub(crate) continuation: Vec<bool>, } impl Reordered { @@ -27,7 +27,7 @@ impl Reordered { } /// Returns the nodes in source order, or `None` when they already are. -pub(super) fn reorder(nodes: &[Node]) -> Option<Reordered> { +pub(crate) fn reorder(nodes: &[Node]) -> Option<Reordered> { // Some child of an item starts after the item nested right after it. let needed = nodes.windows(2).any(|pair| match (&pair[0], &pair[1]) { (Node::List(item), Node::List(next)) if next.level > item.level => { diff --git a/crates/mq-markdown/src/node/render.rs b/crates/mq-markdown/src/node/render.rs index cc463f228..d08eb842f 100644 --- a/crates/mq-markdown/src/node/render.rs +++ b/crates/mq-markdown/src/node/render.rs @@ -768,15 +768,50 @@ pub(crate) fn render_values(values: &[Node], options: &RenderOptions, theme: &Co .collect::<String>() } +/// Writes an item, or the rest of one that follows the items nested in it, inside a container. +/// +/// `indents` holds the column of the content of each open item. It is kept out of the container loop +/// so that the frame of that recursion stays small. +#[inline(never)] +fn render_list_item( + node: &Node, + item: &List, + rest: bool, + indents: &mut Vec<usize>, + options: &RenderOptions, + theme: &ColorTheme<'_>, +) -> String { + if rest { + // Indented to the content of the item and written without a marker. + indents.truncate(item.level as usize + 1); + let indent = indents.last().copied().unwrap_or(0); + return indent_lines(&render_values_block(&item.values, options, theme), indent); + } + // Nested items are indented to the content of the item that holds them. + indents.truncate(item.level as usize); + let own_indent = indents.last().copied().unwrap_or(0); + let delta = own_indent as isize - item.level as isize * 2; + indents.push(own_indent + list_own_prefix_width(item.ordered, item.index, item.start)); + reindent_all_lines(&node.render_with_theme(options, theme), delta) +} + /// Like `render_values` but without column-offset–based indentation. /// /// Inside blockquotes and callouts, node positions include the `> ` prefix offset, /// so applying column-based spacing causes double-indentation. This variant preserves /// blank lines between values (via newline counting) but ignores the column position. pub(crate) fn render_values_block(values: &[Node], options: &RenderOptions, theme: &ColorTheme<'_>) -> String { + // The children of an item that come after the items nested in it, as at the top level. + let reordered = crate::markdown::list_order::reorder(values); + let (values, continuation) = match &reordered { + Some(reordered) => (reordered.nodes.as_slice(), reordered.continuation.as_slice()), + None => (values, [].as_slice()), + }; let mut result = String::new(); let mut pre_position: Option<Position> = None; let mut index = 0; + // The column of the content of the open items, one for each level. + let mut list_indent_stack: Vec<usize> = Vec::new(); // The last node written, to tell paragraphs from the inline nodes of one when there are no positions. let mut previous: Option<&Node> = None; // What `render_before` returned for `previous`, or `None` when it is part of a table. @@ -785,9 +820,18 @@ pub(crate) fn render_values_block(values: &[Node], options: &RenderOptions, them while index < values.len() { let first = &values[index]; // The cells of a table are laid out together, as at the top level. - let (rendered, position, next) = if is_table_part(first) { + let (mut rendered, position, next) = if is_table_part(first) { + list_indent_stack.clear(); render_table_run(values, index, options, theme) + } else if let Node::List(item) = first { + let rest = continuation.get(index) == Some(&true); + ( + render_list_item(first, item, rest, &mut list_indent_stack, options, theme), + first.position(), + index + 1, + ) } else { + list_indent_stack.clear(); ( render_before( first, @@ -810,6 +854,10 @@ pub(crate) fn render_values_block(values: &[Node], options: &RenderOptions, them .unwrap_or_default(); pre_position = Some(pos); result.push_str(&"\n".repeat(new_line_count)); + // An item that starts on the line of the marker that holds it follows it directly. + if new_line_count == 0 && matches!((previous, first), (Some(Node::List(_)), Node::List(_))) { + rendered = rendered.trim_start().to_string(); + } } else { // Text never follows text inside one paragraph, so without positions that is a new one. if pre_position.is_none() && matches!((previous, first), (Some(Node::Text(_)), Node::Text(_))) { From cff56befc45f0cc65b2d7978a8dbe7f51e2c8ecd Mon Sep 17 00:00:00 2001 From: harehare <harehare1110@gmail.com> Date: Wed, 14 Oct 2026 23:03:54 +0900 Subject: [PATCH 88/88] =?UTF-8?q?=F0=9F=90=9B=20fix(markdown):=20keep=20a?= =?UTF-8?q?=20blank=20line=20after=20a=20quote=20or=20footnote=20inside=20?= =?UTF-8?q?a=20container?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A block after a nested quote or after a footnote in an item or quote was written on the next line, so it read back as lazy continuation. Share the rules for the blank line with the top level. Write an empty math block without a blank line, which was read back as a line of content. --- crates/mq-markdown/src/markdown.rs | 28 +++------- crates/mq-markdown/src/node.rs | 2 +- crates/mq-markdown/src/node/render.rs | 74 ++++++++++++++++++++------- 3 files changed, 62 insertions(+), 42 deletions(-) diff --git a/crates/mq-markdown/src/markdown.rs b/crates/mq-markdown/src/markdown.rs index e62149e86..92b2aa8b4 100644 --- a/crates/mq-markdown/src/markdown.rs +++ b/crates/mq-markdown/src/markdown.rs @@ -6,7 +6,7 @@ use crate::html_to_markdown::ConversionOptions; use crate::node::ListStyle; use crate::node::{ Code, ColorTheme, Node, Position, RenderOptions, TableAlign, TableCell, indent_lines, list_own_prefix_width, - reindent_all_lines, render_before, render_cell_values, render_values_block, + reindent_all_lines, render_before, render_cell_values, render_values_block, separated_from, }; #[cfg(any(feature = "json", feature = "html-to-markdown"))] use miette::miette; @@ -259,27 +259,7 @@ impl Markdown { .unwrap_or_else(|| if is_first { 0 } else { 1 }) .min(2); - // Single newline after a block quote reads back as lazy continuation. - if new_line_count < 2 && prev_node.is_some_and(Node::is_blockquote_like) { - new_line_count = 2; - } - - // A line of dashes right under paragraph text reads back as a setext heading. - if new_line_count < 2 - && value.starts_with("---") - && matches!(node, Node::HorizontalRule(_)) - && prev_node.is_some_and(Node::is_paragraph_text) - { - new_line_count = 2; - } - - // A footnote ends where the next line starts, so a block after it needs a blank line. - if new_line_count < 2 - && matches!(prev_node, Some(Node::Footnote(_))) - && !matches!(node, Node::Footnote(_)) - { - new_line_count = 2; - } + new_line_count = separated_from(new_line_count, prev_node, node, &value); // An item that starts on the line of the marker that holds it follows it directly. let value = @@ -642,6 +622,10 @@ mod tests { #[case::quote_list_in_list_on_one_line("> - - - x", 1, "> - - - x\n")] #[case::quote_paragraph_after_nested_item("> - a\n> - b\n>\n> para", 1, "> - a\n> - b\n> \n> para\n")] #[case::quote_bullet_nested_under_ordered("> 1. a\n> - b", 1, "> 1. a\n> - b\n")] + // inside a container a block after a quote or footnote needs a blank line, and empty math stays empty + #[case::quote_after_nested_quote("> > a\n> >\n> b", 1, "> > a\n> \n> b\n")] + #[case::paragraph_after_footnote_in_item("- [^1]: b\n\n c", 1, "- [^1]: b\n\n c\n")] + #[case::empty_math("$$\n$$", 1, "$$\n$$\n")] // an indented code block right after a list must not read back as list continuation #[case::code_after_list_uses_fence(" - one\n\n two", 2, "- one\n\n```\n two\n```\n")] // a block quote nested in a list item needs its continuation line's own indent diff --git a/crates/mq-markdown/src/node.rs b/crates/mq-markdown/src/node.rs index 0c6efdfd1..ced775fa6 100644 --- a/crates/mq-markdown/src/node.rs +++ b/crates/mq-markdown/src/node.rs @@ -17,7 +17,7 @@ mod expand; mod render; pub(crate) use render::{ for_html, indent_lines, list_own_prefix_width, reindent_all_lines, render_before, render_cell_values, - render_values_block, values_to_string, + render_values_block, separated_from, values_to_string, }; use render::{render_link_destination, render_link_title}; #[cfg(test)] diff --git a/crates/mq-markdown/src/node/render.rs b/crates/mq-markdown/src/node/render.rs index d08eb842f..f3e8ea88d 100644 --- a/crates/mq-markdown/src/node/render.rs +++ b/crates/mq-markdown/src/node/render.rs @@ -342,7 +342,11 @@ impl Node { Self::Math(Math { value, .. }) => { let (ms, me) = &theme.math; let fence = "$".repeat((longest_run(value, '$') + 1).max(2)); - format!("{}{}\n{}\n{}{}", ms, fence, value, fence, me) + if value.is_empty() { + format!("{}{}\n{}{}", ms, fence, fence, me) + } else { + format!("{}{}\n{}\n{}{}", ms, fence, value, fence, me) + } } // Only escape text parsed from real markdown (has a position); programmatic // values like JSON output or attr lookups must stay untouched. @@ -768,6 +772,23 @@ pub(crate) fn render_values(values: &[Node], options: &RenderOptions, theme: &Co .collect::<String>() } +/// The number of line endings to write before `node` after `previous`, raised to a blank line where +/// the node would otherwise read back as part of `previous`. +pub(crate) fn separated_from(count: usize, previous: Option<&Node>, node: &Node, rendered: &str) -> usize { + if count >= 2 { + return count; + } + // A single line ending after a block quote reads back as lazy continuation. + let lazy = previous.is_some_and(Node::is_blockquote_like); + // A line of dashes right under paragraph text reads back as a setext heading. + let setext = matches!(node, Node::HorizontalRule(_)) + && rendered.starts_with("---") + && previous.is_some_and(Node::is_paragraph_text); + // A footnote ends where the next line starts, so a block after it needs a blank line. + let footnote = matches!(previous, Some(Node::Footnote(_))) && !matches!(node, Node::Footnote(_)); + if lazy || setext || footnote { 2 } else { count } +} + /// Writes an item, or the rest of one that follows the items nested in it, inside a container. /// /// `indents` holds the column of the content of each open item. It is kept out of the container loop @@ -795,6 +816,38 @@ fn render_list_item( reindent_all_lines(&node.render_with_theme(options, theme), delta) } +/// Writes what separates `node` from the one written before it, and trims `rendered` when it follows +/// the marker that holds it. Not inlined, to keep the frame of the container loop small. +#[inline(never)] +fn separate( + result: &mut String, + rendered: &mut String, + pre_position: &mut Option<Position>, + position: Option<Position>, + previous: Option<&Node>, + node: &Node, +) { + if let Some(pos) = position { + let new_line_count = pre_position + .as_ref() + .map(|p: &Position| pos.start.line.saturating_sub(p.end.line)) + .unwrap_or_default(); + let new_line_count = separated_from(new_line_count, previous, node, rendered); + *pre_position = Some(pos); + result.push_str(&"\n".repeat(new_line_count)); + // An item that starts on the line of the marker that holds it follows it directly. + if new_line_count == 0 && matches!((previous, node), (Some(Node::List(_)), Node::List(_))) { + *rendered = rendered.trim_start().to_string(); + } + } else { + // Text never follows text inside one paragraph, so without positions that is a new one. + if pre_position.is_none() && matches!((previous, node), (Some(Node::Text(_)), Node::Text(_))) { + result.push_str("\n\n"); + } + *pre_position = None; + } +} + /// Like `render_values` but without column-offset–based indentation. /// /// Inside blockquotes and callouts, node positions include the `> ` prefix offset, @@ -847,24 +900,7 @@ pub(crate) fn render_values_block(values: &[Node], options: &RenderOptions, them }; index = next; - if let Some(pos) = position { - let new_line_count = pre_position - .as_ref() - .map(|p: &Position| pos.start.line.saturating_sub(p.end.line)) - .unwrap_or_default(); - pre_position = Some(pos); - result.push_str(&"\n".repeat(new_line_count)); - // An item that starts on the line of the marker that holds it follows it directly. - if new_line_count == 0 && matches!((previous, first), (Some(Node::List(_)), Node::List(_))) { - rendered = rendered.trim_start().to_string(); - } - } else { - // Text never follows text inside one paragraph, so without positions that is a new one. - if pre_position.is_none() && matches!((previous, first), (Some(Node::Text(_)), Node::Text(_))) { - result.push_str("\n\n"); - } - pre_position = None; - } + separate(&mut result, &mut rendered, &mut pre_position, position, previous, first); previous = Some(&values[index - 1]); result.push_str(&rendered); previous_rendered = (!is_table_part(first)).then_some(rendered);