From c11d6592c8bdf2fad8e1fb5fc5f5f9175451f0b0 Mon Sep 17 00:00:00 2001 From: harehare Date: Tue, 27 Oct 2026 21:55:38 +0900 Subject: [PATCH 1/2] =?UTF-8?q?=E2=9A=A1=20perf(markdown):=20store=20`Poin?= =?UTF-8?q?t`=20line=20and=20column=20as=20`u32`?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `Option` shrinks from 40 to 20 bytes, which lowers the size of every node. On a 19.6MB document the peak RSS of `nodes | len` goes from 681MB to 601MB. `Point.line` and `Point.column` are public, so this changes the API of mq-markdown. Parser internals keep `usize` and saturate to `u32::MAX` where a `Point` is built. mq-ffi no longer needs its own saturating conversion. --- crates/mq-ffi/src/values.rs | 12 ++++------- crates/mq-markdown/src/markdown.rs | 2 +- crates/mq-markdown/src/markdown/list_order.rs | 6 +++--- crates/mq-markdown/src/node.rs | 4 ++-- crates/mq-markdown/src/node/render.rs | 20 +++++++++++++------ crates/mq-markdown/src/parser.rs | 2 +- crates/mq-markdown/src/parser/error.rs | 9 ++------- crates/mq-markdown/src/parser/line.rs | 13 ++++++++---- crates/mq-markdown/src/parser/render_html.rs | 10 +++++----- crates/mq-markdown/src/parser/tree.rs | 4 ++-- crates/mq-markdown/tests/mdx_error_tests.rs | 4 ++-- .../tests/parser_property_tests.rs | 10 +++++----- .../tests/parser_structure_tests.rs | 10 +++++----- crates/mq-run/src/grep.rs | 6 +++--- 14 files changed, 58 insertions(+), 54 deletions(-) diff --git a/crates/mq-ffi/src/values.rs b/crates/mq-ffi/src/values.rs index 0b31f3ed9..55d36cb4d 100644 --- a/crates/mq-ffi/src/values.rs +++ b/crates/mq-ffi/src/values.rs @@ -233,10 +233,10 @@ pub unsafe extern "C" fn mq_value_position(values: *const MqValues, index: usize .and_then(|values| values.get(index)) .and_then(|(value, _)| value.position()) .map(|p| MqPosition { - start_line: saturate(p.start.line), - start_column: saturate(p.start.column), - end_line: saturate(p.end.line), - end_column: saturate(p.end.column), + start_line: p.start.line, + start_column: p.start.column, + end_line: p.end.line, + end_column: p.end.column, }); store(position, out) } @@ -271,10 +271,6 @@ pub unsafe extern "C" fn mq_values_to_json(values: *const MqValues) -> *mut c_ch }) } -fn saturate(n: usize) -> u32 { - u32::try_from(n).unwrap_or(u32::MAX) -} - // Writes `value` to `out` if both exist. fn store(value: Option, out: *mut T) -> bool { match (value, unsafe { out.as_mut() }) { diff --git a/crates/mq-markdown/src/markdown.rs b/crates/mq-markdown/src/markdown.rs index 3089b57a1..24ad7d928 100644 --- a/crates/mq-markdown/src/markdown.rs +++ b/crates/mq-markdown/src/markdown.rs @@ -65,7 +65,7 @@ impl fmt::Display for Markdown { /// The line where the block before a node ends. An item that ends in a fence without a closing one /// ends at the start of the line after it, which is not yet a line of its own. -fn last_line(position: &Position, node: Option<&Node>) -> usize { +fn last_line(position: &Position, node: Option<&Node>) -> u32 { if matches!(node, Some(Node::List(_))) && position.end.column == 1 && position.end.line > position.start.line { position.end.line - 1 } else { diff --git a/crates/mq-markdown/src/markdown/list_order.rs b/crates/mq-markdown/src/markdown/list_order.rs index 5a74a4751..093d21de1 100644 --- a/crates/mq-markdown/src/markdown/list_order.rs +++ b/crates/mq-markdown/src/markdown/list_order.rs @@ -50,7 +50,7 @@ pub(crate) fn reorder(nodes: &[Node]) -> Option { Some(out) } -fn start_line(node: &Node) -> Option { +fn start_line(node: &Node) -> Option { node.position().map(|position| position.start.line) } @@ -67,7 +67,7 @@ fn reorder_into(nodes: &[Node], out: &mut Reordered) { .iter() .take_while(|next| matches!(next, Node::List(next) if next.level > item.level)) .count(); - let lines: Option> = item.values.iter().map(start_line).collect(); + let lines: Option> = item.values.iter().map(start_line).collect(); let (true, Some(lines)) = (!item.values.is_empty() && nested > 0, lines) else { out.push(node.clone()); index += 1; @@ -86,7 +86,7 @@ fn reorder_into(nodes: &[Node], out: &mut Reordered) { from = at; } } - let segment_lines: Option> = segments.iter().map(|segment| start_line(&segment[0])).collect(); + let segment_lines: Option> = segments.iter().map(|segment| start_line(&segment[0])).collect(); let Some(segment_lines) = segment_lines else { out.push(node.clone()); index += 1; diff --git a/crates/mq-markdown/src/node.rs b/crates/mq-markdown/src/node.rs index 93e80e895..2d2601cdf 100644 --- a/crates/mq-markdown/src/node.rs +++ b/crates/mq-markdown/src/node.rs @@ -1334,9 +1334,9 @@ pub struct Position { )] pub struct Point { /// The one-based line. - pub line: usize, + pub line: u32, /// The one-based column, counted in UTF-16 code units. A tab is one column. - pub column: usize, + pub column: u32, } impl From for Node { diff --git a/crates/mq-markdown/src/node/render.rs b/crates/mq-markdown/src/node/render.rs index 7f6cf2fe2..a82efdfbb 100644 --- a/crates/mq-markdown/src/node/render.rs +++ b/crates/mq-markdown/src/node/render.rs @@ -142,7 +142,15 @@ impl Node { let blank = position .as_ref() .map_or(0, |p| p.end.line.saturating_sub(p.start.line).saturating_sub(1)); - format!("{}{}{}\n{}{}{}", cs, fence_str, info, "\n".repeat(blank), fence_str, ce) + format!( + "{}{}{}\n{}{}{}", + cs, + fence_str, + info, + "\n".repeat(blank as usize), + fence_str, + ce + ) } else { format!("{}{}{}\n{}\n{}{}", cs, fence_str, info, value, fence_str, ce) } @@ -745,7 +753,7 @@ pub(crate) fn render_values(values: &[Node], options: &RenderOptions, theme: &Co .map(|p| pos.start.line > p.end.line) .unwrap_or_default() { - " ".repeat(pos.start.column.saturating_sub(1)) + " ".repeat(pos.start.column.saturating_sub(1) as usize) } else { "".to_string() }; @@ -753,11 +761,11 @@ pub(crate) fn render_values(values: &[Node], options: &RenderOptions, theme: &Co pre_position = Some(pos); let piece = if space.is_empty() { - format!("{}{}", "\n".repeat(new_line_count), rendered()) + format!("{}{}", "\n".repeat(new_line_count as usize), rendered()) } else { format!( "{}{}", - "\n".repeat(new_line_count), + "\n".repeat(new_line_count as usize), rendered().lines().map(|line| format!("{}{}", space, line)).join("\n") ) }; @@ -775,7 +783,7 @@ pub(crate) fn render_values(values: &[Node], options: &RenderOptions, theme: &Co /// The number of line endings to write before `node` after `previous`, raised to a blank line where /// the node would otherwise read back as part of `previous`. -pub(crate) fn separated_from(count: usize, previous: Option<&Node>, node: &Node, rendered: &str) -> usize { +pub(crate) fn separated_from(count: u32, previous: Option<&Node>, node: &Node, rendered: &str) -> u32 { if count >= 2 { return count; } @@ -837,7 +845,7 @@ fn separate( .unwrap_or_default(); let new_line_count = separated_from(new_line_count, previous, node, rendered); *pre_position = Some(pos); - result.push_str(&"\n".repeat(new_line_count)); + result.push_str(&"\n".repeat(new_line_count as usize)); // An item that starts on the line of the marker that holds it follows it directly. if new_line_count == 0 && matches!((previous, node), (Some(Node::List(_)), Node::List(_))) { *rendered = rendered.trim_start().to_string(); diff --git a/crates/mq-markdown/src/parser.rs b/crates/mq-markdown/src/parser.rs index 043833314..a8789cc6e 100644 --- a/crates/mq-markdown/src/parser.rs +++ b/crates/mq-markdown/src/parser.rs @@ -413,7 +413,7 @@ mod tests { let (start, end) = (&position.start, &position.end); let ordered = (start.line, start.column) <= (end.line, end.column); assert!( - ordered && start.line >= 1 && start.column >= 1 && end.line <= lines, + ordered && start.line >= 1 && start.column >= 1 && end.line as usize <= lines, "{node:?} has a position outside of {input:?}" ); } diff --git a/crates/mq-markdown/src/parser/error.rs b/crates/mq-markdown/src/parser/error.rs index e837cf747..9339371c1 100644 --- a/crates/mq-markdown/src/parser/error.rs +++ b/crates/mq-markdown/src/parser/error.rs @@ -293,7 +293,7 @@ fn offset_of(content: &str, point: &Point) -> Option { if column + 1 >= point.column { return Some(start + index); } - column += char.len_utf16(); + column += char.len_utf16() as u32; } Some(start + line.len()) } @@ -312,12 +312,7 @@ mod tests { #[case::astral("😀x", 1, 3, Some(4))] #[case::end_of_line("ab\ncd", 1, 3, Some(2))] #[case::missing_line("ab", 3, 1, None)] - fn offset_of_point( - #[case] content: &str, - #[case] line: usize, - #[case] column: usize, - #[case] expected: Option, - ) { + fn offset_of_point(#[case] content: &str, #[case] line: u32, #[case] column: u32, #[case] expected: Option) { assert_eq!(offset_of(content, &Point { line, column }), expected); } } diff --git a/crates/mq-markdown/src/parser/line.rs b/crates/mq-markdown/src/parser/line.rs index 75ccf660d..cdb10d25d 100644 --- a/crates/mq-markdown/src/parser/line.rs +++ b/crates/mq-markdown/src/parser/line.rs @@ -17,7 +17,7 @@ pub(super) struct Indent { #[derive(Clone, Copy)] pub(super) struct Line<'a> { - pub(super) number: usize, + pub(super) number: u32, /// Byte offset of `text` within the original line, non-zero inside containers. pub(super) column: usize, pub(super) text: &'a str, @@ -94,7 +94,7 @@ impl<'a> Line<'a> { let offset = self.column + byte; Point { line: self.number, - column: offset - extra_bytes(&self.whole.as_bytes()[..offset.min(self.whole.len())]) + 1, + column: to_u32(offset - extra_bytes(&self.whole.as_bytes()[..offset.min(self.whole.len())]) + 1), } } @@ -185,6 +185,11 @@ pub(super) fn tab_stops(src: &str) -> Vec { stops } +/// Converts a count to a [`Point`] coordinate, saturating on inputs past `u32::MAX`. +pub(super) fn to_u32(value: usize) -> u32 { + u32::try_from(value).unwrap_or(u32::MAX) +} + /// How many of `bytes`, which are whole characters, are more than UTF-16 code units: each continuation /// byte is one, except that a character of four bytes is two code units. pub(super) fn extra_bytes(bytes: &[u8]) -> usize { @@ -242,7 +247,7 @@ pub(super) fn split_lines<'a>(src: &'a str, flavor: Flavor, stops: &'a [TabStop] } } lines.push(Line { - number: lines.len() + 1, + number: to_u32(lines.len() + 1), column: 0, text: &src[start..end], eol: &src[end..index], @@ -262,7 +267,7 @@ pub(super) fn split_lines<'a>(src: &'a str, flavor: Flavor, stops: &'a [TabStop] if start < src.len() { lines.push(Line { - number: lines.len() + 1, + number: to_u32(lines.len() + 1), column: 0, text: &src[start..], eol: "", diff --git a/crates/mq-markdown/src/parser/render_html.rs b/crates/mq-markdown/src/parser/render_html.rs index d37db8b35..677a0b1eb 100644 --- a/crates/mq-markdown/src/parser/render_html.rs +++ b/crates/mq-markdown/src/parser/render_html.rs @@ -58,7 +58,7 @@ fn first_line_ending(content: &str) -> &'static str { /// Whether the source has a line ending after the line on which `end` ends. A document that has no /// block to render has none. -fn has_line_ending_after(content: &str, end: Option) -> bool { +fn has_line_ending_after(content: &str, end: Option) -> bool { let Some(end) = end else { return false }; let mut lines = 1; let mut bytes = content.bytes().peekable(); @@ -74,7 +74,7 @@ fn has_line_ending_after(content: &str, end: Option) -> bool { _ => {} } } - end < lines + (end as usize) < lines } /// Whether a list is followed by nothing but blocks without output, such as definitions, and no blank @@ -103,14 +103,14 @@ fn list_then_hidden(content: &str, blocks: &[Block]) -> bool { } /// The line on which the last block that produces output ends. -fn last_rendered(blocks: &[Block]) -> Option { +fn last_rendered(blocks: &[Block]) -> Option { blocks.iter().rev().find_map(|block| match block { Block::Node(Node::Definition(_) | Node::Yaml(_) | Node::Toml(_)) | Block::Footnote(_) => None, _ => end_line(block), }) } -fn end_line(block: &Block) -> Option { +fn end_line(block: &Block) -> Option { match block { Block::Node(node) => node.position().map(|position| position.end.line), Block::Fenced(fenced) => fenced.node.position().map(|position| position.end.line), @@ -136,7 +136,7 @@ fn end_line(block: &Block) -> Option { } } -fn start_line(block: &Block) -> Option { +fn start_line(block: &Block) -> Option { match block { Block::Node(node) => node.position().map(|position| position.start.line), Block::Fenced(fenced) => fenced.node.position().map(|position| position.start.line), diff --git a/crates/mq-markdown/src/parser/tree.rs b/crates/mq-markdown/src/parser/tree.rs index af0935468..1f5ab5895 100644 --- a/crates/mq-markdown/src/parser/tree.rs +++ b/crates/mq-markdown/src/parser/tree.rs @@ -4,7 +4,7 @@ //! it runs in [`super::resolve`] once all blocks are known. use super::error::MdxError; -use super::line::extra_bytes; +use super::line::{extra_bytes, to_u32}; use super::mdx::TagKind; use super::scan::eol_len; use crate::node::{HeadingDepth, ListMarker, MdxAttributeContent, Node, Point, Position, TableAlignKind}; @@ -163,7 +163,7 @@ impl InlineSource { }; Point { line: point.line, - column: point.column + (offset - start) - extra, + column: point.column + to_u32(offset - start - extra), } } } diff --git a/crates/mq-markdown/tests/mdx_error_tests.rs b/crates/mq-markdown/tests/mdx_error_tests.rs index 59a2c49b0..d6caaab7c 100644 --- a/crates/mq-markdown/tests/mdx_error_tests.rs +++ b/crates/mq-markdown/tests/mdx_error_tests.rs @@ -62,8 +62,8 @@ fn diagnostic(input: &str) -> MdxDiagnostic { fn reports_kind_and_position( #[case] input: &str, #[case] kind: MdxErrorKind, - #[case] line: usize, - #[case] column: usize, + #[case] line: u32, + #[case] column: u32, #[case] offset: usize, ) { let diagnostic = diagnostic(input); diff --git a/crates/mq-markdown/tests/parser_property_tests.rs b/crates/mq-markdown/tests/parser_property_tests.rs index 21bc79180..1f420ca51 100644 --- a/crates/mq-markdown/tests/parser_property_tests.rs +++ b/crates/mq-markdown/tests/parser_property_tests.rs @@ -208,8 +208,8 @@ proptest! { let source = blocks.join("\n\n") + "\n"; for node in walk(parse(&source)) { let Some(position) = node.position() else { continue }; - let start = byte_offset(&source, position.start.line, position.start.column); - let end = byte_offset(&source, position.end.line, position.end.column); + let start = byte_offset(&source, position.start.line as usize, position.start.column as usize); + let end = byte_offset(&source, position.end.line as usize, position.end.column as usize); prop_assert!(start.is_some() && end.is_some(), "{node:?}"); let (start, end) = (start.unwrap(), end.unwrap()); prop_assert!(start <= end && end <= source.len(), "{node:?} in {source:?}"); @@ -332,13 +332,13 @@ proptest! { replacement in prop::sample::select(vec!['あ', 'é', '🎉', 'ａ']) ) { let replaced = replace_letters(&source, &[replacement; 6]); - let lines = replaced.matches('\n').count() + 1; + let lines = replaced.matches('\n').count() as u32 + 1; for node in walk(parse(&replaced)) { let Some(position) = node.position() else { continue }; prop_assert!(position.end.line <= lines, "{node:?}"); - let line = replaced.split('\n').nth(position.end.line - 1).unwrap_or_default(); + let line = replaced.split('\n').nth(position.end.line as usize - 1).unwrap_or_default(); // A column can be one past the end of its line, and no further. - prop_assert!(position.end.column <= line.len() + 1, "{node:?} in {replaced:?}"); + prop_assert!(position.end.column as usize <= line.len() + 1, "{node:?} in {replaced:?}"); } } } diff --git a/crates/mq-markdown/tests/parser_structure_tests.rs b/crates/mq-markdown/tests/parser_structure_tests.rs index b019c56a4..1868673c4 100644 --- a/crates/mq-markdown/tests/parser_structure_tests.rs +++ b/crates/mq-markdown/tests/parser_structure_tests.rs @@ -157,7 +157,7 @@ fn attributes(#[case] input: &str, #[case] name: &str, #[case] expected: &str) { #[case::escape_at_the_start("\\*a", 0, (1, 1), (1, 4))] #[case::escape_after_text("a\\*b", 0, (1, 1), (1, 5))] #[case::after_footnote("a[^1]\n\n[^1]: b", 1, (1, 2), (1, 6))] -fn positions(#[case] input: &str, #[case] index: usize, #[case] start: (usize, usize), #[case] end: (usize, usize)) { +fn positions(#[case] input: &str, #[case] index: usize, #[case] start: (u32, u32), #[case] end: (u32, u32)) { let nodes = parse(input); let position = nodes[index] .position() @@ -179,8 +179,8 @@ fn positions(#[case] input: &str, #[case] index: usize, #[case] start: (usize, u fn table_rows_include_trailing_whitespace(#[case] input: &str) { for node in parse(input) { let position = node.position().unwrap(); - let line = input.lines().nth(position.end.line - 1).unwrap(); - assert!(position.end.column <= line.len() + 1, "{node:?}"); + let line = input.lines().nth(position.end.line as usize - 1).unwrap(); + assert!(position.end.column as usize <= line.len() + 1, "{node:?}"); } let last_cell = parse(input) .into_iter() @@ -188,8 +188,8 @@ fn table_rows_include_trailing_whitespace(#[case] input: &str) { .unwrap(); let end = last_cell.position().unwrap().end; assert_eq!( - end.column, - input.lines().nth(end.line - 1).unwrap().len() + 1, + end.column as usize, + input.lines().nth(end.line as usize - 1).unwrap().len() + 1, "{last_cell:?}" ); } diff --git a/crates/mq-run/src/grep.rs b/crates/mq-run/src/grep.rs index e93057e00..ca3e518dc 100644 --- a/crates/mq-run/src/grep.rs +++ b/crates/mq-run/src/grep.rs @@ -25,7 +25,7 @@ pub(crate) fn print_grep( let filename = file.as_ref().map(|p| p.to_string_lossy().into_owned()); // Collect start lines of matched nodes for quick lookup. - let match_lines: HashSet = runtime_values + let match_lines: HashSet = runtime_values .values() .iter() .filter_map(|v| { @@ -105,7 +105,7 @@ pub(crate) fn print_grep( handle.finish() } -fn format_line(filename: &Option, line_num: Option, content: &str, sep: &str) -> String { +fn format_line(filename: &Option, line_num: Option, content: &str, sep: &str) -> String { match (filename, line_num) { (Some(f), Some(l)) => format!("{}{}{}{}{}\n", f, sep, l, sep, content), (Some(f), None) => format!("{}{}{}\n", f, sep, content), @@ -208,7 +208,7 @@ mod tests { #[case::no_filename_no_line(None, None, "plain", ":", "plain\n")] fn test_format_line( #[case] filename: Option, - #[case] line_num: Option, + #[case] line_num: Option, #[case] content: &str, #[case] sep: &str, #[case] expected: &str, From d03e9a6133a9ce2780c2d4f113509448bef508f5 Mon Sep 17 00:00:00 2001 From: harehare Date: Tue, 27 Oct 2026 21:55:38 +0900 Subject: [PATCH 2/2] =?UTF-8?q?=E2=9A=A1=20perf(markdown):=20trim=20spare?= =?UTF-8?q?=20capacity=20while=20building=20the=20tree?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A `Vec` starts at four elements, so a container with one child held room for four blocks. Nested block vectors are now shrunk once they are complete, which takes the block tree from 267MB to 196MB on a 19.6MB document. `resolve` also copied the whole root vector when appending it into the empty output. It now takes the vector and shrinks it, which lowers the peak during resolution from 506MB to 380MB. Peak RSS of `nodes | len` goes from 681MB to 532MB overall, and the run is about 10% faster. --- crates/mq-markdown/src/parser/block.rs | 4 ++++ crates/mq-markdown/src/parser/resolve.rs | 7 ++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/crates/mq-markdown/src/parser/block.rs b/crates/mq-markdown/src/parser/block.rs index 0fc5a0470..0e45c1a72 100644 --- a/crates/mq-markdown/src/parser/block.rs +++ b/crates/mq-markdown/src/parser/block.rs @@ -140,6 +140,10 @@ fn parse_blocks(lines: &[Line<'_>], depth: usize, interrupting: Interrupt) -> Ve after_paragraph = step.interrupt; } + // Containers mostly hold a few blocks, and a `Vec` starts at four. + if depth > 0 { + blocks.shrink_to_fit(); + } blocks } diff --git a/crates/mq-markdown/src/parser/resolve.rs b/crates/mq-markdown/src/parser/resolve.rs index f460e58d4..f80fe290e 100644 --- a/crates/mq-markdown/src/parser/resolve.rs +++ b/crates/mq-markdown/src/parser/resolve.rs @@ -192,7 +192,12 @@ fn flatten(blocks: Vec, doc: Document<'_>, nodes: &mut Vec) -> Resu Some(opened_at), )); } - nodes.append(&mut frames.root); + if nodes.is_empty() { + *nodes = frames.root; + nodes.shrink_to_fit(); + } else { + nodes.append(&mut frames.root); + } Ok(()) }