⚡ perf(markdown): reduce memory used by large Markdown inputs

crates/mq-ffi/src/values.rs

@@ -233,10 +233,10 @@ pub unsafe extern "C" fn mq_value_position(values: *const MqValues, index: usize
         .and_then(|values| values.get(index))
         .and_then(|(value, _)| value.position())
         .map(|p| MqPosition {
-            start_line: saturate(p.start.line),
-            start_column: saturate(p.start.column),
-            end_line: saturate(p.end.line),
-            end_column: saturate(p.end.column),
+            start_line: p.start.line,
+            start_column: p.start.column,
+            end_line: p.end.line,
+            end_column: p.end.column,
         });
     store(position, out)
 }
@@ -271,10 +271,6 @@ pub unsafe extern "C" fn mq_values_to_json(values: *const MqValues) -> *mut c_ch
     })
 }
 
-fn saturate(n: usize) -> u32 {
-    u32::try_from(n).unwrap_or(u32::MAX)
-}
-
 // Writes `value` to `out` if both exist.
 fn store<T>(value: Option<T>, out: *mut T) -> bool {
     match (value, unsafe { out.as_mut() }) {

crates/mq-markdown/src/markdown.rs

@@ -65,7 +65,7 @@ impl fmt::Display for Markdown {
 
 /// The line where the block before a node ends. An item that ends in a fence without a closing one
 /// ends at the start of the line after it, which is not yet a line of its own.
-fn last_line(position: &Position, node: Option<&Node>) -> usize {
+fn last_line(position: &Position, node: Option<&Node>) -> u32 {
     if matches!(node, Some(Node::List(_))) && position.end.column == 1 && position.end.line > position.start.line {
         position.end.line - 1
     } else {

crates/mq-markdown/src/markdown/list_order.rs

@@ -50,7 +50,7 @@ pub(crate) fn reorder(nodes: &[Node]) -> Option<Reordered> {
     Some(out)
 }
 
-fn start_line(node: &Node) -> Option<usize> {
+fn start_line(node: &Node) -> Option<u32> {
     node.position().map(|position| position.start.line)
 }
 
@@ -67,7 +67,7 @@ fn reorder_into(nodes: &[Node], out: &mut Reordered) {
             .iter()
             .take_while(|next| matches!(next, Node::List(next) if next.level > item.level))
             .count();
-        let lines: Option<Vec<usize>> = item.values.iter().map(start_line).collect();
+        let lines: Option<Vec<u32>> = item.values.iter().map(start_line).collect();
         let (true, Some(lines)) = (!item.values.is_empty() && nested > 0, lines) else {
             out.push(node.clone());
             index += 1;
@@ -86,7 +86,7 @@ fn reorder_into(nodes: &[Node], out: &mut Reordered) {
                 from = at;
             }
         }
-        let segment_lines: Option<Vec<usize>> = segments.iter().map(|segment| start_line(&segment[0])).collect();
+        let segment_lines: Option<Vec<u32>> = segments.iter().map(|segment| start_line(&segment[0])).collect();
         let Some(segment_lines) = segment_lines else {
             out.push(node.clone());
             index += 1;

crates/mq-markdown/src/node.rs

@@ -1334,9 +1334,9 @@ pub struct Position {
 )]
 pub struct Point {
     /// The one-based line.
-    pub line: usize,
+    pub line: u32,
     /// The one-based column, counted in UTF-16 code units. A tab is one column.
-    pub column: usize,
+    pub column: u32,
 }
 
 impl From<String> for Node {

crates/mq-markdown/src/node/render.rs

@@ -142,7 +142,15 @@ impl Node {
                         let blank = position
                             .as_ref()
                             .map_or(0, |p| p.end.line.saturating_sub(p.start.line).saturating_sub(1));
-                        format!("{}{}{}\n{}{}{}", cs, fence_str, info, "\n".repeat(blank), fence_str, ce)
+                        format!(
+                            "{}{}{}\n{}{}{}",
+                            cs,
+                            fence_str,
+                            info,
+                            "\n".repeat(blank as usize),
+                            fence_str,
+                            ce
+                        )
                     } else {
                         format!("{}{}{}\n{}\n{}{}", cs, fence_str, info, value, fence_str, ce)
                     }
@@ -745,19 +753,19 @@ pub(crate) fn render_values(values: &[Node], options: &RenderOptions, theme: &Co
                         .map(|p| pos.start.line > p.end.line)
                         .unwrap_or_default()
                 {
-                    " ".repeat(pos.start.column.saturating_sub(1))
+                    " ".repeat(pos.start.column.saturating_sub(1) as usize)
                 } else {
                     "".to_string()
                 };
 
                 pre_position = Some(pos);
 
                 let piece = if space.is_empty() {
-                    format!("{}{}", "\n".repeat(new_line_count), rendered())
+                    format!("{}{}", "\n".repeat(new_line_count as usize), rendered())
                 } else {
                     format!(
                         "{}{}",
-                        "\n".repeat(new_line_count),
+                        "\n".repeat(new_line_count as usize),
                         rendered().lines().map(|line| format!("{}{}", space, line)).join("\n")
                     )
                 };
@@ -775,7 +783,7 @@ pub(crate) fn render_values(values: &[Node], options: &RenderOptions, theme: &Co
 
 /// The number of line endings to write before `node` after `previous`, raised to a blank line where
 /// the node would otherwise read back as part of `previous`.
-pub(crate) fn separated_from(count: usize, previous: Option<&Node>, node: &Node, rendered: &str) -> usize {
+pub(crate) fn separated_from(count: u32, previous: Option<&Node>, node: &Node, rendered: &str) -> u32 {
     if count >= 2 {
         return count;
     }
@@ -837,7 +845,7 @@ fn separate(
             .unwrap_or_default();
         let new_line_count = separated_from(new_line_count, previous, node, rendered);
         *pre_position = Some(pos);
-        result.push_str(&"\n".repeat(new_line_count));
+        result.push_str(&"\n".repeat(new_line_count as usize));
         // An item that starts on the line of the marker that holds it follows it directly.
         if new_line_count == 0 && matches!((previous, node), (Some(Node::List(_)), Node::List(_))) {
             *rendered = rendered.trim_start().to_string();

crates/mq-markdown/src/parser.rs

@@ -413,7 +413,7 @@ mod tests {
                 let (start, end) = (&position.start, &position.end);
                 let ordered = (start.line, start.column) <= (end.line, end.column);
                 assert!(
-                    ordered && start.line >= 1 && start.column >= 1 && end.line <= lines,
+                    ordered && start.line >= 1 && start.column >= 1 && end.line as usize <= lines,
                     "{node:?} has a position outside of {input:?}"
                 );
             }

crates/mq-markdown/src/parser/block.rs

@@ -140,6 +140,10 @@ fn parse_blocks(lines: &[Line<'_>], depth: usize, interrupting: Interrupt) -> Ve
         after_paragraph = step.interrupt;
     }
 
+    // Containers mostly hold a few blocks, and a `Vec` starts at four.
+    if depth > 0 {
+        blocks.shrink_to_fit();
+    }
     blocks
 }
 

crates/mq-markdown/src/parser/error.rs

@@ -293,7 +293,7 @@ fn offset_of(content: &str, point: &Point) -> Option<usize> {
         if column + 1 >= point.column {
             return Some(start + index);
         }
-        column += char.len_utf16();
+        column += char.len_utf16() as u32;
     }
     Some(start + line.len())
 }
@@ -312,12 +312,7 @@ mod tests {
     #[case::astral("😀x", 1, 3, Some(4))]
     #[case::end_of_line("ab\ncd", 1, 3, Some(2))]
     #[case::missing_line("ab", 3, 1, None)]
-    fn offset_of_point(
-        #[case] content: &str,
-        #[case] line: usize,
-        #[case] column: usize,
-        #[case] expected: Option<usize>,
-    ) {
+    fn offset_of_point(#[case] content: &str, #[case] line: u32, #[case] column: u32, #[case] expected: Option<usize>) {
         assert_eq!(offset_of(content, &Point { line, column }), expected);
     }
 }

crates/mq-markdown/src/parser/line.rs

@@ -17,7 +17,7 @@ pub(super) struct Indent {
 
 #[derive(Clone, Copy)]
 pub(super) struct Line<'a> {
-    pub(super) number: usize,
+    pub(super) number: u32,
     /// Byte offset of `text` within the original line, non-zero inside containers.
     pub(super) column: usize,
     pub(super) text: &'a str,
@@ -94,7 +94,7 @@ impl<'a> Line<'a> {
         let offset = self.column + byte;
         Point {
             line: self.number,
-            column: offset - extra_bytes(&self.whole.as_bytes()[..offset.min(self.whole.len())]) + 1,
+            column: to_u32(offset - extra_bytes(&self.whole.as_bytes()[..offset.min(self.whole.len())]) + 1),
         }
     }
 
@@ -185,6 +185,11 @@ pub(super) fn tab_stops(src: &str) -> Vec<TabStop> {
     stops
 }
 
+/// Converts a count to a [`Point`] coordinate, saturating on inputs past `u32::MAX`.
+pub(super) fn to_u32(value: usize) -> u32 {
+    u32::try_from(value).unwrap_or(u32::MAX)
+}
+
 /// How many of `bytes`, which are whole characters, are more than UTF-16 code units: each continuation
 /// byte is one, except that a character of four bytes is two code units.
 pub(super) fn extra_bytes(bytes: &[u8]) -> usize {
@@ -242,7 +247,7 @@ pub(super) fn split_lines<'a>(src: &'a str, flavor: Flavor, stops: &'a [TabStop]
             }
         }
         lines.push(Line {
-            number: lines.len() + 1,
+            number: to_u32(lines.len() + 1),
             column: 0,
             text: &src[start..end],
             eol: &src[end..index],
@@ -262,7 +267,7 @@ pub(super) fn split_lines<'a>(src: &'a str, flavor: Flavor, stops: &'a [TabStop]
 
     if start < src.len() {
         lines.push(Line {
-            number: lines.len() + 1,
+            number: to_u32(lines.len() + 1),
             column: 0,
             text: &src[start..],
             eol: "",

crates/mq-markdown/src/parser/render_html.rs

@@ -58,7 +58,7 @@ fn first_line_ending(content: &str) -> &'static str {
 
 /// Whether the source has a line ending after the line on which `end` ends. A document that has no
 /// block to render has none.
-fn has_line_ending_after(content: &str, end: Option<usize>) -> bool {
+fn has_line_ending_after(content: &str, end: Option<u32>) -> bool {
     let Some(end) = end else { return false };
     let mut lines = 1;
     let mut bytes = content.bytes().peekable();
@@ -74,7 +74,7 @@ fn has_line_ending_after(content: &str, end: Option<usize>) -> bool {
             _ => {}
         }
     }
-    end < lines
+    (end as usize) < lines
 }
 
 /// Whether a list is followed by nothing but blocks without output, such as definitions, and no blank
@@ -103,14 +103,14 @@ fn list_then_hidden(content: &str, blocks: &[Block]) -> bool {
 }
 
 /// The line on which the last block that produces output ends.
-fn last_rendered(blocks: &[Block]) -> Option<usize> {
+fn last_rendered(blocks: &[Block]) -> Option<u32> {
     blocks.iter().rev().find_map(|block| match block {
         Block::Node(Node::Definition(_) | Node::Yaml(_) | Node::Toml(_)) | Block::Footnote(_) => None,
         _ => end_line(block),
     })
 }
 
-fn end_line(block: &Block) -> Option<usize> {
+fn end_line(block: &Block) -> Option<u32> {
     match block {
         Block::Node(node) => node.position().map(|position| position.end.line),
         Block::Fenced(fenced) => fenced.node.position().map(|position| position.end.line),
@@ -136,7 +136,7 @@ fn end_line(block: &Block) -> Option<usize> {
     }
 }
 
-fn start_line(block: &Block) -> Option<usize> {
+fn start_line(block: &Block) -> Option<u32> {
     match block {
         Block::Node(node) => node.position().map(|position| position.start.line),
         Block::Fenced(fenced) => fenced.node.position().map(|position| position.start.line),

crates/mq-markdown/src/parser/resolve.rs

@@ -192,7 +192,12 @@ fn flatten(blocks: Vec<Block>, doc: Document<'_>, nodes: &mut Vec<Node>) -> Resu
             Some(opened_at),
         ));
     }
-    nodes.append(&mut frames.root);
+    if nodes.is_empty() {
+        *nodes = frames.root;
+        nodes.shrink_to_fit();
+    } else {
+        nodes.append(&mut frames.root);
+    }
 
     Ok(())
 }

crates/mq-markdown/src/parser/tree.rs

@@ -4,7 +4,7 @@
 //! it runs in [`super::resolve`] once all blocks are known.
 
 use super::error::MdxError;
-use super::line::extra_bytes;
+use super::line::{extra_bytes, to_u32};
 use super::mdx::TagKind;
 use super::scan::eol_len;
 use crate::node::{HeadingDepth, ListMarker, MdxAttributeContent, Node, Point, Position, TableAlignKind};
@@ -163,7 +163,7 @@ impl InlineSource {
         };
         Point {
             line: point.line,
-            column: point.column + (offset - start) - extra,
+            column: point.column + to_u32(offset - start - extra),
         }
     }
 }

crates/mq-markdown/tests/mdx_error_tests.rs

@@ -62,8 +62,8 @@ fn diagnostic(input: &str) -> MdxDiagnostic {
 fn reports_kind_and_position(
     #[case] input: &str,
     #[case] kind: MdxErrorKind,
-    #[case] line: usize,
-    #[case] column: usize,
+    #[case] line: u32,
+    #[case] column: u32,
     #[case] offset: usize,
 ) {
     let diagnostic = diagnostic(input);

crates/mq-markdown/tests/parser_property_tests.rs

@@ -208,8 +208,8 @@ proptest! {
         let source = blocks.join("\n\n") + "\n";
         for node in walk(parse(&source)) {
             let Some(position) = node.position() else { continue };
-            let start = byte_offset(&source, position.start.line, position.start.column);
-            let end = byte_offset(&source, position.end.line, position.end.column);
+            let start = byte_offset(&source, position.start.line as usize, position.start.column as usize);
+            let end = byte_offset(&source, position.end.line as usize, position.end.column as usize);
             prop_assert!(start.is_some() && end.is_some(), "{node:?}");
             let (start, end) = (start.unwrap(), end.unwrap());
             prop_assert!(start <= end && end <= source.len(), "{node:?} in {source:?}");
@@ -332,13 +332,13 @@ proptest! {
         replacement in prop::sample::select(vec!['あ', 'é', '🎉', 'ａ'])
     ) {
         let replaced = replace_letters(&source, &[replacement; 6]);
-        let lines = replaced.matches('\n').count() + 1;
+        let lines = replaced.matches('\n').count() as u32 + 1;
         for node in walk(parse(&replaced)) {
             let Some(position) = node.position() else { continue };
             prop_assert!(position.end.line <= lines, "{node:?}");
-            let line = replaced.split('\n').nth(position.end.line - 1).unwrap_or_default();
+            let line = replaced.split('\n').nth(position.end.line as usize - 1).unwrap_or_default();
             // A column can be one past the end of its line, and no further.
-            prop_assert!(position.end.column <= line.len() + 1, "{node:?} in {replaced:?}");
+            prop_assert!(position.end.column as usize <= line.len() + 1, "{node:?} in {replaced:?}");
         }
     }
 }

crates/mq-markdown/tests/parser_structure_tests.rs

@@ -157,7 +157,7 @@ fn attributes(#[case] input: &str, #[case] name: &str, #[case] expected: &str) {
 #[case::escape_at_the_start("\\*a", 0, (1, 1), (1, 4))]
 #[case::escape_after_text("a\\*b", 0, (1, 1), (1, 5))]
 #[case::after_footnote("a[^1]\n\n[^1]: b", 1, (1, 2), (1, 6))]
-fn positions(#[case] input: &str, #[case] index: usize, #[case] start: (usize, usize), #[case] end: (usize, usize)) {
+fn positions(#[case] input: &str, #[case] index: usize, #[case] start: (u32, u32), #[case] end: (u32, u32)) {
     let nodes = parse(input);
     let position = nodes[index]
         .position()
@@ -179,17 +179,17 @@ fn positions(#[case] input: &str, #[case] index: usize, #[case] start: (usize, u
 fn table_rows_include_trailing_whitespace(#[case] input: &str) {
     for node in parse(input) {
         let position = node.position().unwrap();
-        let line = input.lines().nth(position.end.line - 1).unwrap();
-        assert!(position.end.column <= line.len() + 1, "{node:?}");
+        let line = input.lines().nth(position.end.line as usize - 1).unwrap();
+        assert!(position.end.column as usize <= line.len() + 1, "{node:?}");
     }
     let last_cell = parse(input)
         .into_iter()
         .rfind(|node| node.name() == "table_cell")
         .unwrap();
     let end = last_cell.position().unwrap().end;
     assert_eq!(
-        end.column,
-        input.lines().nth(end.line - 1).unwrap().len() + 1,
+        end.column as usize,
+        input.lines().nth(end.line as usize - 1).unwrap().len() + 1,
         "{last_cell:?}"
     );
 }

crates/mq-run/src/grep.rs

@@ -25,7 +25,7 @@ pub(crate) fn print_grep(
     let filename = file.as_ref().map(|p| p.to_string_lossy().into_owned());
 
     // Collect start lines of matched nodes for quick lookup.
-    let match_lines: HashSet<usize> = runtime_values
+    let match_lines: HashSet<u32> = runtime_values
         .values()
         .iter()
         .filter_map(|v| {
@@ -105,7 +105,7 @@ pub(crate) fn print_grep(
     handle.finish()
 }
 
-fn format_line(filename: &Option<String>, line_num: Option<usize>, content: &str, sep: &str) -> String {
+fn format_line(filename: &Option<String>, line_num: Option<u32>, content: &str, sep: &str) -> String {
     match (filename, line_num) {
         (Some(f), Some(l)) => format!("{}{}{}{}{}\n", f, sep, l, sep, content),
         (Some(f), None) => format!("{}{}{}\n", f, sep, content),
@@ -208,7 +208,7 @@ mod tests {
     #[case::no_filename_no_line(None, None, "plain", ":", "plain\n")]
     fn test_format_line(
         #[case] filename: Option<String>,
-        #[case] line_num: Option<usize>,
+        #[case] line_num: Option<u32>,
         #[case] content: &str,
         #[case] sep: &str,
         #[case] expected: &str,