✨ feat(markdown): add native Markdown parser

.github/workflows/gfm-spec.yml

@@ -1,4 +1,4 @@
-name: GFM spec round-trip fidelity
+name: Markdown spec conformance
 
 on:
   schedule:
@@ -21,5 +21,7 @@ jobs:
         with:
           just-version: 1.58.0
       - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
+      - name: Render the spec examples
+        run: just test-spec
       - name: Run GFM spec round-trip check
         run: just test-gfm-spec

.typos.toml

@@ -1,3 +1,7 @@
+[files]
+# HTML entity names from the HTML spec, which include short non-words
+extend-exclude = ["crates/mq-markdown/src/parser/inline/entities.rs"]
+
 [default.extend-words]
 # HashiCorp is a proper noun
 Hashi = "Hashi"
@@ -6,6 +10,8 @@ ist = "ist"
 # Intentional misspelling in the fuzzy-match-headings cookbook recipe
 instalation = "instalation"
 juxt = "juxt"
+# `import` with its first letter as a character reference (`import`), as MDX text is written
+mport = "mport"
 reakpoint = "reakpoint"
 sction = "sction"
 

Cargo.toml

@@ -80,7 +80,6 @@ itertools = "0.15.0"
 js-sys = "0.3.77"
 libc = "0.2"
 libfuzzer-sys = "0.4"
-markdown = "1.0.0"
 memchr = "2"
 nix = {version = "0.31", default-features = false, features = ["fs"]}
 md5 = "0.8.0"
@@ -128,6 +127,7 @@ tower-lsp-server = "0.23.0"
 tracing = "0.1.44"
 tracing-opentelemetry = "0.34"
 tracing-subscriber = "0.3.23"
+unicode-id = "0.3.6"
 unicode-normalization = "0.1.25"
 unicode-segmentation = "1.13.3"
 unicode-width = "0.2.2"

crates/mq-check/src/constraint.rs

@@ -149,7 +149,7 @@ pub fn generate_constraints(hir: &Hir, ctx: &mut InferenceContext) -> ChildrenIn
     // Pass 2.5: Process Assign symbols before other operators/calls.
     // This ensures that variable types are updated by assignments before
     // Refs and Calls in Pass 3 resolve against the (potentially stale) type.
-    // e.g., `var x = 10 | x = "hello" | upcase(x)` — the Assign updates
+    // e.g., `var x = 10 | x = "hello" | upcase(x)`: the Assign updates
     // x's type to String before upcase(x) resolves its argument type.
     for (symbol_id, kind) in &cats.assign_symbols {
         generate_symbol_constraints(hir, *symbol_id, kind.clone(), ctx, &children_index);
@@ -675,7 +675,7 @@ pub(super) fn generate_symbol_constraints(
                     ctx.set_symbol_type(symbol_id, last_ty);
                 }
             } else {
-                // No HIR resolution — try builtin registry by name as fallback
+                // No HIR resolution: try builtin registry by name as fallback
                 if let Some(symbol) = hir.symbol(symbol_id)
                     && let Some(name) = &symbol.value
                     && ctx.get_builtin_overloads(name.as_str()).is_some()
@@ -765,7 +765,7 @@ pub(super) fn generate_symbol_constraints(
                                 },
                             ));
 
-                            // No need to re-bind subsequent Refs — Assigns are processed
+                            // No need to re-bind subsequent Refs: Assigns are processed
                             // in Pass 2.5 (before Refs/Calls in Pass 3), so Refs will
                             // pick up the updated variable type when they are processed.
                         } else {
@@ -912,7 +912,7 @@ pub(super) fn generate_symbol_constraints(
                         return;
                     }
 
-                    // Homogeneous or unresolved — unify all element types
+                    // Homogeneous or unresolved: unify all element types
                     let elem_ty = elem_tys[0].clone();
                     let range = get_symbol_range(hir, symbol_id);
                     for ty in &elem_tys[1..] {
@@ -1037,7 +1037,7 @@ pub(super) fn generate_symbol_constraints(
             if !children.is_empty() {
                 let range = get_symbol_range(hir, symbol_id);
 
-                // First child is the condition — mq is dynamically typed, so any
+                // First child is the condition: mq is dynamically typed, so any
                 // value can be used as a condition (truthy/falsy), not just Bool.
                 let _cond_ty = ctx.get_or_create_symbol_type(children[0]);
 
@@ -1080,7 +1080,7 @@ pub(super) fn generate_symbol_constraints(
                         true
                     };
 
-                    // Check if any branch is None or a Union containing None — in mq,
+                    // Check if any branch is None or a Union containing None: in mq,
                     // `if (...): value else: None` (or a while-loop branch that returns
                     // `Union(T, None)`) should not unify `value` with None; instead treat
                     // as different types so the overall if-expression preserves the None
@@ -1129,15 +1129,15 @@ pub(super) fn generate_symbol_constraints(
                         || (has_none_branch && resolved.len() >= 2)
                         || has_parameter_branch
                     {
-                        // Different concrete types across branches — use Union type.
+                        // Different concrete types across branches: use Union type.
                         // Include ALL resolved branch types (vars and concrete) so that
                         // branches whose types are not yet fully resolved can still be
                         // tracked and resolved later (e.g., `if (...): items else: None`
                         // where `items` is still a Var at constraint-generation time).
                         let union_ty = Type::union(resolved.clone());
                         ctx.set_symbol_type(symbol_id, union_ty);
                     } else {
-                        // Homogeneous or unresolved — unify all branch types
+                        // Homogeneous or unresolved: unify all branch types
                         ctx.set_symbol_type(symbol_id, then_ty.clone());
                         for ty in &branch_tys[1..] {
                             ctx.add_constraint(Constraint::Equal(
@@ -1446,12 +1446,12 @@ pub(super) fn generate_symbol_constraints(
                     };
 
                     if !all_same && concrete.len() >= 2 {
-                        // Different concrete types in arms — use Union type
+                        // Different concrete types in arms: use Union type
                         let unique_types: Vec<Type> = concrete.into_iter().cloned().collect();
                         let union_ty = Type::union(unique_types);
                         ctx.set_symbol_type(symbol_id, union_ty);
                     } else if concrete.len() == 1 && !var_tys.is_empty() {
-                        // One concrete arm (e.g. `None`) plus still-unresolved arms — union
+                        // One concrete arm (e.g. `None`) plus still-unresolved arms: union
                         // them instead of unifying, or the unresolved type collapses to the
                         // concrete one. Mirrors `merge_loop_types`.
                         let mut unified_var = var_tys[0].clone();
@@ -1535,7 +1535,7 @@ pub(super) fn generate_symbol_constraints(
                     ));
                     ctx.set_symbol_type(symbol_id, try_ty);
                 } else {
-                    // A branch type is still pending deferred resolution — decide later.
+                    // A branch type is still pending deferred resolution: decide later.
                     let ty_var = ctx.fresh_var();
                     ctx.set_symbol_type(symbol_id, Type::Var(ty_var));
                     ctx.add_deferred_try_catch(infer::DeferredTryCatch {
@@ -1638,7 +1638,7 @@ pub(super) fn generate_symbol_constraints(
                         Type::Var(ty_var)
                     }
                 } else {
-                    // Piped type not yet resolved — defer to post-unification.
+                    // Piped type not yet resolved: defer to post-unification.
                     // Keep the selector so that if the piped type resolves to a Markdown node,
                     // the right type can be returned (e.g., `md.depth` → number).
                     let ty_var = ctx.fresh_var();
@@ -1741,7 +1741,7 @@ pub(super) fn generate_symbol_constraints(
             }
         }
 
-        // Block: pipe chain — compute type from children in pass 3.
+        // Block: pipe chain: compute type from children in pass 3.
         // Block has higher insertion order than its parent (If/Else/etc.), so in
         // the reverse-order pass 3 loop it is processed before its parent.  This
         // means the If handler can see the Block's real type (e.g. Union(T, None)
@@ -1926,7 +1926,7 @@ fn generate_call_constraints(
 
                         let real_children = &children[..children.len() - trailing_bracket_count];
                         let arg_symbol_ids = if prepend_piped {
-                            // piped input was prepended — include a placeholder
+                            // piped input was prepended: include a placeholder
                             let mut ids = vec![symbol_id]; // placeholder for piped arg
                             ids.extend_from_slice(real_children);
                             ids
@@ -2037,7 +2037,7 @@ fn generate_call_constraints(
                                     // Only constrain the index to Number when it is a literal
                                     // numeric index (e.g. v[0]). For variable indices
                                     // (e.g. a String key used for Dict access), skip this
-                                    // constraint — the correct element type is resolved via
+                                    // constraint: the correct element type is resolved via
                                     // resolve_deferred_tuple_accesses once the container's
                                     // type is known.
                                     if literal_index.is_some()

crates/mq-help/src/docs/table.rs

@@ -3640,7 +3640,7 @@ x = 1
     BuiltinDoc {
         name: "to_h",
         aliases: &[],
-        description: "Creates a markdown heading node with the given value and depth.",
+        description: "Creates a markdown heading node with the given value and depth. A depth outside of 1 to 6 becomes the nearest of them.",
         params: &["value", "depth"],
         param_types: &["dynamic", "number"],
         returns: "markdown",

crates/mq-lang/modules/section_test.mq

@@ -387,8 +387,3 @@ end
 def test_section_toc_markdown_empty():
   assert_eq(section::toc_markdown([]), [])
 end
-
-def test_section_sections_invalid_heading_depth():
-  let secs = section::sections([to_h("A", 7)])
-  | assert(is_none(secs))
-end

crates/mq-lang/src/lib.rs

@@ -212,7 +212,12 @@ pub(crate) fn parse_in_module(
 
 /// Parses an MDX string and returns an iterator over `Value` nodes.
 pub fn parse_mdx_input(input: &str) -> miette::Result<Vec<RuntimeValue>> {
-    let mdx = mq_markdown::Markdown::from_mdx_str(input)?;
+    parse_mdx_input_with(input, mq_markdown::ParseOptions::default())
+}
+
+/// Parses an MDX string with the given options and returns an iterator over `Value` nodes.
+pub fn parse_mdx_input_with(input: &str, options: mq_markdown::ParseOptions) -> miette::Result<Vec<RuntimeValue>> {
+    let mdx = mq_markdown::Markdown::from_mdx_str_with(input, options)?;
     Ok(mdx.nodes.into_iter().map(RuntimeValue::from).collect())
 }
 
@@ -233,7 +238,12 @@ pub fn parse_html_input_with_options(
 
 /// Parses a Markdown string and returns an iterator over `Value` nodes.
 pub fn parse_markdown_input(input: &str) -> miette::Result<Vec<RuntimeValue>> {
-    let md = mq_markdown::Markdown::from_markdown_str(input)?;
+    parse_markdown_input_with(input, mq_markdown::ParseOptions::default())
+}
+
+/// Parses a Markdown string with the given options and returns an iterator over `Value` nodes.
+pub fn parse_markdown_input_with(input: &str, options: mq_markdown::ParseOptions) -> miette::Result<Vec<RuntimeValue>> {
+    let md = mq_markdown::Markdown::from_markdown_str_with(input, options)?;
     Ok(md.nodes.into_iter().map(RuntimeValue::from).collect())
 }