From a166f813233cf509d6d8fdfc2aaf4e43153a4ed6 Mon Sep 17 00:00:00 2001 From: harehare Date: Tue, 27 Oct 2026 21:12:50 +0900 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20fix(lang):=20skip=20leading=20BO?= =?UTF-8?q?M=20in=20query=20and=20module=20sources?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- crates/mq-lang/src/lexer.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/crates/mq-lang/src/lexer.rs b/crates/mq-lang/src/lexer.rs index 26a90e264..4d543a13e 100644 --- a/crates/mq-lang/src/lexer.rs +++ b/crates/mq-lang/src/lexer.rs @@ -32,6 +32,7 @@ impl Lexer { /// Like [`Lexer::tokenize`], but token ranges are reported as if `input` began at `start`. pub fn tokenize_from(&self, input: &str, module_id: ModuleId, start: Position) -> Result, SyntaxError> { + let input = input.strip_prefix('\u{FEFF}').unwrap_or(input); let mut cursor = Cursor::new(input, module_id); cursor.line = start.line; cursor.col = start.column; @@ -921,6 +922,16 @@ mod tests { assert_eq!(tokens[1].range.start, Position { line: 4, column: 2 }); } + #[rstest] + #[case(Options::default())] + #[case(Options { include_spaces: true, ..Options::default() })] + fn tokenize_skips_leading_bom(#[case] options: Options) { + let lexer = Lexer::new(options); + let with_bom = lexer.tokenize("\u{FEFF}upcase()", 1.into()).unwrap(); + let without_bom = lexer.tokenize("upcase()", 1.into()).unwrap(); + assert_eq!(with_bom, without_bom); + } + #[rstest] #[case("and(contains(\"test\"))", Options::default(),