From 77feb295fba1efd8660e837bf187549176d3cb0a Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Thu, 10 Sep 2026 21:37:53 +0900 Subject: [PATCH 001/157] Replace Prism action lexing with a Ruby scanner --- arpaka.gemspec | 1 - lib/lrama/ruby.rb | 6 +- lib/lrama/ruby/action_code.rb | 59 +++--- lib/lrama/ruby/action_scanner.rb | 312 ++++++++++++++++++++++++++++++ lib/lrama/ruby/backend.rb | 9 +- lib/lrama/ruby/generator.rb | 1 - test/lrama/action_scanner_test.rb | 162 ++++++++++++++++ 7 files changed, 507 insertions(+), 43 deletions(-) create mode 100644 lib/lrama/ruby/action_scanner.rb create mode 100644 test/lrama/action_scanner_test.rb diff --git a/arpaka.gemspec b/arpaka.gemspec index ff928b4..a012033 100644 --- a/arpaka.gemspec +++ b/arpaka.gemspec @@ -28,5 +28,4 @@ Gem::Specification.new do |spec| spec.require_paths = ["lib"] spec.add_dependency "lrama", ">= 0.8.0" - spec.add_dependency "prism", "~> 1.0" end diff --git a/lib/lrama/ruby.rb b/lib/lrama/ruby.rb index a2e4b0f..5e76003 100644 --- a/lib/lrama/ruby.rb +++ b/lib/lrama/ruby.rb @@ -22,7 +22,11 @@ def self.compile(source, filename: "(grammar)", class_name: "Parser", mode: :par code = generate(source, filename: filename, class_name: class_name, mode: mode, allow_error_rules: allow_error_rules) box = ::Ruby::Box.new - box.eval(code) + begin + box.eval(code) + rescue SyntaxError => error + raise Error, "Invalid generated Ruby for #{filename}: #{error.message}" + end box.const_get(class_name, false) end end diff --git a/lib/lrama/ruby/action_code.rb b/lib/lrama/ruby/action_code.rb index 3295b60..a670c86 100644 --- a/lib/lrama/ruby/action_code.rb +++ b/lib/lrama/ruby/action_code.rb @@ -1,31 +1,34 @@ # frozen_string_literal: true -require "prism" +require_relative "action_scanner" module Lrama module Ruby # Loaded only in the generator's box. Ruby strings, comments, regular # expressions and instance variables must not become grammar references. module ActionReferences + def ruby_short_reference?(offset) + references + @ruby_short_references.include?(offset) + end + private def _references - Prism.lex(s_value).value.filter_map do |token, _state| - next unless [:GLOBAL_VARIABLE, :NUMBERED_REFERENCE].include?(token.type) - - value = token.value - unless /\A\$(?:\$|[1-9][0-9]*|[a-zA-Z_][a-zA-Z0-9_]*)\z/.match?(value) - raise Backend::Error, "Unsupported semantic reference: #{value}" - end - - attributes = if /\A\$[0-9]+\z/.match?(value) - { number: value.delete_prefix("$").to_i, index: value.delete_prefix("$").to_i } + result = ActionScanner.new(s_value, filename: location.grammar_file.path, + line: location.first_line, column: location.first_column).scan + @ruby_short_references = result.references.select(&:short_interpolation).map(&:first_column) + result.references.map do |reference| + attributes = if reference.number + { number: reference.number, index: reference.number } else - { name: value.delete_prefix("$") } + { name: reference.name } end Grammar::Reference.new(type: :dollar, **attributes, - first_column: token.location.start_offset, last_column: token.location.end_offset) - end.sort_by(&:first_column) + first_column: reference.first_column, last_column: reference.last_column) + end + rescue ActionScanner::Error => error + raise Backend::Error, error.message end end @@ -34,25 +37,15 @@ def lex_c_code return super unless @end_symbol == "}" reset_first_position - depth = 0 - # Prism recovers from the yacc references (which are not yet valid - # Ruby assignments) and still provides byte-accurate token locations. - Prism.lex(@scanner.rest).value.each do |token, _state| - case token.type - when :BRACE_LEFT, :LAMBDA_BEGIN, :EMBEXPR_BEGIN - depth += 1 - when :BRACE_RIGHT, :EMBEXPR_END - if depth.zero? - code = @scanner.rest.byteslice(0, token.location.start_offset) - @scanner.pos += code.bytesize - @line += code.count("\n") - @head = @scanner.pos - code.bytesize + code.b.rindex("\n") + 1 if code.include?("\n") - return [:C_DECLARATION, Lexer::Token::UserCode.new(s_value: code, location: location)] - end - depth -= 1 - end - end - raise Backend::Error, "Unterminated Ruby action at line #{line}" + result = ActionScanner.new(@scanner.rest, filename: @grammar_file.path, + line: line, column: column).scan(action: true) + code = @scanner.string.byteslice(@scanner.pos, result.end_offset) + @scanner.pos += code.bytesize + @line += code.count("\n") + @head = @scanner.pos - code.bytesize + code.b.rindex("\n") + 1 if code.include?("\n") + [:C_DECLARATION, Lexer::Token::UserCode.new(s_value: code, location: location)] + rescue ActionScanner::Error => error + raise Backend::Error, error.message end end end diff --git a/lib/lrama/ruby/action_scanner.rb b/lib/lrama/ruby/action_scanner.rb new file mode 100644 index 0000000..957ab2b --- /dev/null +++ b/lib/lrama/ruby/action_scanner.rb @@ -0,0 +1,312 @@ +# frozen_string_literal: true + +require "strscan" + +module Lrama + module Ruby + # A lexer for the supported Ruby action dialect, not a Ruby syntax checker. + # All positions are bytes in the original source; literals are never decoded. + class ActionScanner + Reference = Data.define(:name, :number, :first_column, :last_column, :short_interpolation) + Result = Data.define(:end_offset, :references) + Heredoc = Data.define(:delimiter, :indent, :interpolate, :offset) + class Error < StandardError; end + + IDENTIFIER = /[a-zA-Z_\x80-\xff][a-zA-Z_0-9\x80-\xff]*[!?]?/n + NUMBER = /(?:0[xX][0-9a-fA-F_]+|0[bB][01_]+|0[oO][0-7_]+|[0-9][0-9_]*(?:\.[0-9_]+)?(?:[eE][+-]?[0-9_]+)?)[ri]*/ + BEGIN_WORDS = %w[if unless while until when return yield break next rescue else elsif then do begin and or not in case for].freeze + END_WORDS = %w[nil true false self end __FILE__ __LINE__ __ENCODING__].freeze + PAIRS = { "(" => ")", "[" => "]", "{" => "}" }.freeze + LITERAL_PAIRS = PAIRS.merge("<" => ">").freeze + # Bound recursive interpolation rather than allowing SystemStackError. + MAX_INTERPOLATION_DEPTH = 128 + + def initialize(source, filename: "(grammar)", line: 1, column: 0) + @source = source.b + @scanner = StringScanner.new(@source) + @filename, @line, @column = filename, line, column + @references = [] + @interpolation_depth = 0 + end + + def scan(action: false) + code(action ? "}" : nil) + Result.new(@scanner.pos, @references.sort_by(&:first_column).freeze) + end + + private + + def fail_at(message, offset = @scanner.pos) + prefix = @source.byteslice(0, offset) + lines = prefix.count("\n") + column = lines.zero? ? @column + offset : offset - prefix.rindex("\n") - 1 + raise Error, "#{@filename}:#{@line + lines}:#{column}: #{message}" + end + + def code(terminator) + brackets = [] + heredocs = [] + state = :begin + spaced = false + until @scanner.eos? + offset = @scanner.pos + char = @scanner.peek(1) + if @scanner.scan(/[ \t\r\f]+/) + spaced = true + next + elsif @scanner.scan(/\\\r?\n/) + spaced = true + next + elsif @scanner.scan(/\n/) + heredocs.each { |item| heredoc_body(item) } + heredocs.clear + state = :begin unless [:begin, :method].include?(state) + spaced = true + next + elsif char == "#" + @scanner.scan(/[^\n]*/) + next + elsif (offset.zero? || @source.getbyte(offset - 1) == 10) && @scanner.check(/=begin(?=\s|\z)/) + @scanner.scan(/[^\n]*(?:\n|\z)/) + found = false + until @scanner.eos? + line = @scanner.scan(/[^\n]*(?:\n|\z)/) + if /\A=end(?:\s|\z)/.match?(line) + found = true + break + end + end + fail_at("Unterminated block comment", offset) unless found + next + elsif char == terminator && brackets.empty? + fail_at("Close the action/interpolation after the heredoc body", offset) unless heredocs.empty? + return + elsif PAIRS.key?(char) + brackets << [PAIRS.fetch(char), offset] + @scanner.getch + state = :begin + elsif [")", "]", "}"].include?(char) + fail_at("Mismatched closing #{char}", offset) unless brackets.last&.first == char + brackets.pop + @scanner.getch + state = :end + elsif ["'", '"', "`"].include?(char) + @scanner.getch + literal(char, interpolate: char != "'", offset: offset) + state = :end + elsif char == "$" + reference + state = :end + elsif @scanner.scan(/@@?[a-zA-Z_\x80-\xff][a-zA-Z_0-9\x80-\xff]*/n) + state = :end + elsif @scanner.check(NUMBER) + @scanner.scan(NUMBER) + state = :end + elsif @scanner.check(IDENTIFIER) + word = @scanner.scan(IDENTIFIER) + fail_at("__END__ is not supported in Ruby actions", offset) if word == "__END__" + # Method names after dot/:: are not keywords. + state = if state == :method + :bare + elsif BEGIN_WORDS.include?(word) + :begin + elsif END_WORDS.include?(word) + :end + elsif ["def", "alias", "undef"].include?(word) + fail_at("#{word} is not supported in Ruby actions", offset) + else + :bare + end + elsif char == "/" || char == "%" || @scanner.check(/<|===|==|=~|!~|!=|<=|>=|<<|>>|\*\*|[+\-~]@?|[!*\/%&|^<>`])/) + fail_at("Unsupported symbol literal", offset) + end + state = :end + elsif @scanner.scan(/(?:\.\.\.?|->|\*\*=|&&=?|\|\|=?|<=>|===|==|=>|!=|!~|=~|<=|>=|>>=?|\*\*|[+\-*|&^]=|[=+\-*!,;:<>?~|&^])/) + state = :begin + else + fail_at("Unsupported Ruby action token #{char.inspect}", offset) + end + spaced = false + end + fail_at("Unterminated heredoc", heredocs.first.offset) unless heredocs.empty? + fail_at("Unclosed #{brackets.last.first}", brackets.last.last) unless brackets.empty? + fail_at("Unterminated Ruby action/interpolation") if terminator + end + + def reference(short: false) + offset = @scanner.pos + value = @scanner.scan(/\$(?:\$|[0-9]+|[a-zA-Z_][a-zA-Z0-9_]*)/) + if !value || value.match?(/\A\$0/) || @scanner.check(/[a-zA-Z_0-9\x80-\xff]/n) + fail_at("Unsupported semantic reference", offset) + end + number = value.match?(/\A\$[0-9]+\z/) ? value.delete_prefix("$").to_i : nil + @references << Reference.new(number ? nil : value.delete_prefix("$"), number, + offset, @scanner.pos, short) + end + + def interpolation + return false unless @scanner.peek(1) == "#" + if @scanner.scan(/#\{/) + @interpolation_depth += 1 + fail_at("Interpolation nesting exceeds #{MAX_INTERPOLATION_DEPTH}") if @interpolation_depth > MAX_INTERPOLATION_DEPTH + code("}") + @scanner.getch + @interpolation_depth -= 1 + elsif @scanner.scan(/#(?=\$)/) + reference(short: true) + elsif @scanner.scan(/#@@?[a-zA-Z_\x80-\xff][a-zA-Z_0-9\x80-\xff]*/n) + # Instance/class variable interpolation is preserved verbatim. + else + return false + end + true + end + + def literal(close, interpolate:, offset:, open: nil, regexp: false) + depth = 0 + until @scanner.eos? + char = @scanner.peek(1) + if char == "\\" + @scanner.getch + @scanner.getch || fail_at("Unterminated escape", offset) + elsif interpolate && interpolation + next + elsif regexp && char == "[" + regexp_class(offset) + elsif char == close + @scanner.getch + return if depth.zero? + depth -= 1 + elsif open && char == open + depth += 1 + @scanner.getch + else + @scanner.getch + end + end + fail_at("Unterminated literal (expected #{close})", offset) + end + + def regexp_class(offset) + @scanner.getch + @scanner.scan(/\^/) + @scanner.scan(/\]/) # A leading ] is literal. + until @scanner.eos? + if @scanner.scan(/\\[\s\S]/) + next + elsif interpolation + next + elsif @scanner.scan(/\[:[^\]\n]*:\]/) + next # POSIX character class inside a bracket expression. + elsif @scanner.scan(/\]/) + return + else + @scanner.getch + end + end + fail_at("Unterminated regexp character class", offset) + end + + def percent_literal(offset) + @scanner.getch + kind = @scanner.scan(/[qQwWiIrsx]/) || "Q" + delimiter = @scanner.getch + unless delimiter && /\A[!-~]\z/.match?(delimiter) && !/[a-zA-Z0-9\\]/.match?(delimiter) + fail_at("Unsupported percent literal delimiter", offset) + end + literal(LITERAL_PAIRS.fetch(delimiter, delimiter), open: LITERAL_PAIRS.key?(delimiter) ? delimiter : nil, + interpolate: !%w[q w i s].include?(kind), regexp: kind == "r", offset: offset) + @scanner.scan(/[a-z]*/) if kind == "r" + end + + def heredoc_start(offset) + @scanner.scan(/<", "%Q!\#{ $1 }!"].each do |source| + assert_equal(["$1"], reference_texts(source)) + end + end + + test "regexp classes escapes and interpolation preserve action boundaries" do + sources = [ + '/[}$]/', '/[[:alpha:]}]/', '/[]}$]/', '/[\]}/]/', + '%r{[}]\#{ignored}}', '/foo\/bar}/', '/\#{ { a: $1 } }/' + ] + sources.each do |source| + result = scan(source + " } next", action: true) + assert_equal(source.bytesize + 1, result.end_offset, source) + end + assert_equal(["$1"], reference_texts('/#{ { a: $1 } }/')) + assert_equal(["$1"], reference_texts('%r!#{ $1 }!im')) + end + + test "division modulo shifts and explicit calls are supported" do + ["$1 / $2", "$1 % $2", "$1 << $2", "(value) / 2", "(value) % 2", "(value) << 2", + "call(%r!}! )", "obj./(2)", "obj.%(2)", "obj.<<(2)"].each do |source| + assert_nothing_raised(source) { scan(source) } + end + end + + test "bare command ambiguity is rejected rather than guessed" do + ["call /}/", "value / 2", "value % 2", "value << 2", "puts <(x) { { x: $2 } }] + RUBY + assert_equal(["$1", "$2"], reference_texts(source)) + end + + test "unsupported references fail in code and interpolation only" do + %w[$0 $01 $& $~ $? $! $-w $1 $[name] $1name $:].each do |ref| + assert_raise(Scanner::Error, ref) { scan(ref) } + assert_raise(Scanner::Error, ref) { scan('"#{' + ref + '}"') } + assert_empty(scan("'#{ref}'").references) + end + end + + test "unclosed constructs and mismatched brackets report source positions" do + ["'text", '"text', '%q{text', '/[abc/', '"#{ $1', "([)]", "(1", "$$ = < Date: Thu, 10 Sep 2026 21:46:38 +0900 Subject: [PATCH 002/157] Expand action scanner to general Ruby source --- lib/lrama/ruby/action_scanner.rb | 7 +------ test/lrama/action_scanner_test.rb | 15 ++++++++++----- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/lib/lrama/ruby/action_scanner.rb b/lib/lrama/ruby/action_scanner.rb index 957ab2b..ab180ac 100644 --- a/lib/lrama/ruby/action_scanner.rb +++ b/lib/lrama/ruby/action_scanner.rb @@ -4,7 +4,7 @@ module Lrama module Ruby - # A lexer for the supported Ruby action dialect, not a Ruby syntax checker. + # A Ruby source lexer used for action boundary detection and references. # All positions are bytes in the original source; literals are never decoded. class ActionScanner Reference = Data.define(:name, :number, :first_column, :last_column, :short_interpolation) @@ -112,15 +112,10 @@ def code(terminator) :begin elsif END_WORDS.include?(word) :end - elsif ["def", "alias", "undef"].include?(word) - fail_at("#{word} is not supported in Ruby actions", offset) else :bare end elsif char == "/" || char == "%" || @scanner.check(/< Date: Thu, 10 Sep 2026 21:51:08 +0900 Subject: [PATCH 003/157] Add local Rails lexer health snapshot --- tool/ruby/rails_snapshot.rb | 44 +++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 tool/ruby/rails_snapshot.rb diff --git a/tool/ruby/rails_snapshot.rb b/tool/ruby/rails_snapshot.rb new file mode 100644 index 0000000..3cd0baf --- /dev/null +++ b/tool/ruby/rails_snapshot.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +# Local-only health snapshot for the Rails checkout. This intentionally uses +# RubyVM::AbstractSyntaxTree as an oracle; it is not part of Arpaka's runtime. + +require "json" +require "lrama/ruby/action_scanner" + +rails = ARGV[0] || [ + File.expand_path("~/Program/Github/rails"), + File.expand_path("~/Program/Ruby/rails") +].find { |path| File.directory?(path) } +abort "Rails checkout not found" unless rails + +files = Dir.glob(File.join(rails, "**/*.rb"), File::FNM_EXTGLOB).sort +snapshot = { + "rails" => rails, + "files" => files.length, + "scanner_failures" => [], + "ast_failures" => [], + "scanner_successes" => 0, + "ast_successes" => 0 +} + +files.each do |path| + source = File.binread(path) + relative = path.delete_prefix("#{rails}/") + + begin + Lrama::Ruby::ActionScanner.new(source, filename: relative).scan + snapshot["scanner_successes"] += 1 + rescue Lrama::Ruby::ActionScanner::Error => error + snapshot["scanner_failures"] << {"file" => relative, "error" => error.message} + end + + begin + RubyVM::AbstractSyntaxTree.parse(source) + snapshot["ast_successes"] += 1 + rescue SyntaxError => error + snapshot["ast_failures"] << {"file" => relative, "error" => error.message.lines.first&.chomp} + end +end + +puts JSON.pretty_generate(snapshot) From b30fd94bb814220f5c01d3393303634ff6aad902 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Thu, 10 Sep 2026 21:58:53 +0900 Subject: [PATCH 004/157] Improve Ruby source scanning for Rails --- lib/lrama/ruby/action_scanner.rb | 19 +++++++++++++++++-- test/lrama/action_scanner_test.rb | 6 +++--- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/lib/lrama/ruby/action_scanner.rb b/lib/lrama/ruby/action_scanner.rb index ab180ac..c65ee63 100644 --- a/lib/lrama/ruby/action_scanner.rb +++ b/lib/lrama/ruby/action_scanner.rb @@ -27,9 +27,11 @@ def initialize(source, filename: "(grammar)", line: 1, column: 0) @filename, @line, @column = filename, line, column @references = [] @interpolation_depth = 0 + @strict_references = false end def scan(action: false) + @strict_references = action code(action ? "}" : nil) Result.new(@scanner.pos, @references.sort_by(&:first_column).freeze) end @@ -145,7 +147,8 @@ def code(terminator) if @scanner.check(/['"]/) quote = @scanner.getch literal(quote, interpolate: quote != "'", offset: offset) - elsif !@scanner.scan(IDENTIFIER) && !@scanner.scan(/(?:\[\]=?|<=>|===|==|=~|!~|!=|<=|>=|<<|>>|\*\*|[+\-~]@?|[!*\/%&|^<>`])/) + elsif !@scanner.scan(/[@$]?[a-zA-Z_\x80-\xff][a-zA-Z_0-9\x80-\xff]*[!?]?/n) && + !@scanner.scan(/(?:\[\]=?|<=>|===|==|=~|!~|!=|<=|>=|<<|>>|\*\*|[+\-~]@?|[!*\/%&|^<>`])/) fail_at("Unsupported symbol literal", offset) end state = :end @@ -163,8 +166,14 @@ def code(terminator) def reference(short: false) offset = @scanner.pos + @global_variable_start = offset value = @scanner.scan(/\$(?:\$|[0-9]+|[a-zA-Z_][a-zA-Z0-9_]*)/) - if !value || value.match?(/\A\$0/) || @scanner.check(/[a-zA-Z_0-9\x80-\xff]/n) + unless value + return scan_global_variable unless @strict_references + fail_at("Unsupported semantic reference", offset) + end + if value.match?(/\A\$0/) || @scanner.check(/[a-zA-Z_0-9\x80-\xff]/n) + return scan_global_variable unless @strict_references fail_at("Unsupported semantic reference", offset) end number = value.match?(/\A\$[0-9]+\z/) ? value.delete_prefix("$").to_i : nil @@ -172,6 +181,12 @@ def reference(short: false) offset, @scanner.pos, short) end + def scan_global_variable + @scanner.pos = @global_variable_start || @scanner.pos + @scanner.scan(/\$(?:[<>]?|[!@&`'++~?=\/\\;,.:$-][a-zA-Z]?|[<>][^>\n]*>|[0-9]+|[a-zA-Z_][a-zA-Z0-9_]*)/) + fail_at("Unsupported global variable", @scanner.pos) if @scanner.pos == (@global_variable_start || @scanner.pos) + end + def interpolation return false unless @scanner.peek(1) == "#" if @scanner.scan(/#\{/) diff --git a/test/lrama/action_scanner_test.rb b/test/lrama/action_scanner_test.rb index a32f3e7..3aea135 100644 --- a/test/lrama/action_scanner_test.rb +++ b/test/lrama/action_scanner_test.rb @@ -137,8 +137,8 @@ def reference_texts(source) test "unsupported references fail in code and interpolation only" do %w[$0 $01 $& $~ $? $! $-w $1 $[name] $1name $:].each do |ref| - assert_raise(Scanner::Error, ref) { scan(ref) } - assert_raise(Scanner::Error, ref) { scan('"#{' + ref + '}"') } + assert_raise(Scanner::Error, ref) { scan(ref + " }", action: true) } + assert_raise(Scanner::Error, ref) { scan('"#{' + ref + '}" }', action: true) } assert_empty(scan("'#{ref}'").references) end end @@ -150,7 +150,7 @@ def reference_texts(source) assert_include(error.message, "(grammar):") end error = assert_raise(Scanner::Error) do - Scanner.new("\"日本語\"\r\n$0", filename: "test.y", line: 10, column: 5).scan + Scanner.new("\"日本語\"\r\n$0 }", filename: "test.y", line: 10, column: 5).scan(action: true) end assert_include(error.message, "test.y:11:0:") assert_raise(Scanner::Error) { scan("$$ = 1", action: true) } From 869f9f14be6c3cb39ba700f580fe993cf3af9476 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Thu, 10 Sep 2026 22:15:14 +0900 Subject: [PATCH 005/157] Handle keyword argument labels in Ruby lexer --- lib/lrama/ruby/action_scanner.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/lrama/ruby/action_scanner.rb b/lib/lrama/ruby/action_scanner.rb index c65ee63..021f9cf 100644 --- a/lib/lrama/ruby/action_scanner.rb +++ b/lib/lrama/ruby/action_scanner.rb @@ -142,6 +142,9 @@ def code(terminator) state = :end elsif @scanner.scan(/&\.|::|\.(?!\.)/) state = :method + elsif char == ":" && state == :begin && @scanner.check(/:\s/) + @scanner.getch + state = :begin elsif char == ":" && state == :begin @scanner.getch if @scanner.check(/['"]/) From 9e1d5b5fda2d0d2362851ace8300fa9d0a7c8e35 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Thu, 10 Sep 2026 22:16:51 +0900 Subject: [PATCH 006/157] Recognize percent literals after calls --- lib/lrama/ruby/action_scanner.rb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/action_scanner.rb b/lib/lrama/ruby/action_scanner.rb index 021f9cf..65bbe2e 100644 --- a/lib/lrama/ruby/action_scanner.rb +++ b/lib/lrama/ruby/action_scanner.rb @@ -118,7 +118,10 @@ def code(terminator) :bare end elsif char == "/" || char == "%" || @scanner.check(/< Date: Thu, 10 Sep 2026 22:18:43 +0900 Subject: [PATCH 007/157] Support interpolated Ruby symbols --- lib/lrama/ruby/action_scanner.rb | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/action_scanner.rb b/lib/lrama/ruby/action_scanner.rb index 65bbe2e..aaaa5a3 100644 --- a/lib/lrama/ruby/action_scanner.rb +++ b/lib/lrama/ruby/action_scanner.rb @@ -153,7 +153,12 @@ def code(terminator) if @scanner.check(/['"]/) quote = @scanner.getch literal(quote, interpolate: quote != "'", offset: offset) - elsif !@scanner.scan(/[@$]?[a-zA-Z_\x80-\xff][a-zA-Z_0-9\x80-\xff]*[!?]?/n) && + elsif @scanner.scan(/#\{/) + @interpolation_depth += 1 + code("}") + @scanner.getch + @interpolation_depth -= 1 + elsif !@scanner.scan(/(?:@{1,2}|\$?)[a-zA-Z_\x80-\xff][a-zA-Z_0-9\x80-\xff]*[!?]?/n) && !@scanner.scan(/(?:\[\]=?|<=>|===|==|=~|!~|!=|<=|>=|<<|>>|\*\*|[+\-~]@?|[!*\/%&|^<>`])/) fail_at("Unsupported symbol literal", offset) end From 3711dc4c2eab5d91bcebdff77d2aaa4ead008a71 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Thu, 10 Sep 2026 22:20:21 +0900 Subject: [PATCH 008/157] Distinguish shifts from heredocs --- lib/lrama/ruby/action_scanner.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/action_scanner.rb b/lib/lrama/ruby/action_scanner.rb index aaaa5a3..d1b91e6 100644 --- a/lib/lrama/ruby/action_scanner.rb +++ b/lib/lrama/ruby/action_scanner.rb @@ -131,8 +131,10 @@ def code(terminator) @scanner.scan(/[a-z]*/) elsif char == "%" percent_literal(offset) - else + elsif @scanner.check(/<<[-~]?(?:['"`][^\r\n]+['"`]|[a-zA-Z_][a-zA-Z_0-9]*)/n) heredocs << heredoc_start(offset) + else + @scanner.scan(/< Date: Thu, 10 Sep 2026 22:22:08 +0900 Subject: [PATCH 009/157] Handle operator methods and call-site heredocs --- lib/lrama/ruby/action_scanner.rb | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/lrama/ruby/action_scanner.rb b/lib/lrama/ruby/action_scanner.rb index d1b91e6..8390bac 100644 --- a/lib/lrama/ruby/action_scanner.rb +++ b/lib/lrama/ruby/action_scanner.rb @@ -121,6 +121,9 @@ def code(terminator) if char == "%" && @scanner.check(/%[qQwWiIrsx]?[!-~]/n) percent_literal(offset) state = :end + elsif @scanner.check(/<<[-~]?(?:['"`][^\r\n]+['"`]|[a-zA-Z_][a-zA-Z_0-9]*)/n) + heredocs << heredoc_start(offset) + state = :end elsif state == :method @scanner.scan(/(?:<<|\/|%)/) state = :bare @@ -165,6 +168,9 @@ def code(terminator) fail_at("Unsupported symbol literal", offset) end state = :end + elsif char == "@" && @scanner.check(/@@?[a-zA-Z_\x80-\xff][a-zA-Z_0-9\x80-\xff]*/n) + @scanner.scan(/@@?[a-zA-Z_\x80-\xff][a-zA-Z_0-9\x80-\xff]*/n) + state = :end elsif @scanner.scan(/(?:\.\.\.?|->|\*\*=|&&=?|\|\|=?|<=>|===|==|=>|!=|!~|=~|<=|>=|>>=?|\*\*|[+\-*|&^]=|[=+\-*!,;:<>?~|&^])/) state = :begin else From 42bda35c06839c15f0034afe77f63eeb625b1deb Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Thu, 10 Sep 2026 22:23:51 +0900 Subject: [PATCH 010/157] Support Ruby operator globals and backreferences --- lib/lrama/ruby/action_scanner.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/action_scanner.rb b/lib/lrama/ruby/action_scanner.rb index 8390bac..8b3a170 100644 --- a/lib/lrama/ruby/action_scanner.rb +++ b/lib/lrama/ruby/action_scanner.rb @@ -171,6 +171,9 @@ def code(terminator) elsif char == "@" && @scanner.check(/@@?[a-zA-Z_\x80-\xff][a-zA-Z_0-9\x80-\xff]*/n) @scanner.scan(/@@?[a-zA-Z_\x80-\xff][a-zA-Z_0-9\x80-\xff]*/n) state = :end + elsif char == "@" && state == :begin + @scanner.getch + state = :end elsif @scanner.scan(/(?:\.\.\.?|->|\*\*=|&&=?|\|\|=?|<=>|===|==|=>|!=|!~|=~|<=|>=|>>=?|\*\*|[+\-*|&^]=|[=+\-*!,;:<>?~|&^])/) state = :begin else @@ -202,7 +205,8 @@ def reference(short: false) def scan_global_variable @scanner.pos = @global_variable_start || @scanner.pos - @scanner.scan(/\$(?:[<>]?|[!@&`'++~?=\/\\;,.:$-][a-zA-Z]?|[<>][^>\n]*>|[0-9]+|[a-zA-Z_][a-zA-Z0-9_]*)/) + @scanner.scan(/\$(?:`|')/) || + @scanner.scan(/\$(?:[<>]?|[!@&+~?=\/\\;,.:$-][a-zA-Z]?|[<>][^>\n]*>|[0-9]+|[a-zA-Z_][a-zA-Z0-9_]*)/) fail_at("Unsupported global variable", @scanner.pos) if @scanner.pos == (@global_variable_start || @scanner.pos) end From aa2498829ba438c8ba053d4ca0c8890ff835c121 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Thu, 10 Sep 2026 22:26:29 +0900 Subject: [PATCH 011/157] Allow non-ASCII percent literal delimiters --- lib/lrama/ruby/action_scanner.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/action_scanner.rb b/lib/lrama/ruby/action_scanner.rb index 8b3a170..42fec83 100644 --- a/lib/lrama/ruby/action_scanner.rb +++ b/lib/lrama/ruby/action_scanner.rb @@ -277,7 +277,8 @@ def percent_literal(offset) @scanner.getch kind = @scanner.scan(/[qQwWiIrsx]/) || "Q" delimiter = @scanner.getch - unless delimiter && /\A[!-~]\z/.match?(delimiter) && !/[a-zA-Z0-9\\]/.match?(delimiter) + unless delimiter && delimiter.bytesize == 1 && + !delimiter.match?(/[a-zA-Z0-9\\\r\n]/n) fail_at("Unsupported percent literal delimiter", offset) end literal(LITERAL_PAIRS.fetch(delimiter, delimiter), open: LITERAL_PAIRS.key?(delimiter) ? delimiter : nil, From 701eab910d45fdeace58418dc244f7fe9d46df1d Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Thu, 10 Sep 2026 22:27:31 +0900 Subject: [PATCH 012/157] Disambiguate percent operators from literals --- lib/lrama/ruby/action_scanner.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/lrama/ruby/action_scanner.rb b/lib/lrama/ruby/action_scanner.rb index 42fec83..432657a 100644 --- a/lib/lrama/ruby/action_scanner.rb +++ b/lib/lrama/ruby/action_scanner.rb @@ -118,7 +118,7 @@ def code(terminator) :bare end elsif char == "/" || char == "%" || @scanner.check(/< Date: Thu, 10 Sep 2026 22:28:37 +0900 Subject: [PATCH 013/157] Disambiguate compressed shifts from heredocs --- lib/lrama/ruby/action_scanner.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/action_scanner.rb b/lib/lrama/ruby/action_scanner.rb index 432657a..589066f 100644 --- a/lib/lrama/ruby/action_scanner.rb +++ b/lib/lrama/ruby/action_scanner.rb @@ -121,7 +121,8 @@ def code(terminator) if char == "%" && @scanner.check(/%[qQwWiIrsx]?[a-zA-Z0-9_]*[^a-zA-Z0-9_\\\s]/n) percent_literal(offset) state = :end - elsif @scanner.check(/<<[-~]?(?:['"`][^\r\n]+['"`]|[a-zA-Z_][a-zA-Z_0-9]*)/n) + elsif (state != :bare || spaced) && + @scanner.check(/<<[-~]?(?:['"`][^\r\n]+['"`]|[a-zA-Z_][a-zA-Z_0-9]*)/n) heredocs << heredoc_start(offset) state = :end elsif state == :method From a5f181e78739517d87582902e69e3fc375d227d6 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Thu, 10 Sep 2026 22:29:46 +0900 Subject: [PATCH 014/157] Disambiguate percent operators from percent literals --- lib/lrama/ruby/action_scanner.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/action_scanner.rb b/lib/lrama/ruby/action_scanner.rb index 589066f..e9229f7 100644 --- a/lib/lrama/ruby/action_scanner.rb +++ b/lib/lrama/ruby/action_scanner.rb @@ -118,7 +118,9 @@ def code(terminator) :bare end elsif char == "/" || char == "%" || @scanner.check(/< Date: Thu, 10 Sep 2026 22:31:03 +0900 Subject: [PATCH 015/157] Stop scanning Ruby source at __END__ --- lib/lrama/ruby/action_scanner.rb | 4 ++++ test/lrama/action_scanner_test.rb | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/action_scanner.rb b/lib/lrama/ruby/action_scanner.rb index e9229f7..823d104 100644 --- a/lib/lrama/ruby/action_scanner.rb +++ b/lib/lrama/ruby/action_scanner.rb @@ -80,6 +80,10 @@ def code(terminator) end fail_at("Unterminated block comment", offset) unless found next + elsif terminator.nil? && (offset.zero? || @source.getbyte(offset - 1) == 10) && + @scanner.check(/__END__(?:\s|\z)/) + @scanner.terminate + return elsif char == terminator && brackets.empty? fail_at("Close the action/interpolation after the heredoc body", offset) unless heredocs.empty? return diff --git a/test/lrama/action_scanner_test.rb b/test/lrama/action_scanner_test.rb index 3aea135..69520fd 100644 --- a/test/lrama/action_scanner_test.rb +++ b/test/lrama/action_scanner_test.rb @@ -145,7 +145,7 @@ def reference_texts(source) test "unclosed constructs and mismatched brackets report source positions" do ["'text", '"text', '%q{text', '/[abc/', '"#{ $1', "([)]", "(1", "$$ = < Date: Thu, 10 Sep 2026 22:33:39 +0900 Subject: [PATCH 016/157] Avoid treating shifts after variables as heredocs --- lib/lrama/ruby/action_scanner.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/lrama/ruby/action_scanner.rb b/lib/lrama/ruby/action_scanner.rb index 823d104..1b1f709 100644 --- a/lib/lrama/ruby/action_scanner.rb +++ b/lib/lrama/ruby/action_scanner.rb @@ -127,7 +127,7 @@ def code(terminator) !@scanner.peek(2).byteslice(1, 1).match?(/[a-zA-Z0-9_]/n)) percent_literal(offset) state = :end - elsif (state != :bare || spaced) && + elsif (state == :begin || (state == :bare && spaced)) && @scanner.check(/<<[-~]?(?:['"`][^\r\n]+['"`]|[a-zA-Z_][a-zA-Z_0-9]*)/n) heredocs << heredoc_start(offset) state = :end From 2e757917a4a97ce85d292053ba32e349b1cf23fa Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Thu, 10 Sep 2026 22:34:58 +0900 Subject: [PATCH 017/157] Require a delimiter boundary for heredocs --- lib/lrama/ruby/action_scanner.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/lrama/ruby/action_scanner.rb b/lib/lrama/ruby/action_scanner.rb index 1b1f709..f64c347 100644 --- a/lib/lrama/ruby/action_scanner.rb +++ b/lib/lrama/ruby/action_scanner.rb @@ -128,7 +128,7 @@ def code(terminator) percent_literal(offset) state = :end elsif (state == :begin || (state == :bare && spaced)) && - @scanner.check(/<<[-~]?(?:['"`][^\r\n]+['"`]|[a-zA-Z_][a-zA-Z_0-9]*)/n) + @scanner.check(/<<[-~]?(?:['"`][^\r\n]+['"`]|[a-zA-Z_][a-zA-Z_0-9]*)(?=\s|[;,)\]}]|\z)/n) heredocs << heredoc_start(offset) state = :end elsif state == :method @@ -141,7 +141,7 @@ def code(terminator) @scanner.scan(/[a-z]*/) elsif char == "%" percent_literal(offset) - elsif @scanner.check(/<<[-~]?(?:['"`][^\r\n]+['"`]|[a-zA-Z_][a-zA-Z_0-9]*)/n) + elsif @scanner.check(/<<[-~]?(?:['"`][^\r\n]+['"`]|[a-zA-Z_][a-zA-Z_0-9]*)(?=\s|[;,)\]}]|\z)/n) heredocs << heredoc_start(offset) else @scanner.scan(/< Date: Sat, 12 Sep 2026 11:03:57 +0900 Subject: [PATCH 018/157] Add Ruby source lexer literal support --- README.md | 13 + lib/lrama/ruby/languages/ruby.rb | 9 + lib/lrama/ruby/languages/ruby/lexer.rb | 516 +++++++++++++++++++++++++ sig/arpaka.rbs | 4 + sig/lrama/ruby.rbs | 4 + test/lrama/ruby_language_test.rb | 34 ++ 6 files changed, 580 insertions(+) create mode 100644 lib/lrama/ruby/languages/ruby/lexer.rb diff --git a/README.md b/README.md index b57032e..7361ba5 100644 --- a/README.md +++ b/README.md @@ -132,6 +132,19 @@ values are Floats, and `tIDENTIFIER` values are Symbols or Strings. Operators an keywords may carry `nil`. Parentheses use Ruby's context-dependent token names (for example `tLPAREN` followed by `")"`), not a source string tokenizer. +For source input, use `Arpaka.parse_source`: + +```ruby +tree = Arpaka.parse_source("a = 1\na + 2", filename: "example.rb") +``` + +The source lexer is byte-oriented and context-sensitive. It supports the same +frontend slice as the token API, including numbers, variables, calls, arrays, +hashes, control flow, definitions, strings, regular expressions, percent +literals and static heredocs. Lexer failures include filename, line and byte +column. Ruby interpolation and syntax whose value cannot be represented by the +current AST raises an explicit lexer error. + The supported slice includes numeric and nil/boolean literals, binary `+ - * /`, unary `+ -`, single-expression parentheses, simple local assignments, and statements separated by semicolons or newlines. Every successful parse returns diff --git a/lib/lrama/ruby/languages/ruby.rb b/lib/lrama/ruby/languages/ruby.rb index ca28edc..4e21a9c 100644 --- a/lib/lrama/ruby/languages/ruby.rb +++ b/lib/lrama/ruby/languages/ruby.rb @@ -24,6 +24,9 @@ def initialize(rule, line) end end + class LexerError < ::Lrama::Ruby::Error + end + PARSER_MUTEX = Mutex.new private_constant :PARSER_MUTEX @@ -38,6 +41,11 @@ def self.parse(tokens) end end + def self.parse_source(source, filename: "(ruby)") + lexer = Lexer.new(source, filename: filename) + parse(lexer.each) + end + def self.generated_parser PARSER_MUTEX.synchronize do @generated_parser ||= begin @@ -55,3 +63,4 @@ def self.generated_parser require_relative "ruby/ast" require_relative "ruby/builder" +require_relative "ruby/lexer" diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb new file mode 100644 index 0000000..8745049 --- /dev/null +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -0,0 +1,516 @@ +# frozen_string_literal: true + +module Lrama + module Ruby + module Languages + module Ruby + # Lexer for Ruby source input. This deliberately does not use Prism: + # the parser needs the same small amount of lexical context that CRuby + # keeps in its parser (EXPR_BEG/EXPR_END and delimiter nesting). + class Lexer + Error = LexerError + + KEYWORDS = { + "class" => :keyword_class, "module" => :keyword_module, + "def" => :keyword_def, "begin" => :keyword_begin, + "rescue" => :keyword_rescue, "ensure" => :keyword_ensure, + "end" => :keyword_end, "if" => :keyword_if, + "unless" => :keyword_unless, "then" => :keyword_then, + "elsif" => :keyword_elsif, "else" => :keyword_else, + "case" => :keyword_case, "when" => :keyword_when, + "while" => :keyword_while, "until" => :keyword_until, + "for" => :keyword_for, "break" => :keyword_break, + "next" => :keyword_next, "redo" => :keyword_redo, + "retry" => :keyword_retry, "return" => :keyword_return, + "yield" => :keyword_yield, "super" => :keyword_super, + "self" => :keyword_self, "nil" => :keyword_nil, + "true" => :keyword_true, "false" => :keyword_false, + "and" => :keyword_and, "or" => :keyword_or, + "not" => :keyword_not, "alias" => :keyword_alias, + "defined?" => :keyword_defined, "in" => :keyword_in, + "BEGIN" => :keyword_BEGIN, "END" => :keyword_END, + "__LINE__" => :keyword__LINE__, "__FILE__" => :keyword__FILE__, + "__ENCODING__" => :keyword__ENCODING__ + }.freeze + + OPERATORS = %w[... .. <=> === == != =~ !~ >= <= && || << >> ** => :: &. -> += -= *= /= %= **= <<= >>= &&= ||=].freeze + OP_TOKENS = { + "**" => :tPOW, "<=>" => :tCMP, "==" => :tEQ, "===" => :tEQQ, + "!=" => :tNEQ, ">=" => :tGEQ, "<=" => :tLEQ, "&&" => :tANDOP, + "||" => :tOROP, "=~" => :tMATCH, "!~" => :tNMATCH, + ".." => :tDOT2, "..." => :tDOT3, "<<" => :tLSHFT, + ">>" => :tRSHFT, "&." => :tANDDOT, "::" => :tCOLON2, + "=>" => :tASSOC, "->" => :tLAMBDA + }.freeze + + def initialize(source, filename: "(ruby)") + unless source.is_a?(String) + raise ArgumentError, "source must be a String" + end + unless [Encoding::UTF_8, Encoding::US_ASCII].include?(source.encoding) && source.valid_encoding? + raise Error, "#{filename}:1:0: invalid UTF-8 source" + end + @source = source.b + @filename = filename + @index = 0 + @line = 1 + @column = 0 + @previous = nil + @begin_expression = true + @delimiter_depth = 0 + @pending = [] + @condition_do = false + @condition_line = false + end + + def each + return enum_for(__method__) unless block_given? + until eof? && @pending.empty? + token = next_token + yield token if token + end + yield [0, nil] + end + + private + + def eof? + @index >= @source.bytesize + end + + def byte(offset = 0) + @source.getbyte(@index + offset) + end + + def advance(count = 1) + count.times do + value = byte + @index += 1 + if value == 10 + @line += 1 + @column = 0 + else + @column += 1 + end + end + end + + def fail!(message, index = @index, line = @line, column = @column) + raise Error, "#{@filename}:#{line}:#{column}: #{message}" + end + + def next_token + unless @pending.empty? + value = @pending.shift + @previous = value[0] + @begin_expression = expression_begin_after(value[0]) + return value + end + skip_space_and_comments + return nil if eof? + + start = @index + value = case byte + when 10 + advance + return next_token if newline_ignored? + ["\n", nil] + when 39, 34, 96 + string_token(byte, start) + when 47 + regexp_or_operator(start) + when 37 + percent_token(start) + when 60 + heredoc_or_operator(start) + when 48..57 + number_token(start) + when 36, 64 + variable_token(start) + when 63 + character_or_question(start) + when 58 + symbol_or_colon(start) + when 65..90, 95, 97..122, 128..255 + identifier_token(start) + else + operator_or_punctuation(start) + end + @previous = value && value[0] + @begin_expression = expression_begin_after(value && value[0]) + value + end + + def skip_space_and_comments + loop do + while [9, 11, 12, 13, 32].include?(byte) + advance + end + break unless byte == 35 + advance + advance while !eof? && byte != 10 + end + end + + def newline_ignored? + ignored = @delimiter_depth.positive? || @previous == "\n" || + ["+", "-", "*", "/", "%", "=", "?", ":", ",", ".", "&&", "||", "=>"].include?(@previous) || + next_word_is_terminator? + @condition_line = false if ignored && @condition_line + ignored + end + + def next_word_is_terminator? + position = @index + position += 1 while [9, 11, 12, 13, 32].include?(@source.getbyte(position)) + %w[else elsif end when].any? do |word| + @source.byteslice(position, word.bytesize) == word && + !identifier_byte?(@source.getbyte(position + word.bytesize)) + end + end + + def expression_begin_after(token) + return true if token.nil? || token == "\n" + return false if token == ")" || token == "]" || token == "}" + return false if token == :tINTEGER || token == :tFLOAT || token == :tRATIONAL || token == :tIMAGINARY + return false if token == :tIDENTIFIER || token == :tCONSTANT || token == :tSTRING_END || token == :tREGEXP_END + return false if token == :keyword_true || token == :keyword_false || token == :keyword_nil || token == :keyword_self + return false if token == :tIVAR || token == :tGVAR || token == :tCVAR || token == :tNTH_REF + true + end + + def identifier_token(start) + while identifier_byte?(byte) + advance + end + word = @source.byteslice(start, @index - start).force_encoding(Encoding::UTF_8) + if byte == 63 && word == "defined" + advance + word = "defined?" + end + if byte == 58 && byte(1) != 58 + advance + return [:tLABEL, word.to_sym] + end + token = KEYWORDS[word] + if !@begin_expression && { "if" => :modifier_if, "unless" => :modifier_unless, + "while" => :modifier_while, "until" => :modifier_until, + "rescue" => :modifier_rescue }.key?(word) + token = { "if" => :modifier_if, "unless" => :modifier_unless, + "while" => :modifier_while, "until" => :modifier_until, + "rescue" => :modifier_rescue }.fetch(word, token) + end + token ||= word == "do" ? do_token : (word.getbyte(0) == 65 ? :tCONSTANT : :tIDENTIFIER) + @condition_do = true if [:keyword_while, :keyword_until, :keyword_for].include?(token) + [token, word.to_sym] + rescue EncodingError + fail!("invalid UTF-8 identifier", start) + end + + def identifier_byte?(value) + value && (value == 95 || value.between?(65, 90) || value.between?(97, 122) || value >= 128 || value.between?(48, 57)) + end + + def do_token + if @condition_do + @condition_do = false + :keyword_do_cond + elsif @previous == :tLAMBDA + :keyword_do_LAMBDA + else + :keyword_do_block + end + end + + def number_token(start) + base = 10 + if byte == 48 && [120, 88, 98, 66, 111, 79].include?(byte(1)) + base = { 120 => 16, 88 => 16, 98 => 2, 66 => 2, 111 => 8, 79 => 8 }.fetch(byte(1)) + advance(2) + digit_start = @index + advance while digit_byte?(byte, base) || byte == 95 + fail!("invalid numeric literal", start) if @index == digit_start + text = @source.byteslice(digit_start, @index - digit_start).delete("_") + return [:tINTEGER, text.to_i(base)] + end + advance while digit_byte?(byte, 10) || byte == 95 + float = false + if byte == 46 && digit_byte?(byte(1), 10) + float = true + advance + advance while digit_byte?(byte, 10) || byte == 95 + end + if byte == 101 || byte == 69 + float = true + advance + advance if byte == 43 || byte == 45 + fail!("invalid numeric literal", start) unless digit_byte?(byte, 10) + advance while digit_byte?(byte, 10) || byte == 95 + end + suffix = byte + advance if [105, 114].include?(suffix) + text = @source.byteslice(start, @index - start).delete("_") + return [:tRATIONAL, Rational(text.delete_suffix("r"))] if suffix == 114 + return [:tIMAGINARY, Complex(0, text.delete_suffix("i").to_f)] if suffix == 105 + [float ? :tFLOAT : :tINTEGER, float ? text.to_f : text.to_i] + rescue ArgumentError, ZeroDivisionError + fail!("invalid numeric literal", start) + end + + def digit_byte?(value, base) + return false unless value + value.between?(48, 57) && value - 48 < base || base == 16 && value.between?(65, 70) || base == 16 && value.between?(97, 102) + end + + def variable_token(start) + marker = byte + advance + if marker == 36 && byte && byte.between?(48, 57) + advance while byte && byte.between?(48, 57) + return [:tNTH_REF, @source.byteslice(start + 1, @index - start - 1).to_i] + end + advance while identifier_byte?(byte) + text = @source.byteslice(start, @index - start) + token = marker == 36 ? :tGVAR : (text.start_with?("@@") ? :tCVAR : :tIVAR) + [token, text.to_sym] + end + + def character_or_question(start) + return operator_or_punctuation(start) unless @begin_expression && byte(1) && byte(1) != 32 && byte(1) != 10 + advance + value = if byte == 92 + escape_sequence(start) + else + character = byte + advance + character.chr + end + [:tCHAR, value] + end + + def symbol_or_colon(start) + return operator_or_punctuation(start) unless @begin_expression && byte(1) && byte(1) != 58 + advance + if byte == 39 || byte == 34 + quote = byte + advance + content = read_quoted(quote, interpolate: quote == 34, start: start) + @pending = [[:tSTRING_CONTENT, content], [:tSTRING_END, nil]] + end + [:tSYMBEG, nil] + end + + def string_token(quote, start) + advance + content = read_quoted(quote, interpolate: quote != 39, start: start) + @pending = [[:tSTRING_CONTENT, content], [:tSTRING_END, nil]] + [quote == 96 ? :tXSTRING_BEG : :tSTRING_BEG, nil] + end + + def read_quoted(quote, interpolate:, start:) + result = +"" + until eof? + value = byte + if value == quote + advance + return result + elsif value == 92 + result << escape_sequence(start) + elsif value == 35 && interpolate && byte(1) == 123 + fail!("string interpolation cannot be represented by the AST", start) + else + result << value + advance + end + end + fail!("unterminated literal", start) + end + + def escape_sequence(start) + advance + fail!("unterminated escape", start) if eof? + value = byte + advance + return "\n" if value == 110 + return "\t" if value == 116 + return "\r" if value == 114 + return "\f" if value == 102 + return "\a" if value == 97 + return "\e" if value == 101 + return value.chr if value == 92 || value == 34 || value == 39 + value.chr + end + + def regexp_or_operator(start) + if @begin_expression + advance + value = read_regexp(start) + @pending = [[:tSTRING_CONTENT, value], [:tREGEXP_END, nil]] + return [:tREGEXP_BEG, nil] + end + operator_or_punctuation(start) + end + + def read_regexp(start) + result = +"" + in_class = false + until eof? + value = byte + if value == 92 + result << value.chr + advance + fail!("unterminated regexp", start) if eof? + result << byte.chr + advance + elsif value == 91 + in_class = true + result << value.chr + advance + elsif value == 93 + in_class = false + result << value.chr + advance + elsif value == 47 && !in_class + advance + advance while byte && byte.between?(97, 122) + return result + else + result << value.chr + advance + end + end + fail!("unterminated regexp", start) + end + + def percent_token(start) + advance + kind = byte + if [113, 81, 119, 87, 105, 73, 114, 115, 120, 88].include?(kind) + advance + else + kind = 81 + end + delimiter = byte + fail!("invalid percent literal", start) unless delimiter + advance + closing = { 40 => 41, 91 => 93, 123 => 125, 60 => 62 }.fetch(delimiter, delimiter) + content = read_delimited(closing, start, interpolate: [81, 87, 73, 114, 120, 88].include?(kind)) + token = { 113 => :tSTRING_BEG, 81 => :tSTRING_BEG, 119 => :tWORDS_BEG, 87 => :tQWORDS_BEG, + 105 => :tSYMBOLS_BEG, 73 => :tQSYMBOLS_BEG, 114 => :tREGEXP_BEG, + 115 => :tSYMBEG, 120 => :tXSTRING_BEG, 88 => :tXSTRING_BEG }.fetch(kind) + terminator = kind == 114 ? :tREGEXP_END : :tSTRING_END + @pending = if [119, 87, 105, 73].include?(kind) + word_tokens(content, symbols: [105, 73].include?(kind)) + [[terminator, nil]] + else + [[:tSTRING_CONTENT, content], [terminator, nil]] + end + [token, nil] + end + + def word_tokens(content, symbols:) + words = content.split(/[\t\n\f\r ]+/).reject(&:empty?) + tokens = [[" ", nil]] + words.each do |word| + tokens << [:tSTRING_CONTENT, symbols ? word.to_sym : word] + tokens << [" ", nil] + end + tokens + end + + def read_delimited(closing, start, interpolate: false) + result = +"" + depth = 0 + until eof? + value = byte + if value == 92 + result << escape_sequence(start) + elsif value == 35 && interpolate && byte(1) == 123 + fail!("string interpolation cannot be represented by the AST", start) + elsif value == closing && depth.zero? + advance + return result + else + depth += 1 if value == ({ 41 => 40, 93 => 91, 125 => 123, 62 => 60 }.fetch(closing, -1)) + depth -= 1 if value == closing && depth.positive? + result << value.chr + advance + end + end + fail!("unterminated percent literal", start) + end + + def heredoc_or_operator(start) + if byte(1) == 60 + return heredoc_token(start) if @begin_expression || @previous == :tIDENTIFIER + end + operator_or_punctuation(start) + end + + def heredoc_token(start) + advance(2) + advance if byte == 45 || byte == 126 + quote = byte + if quote == 39 || quote == 34 || quote == 96 + advance + delimiter_start = @index + advance while byte && byte != quote + fail!("unterminated heredoc identifier", start) if eof? + delimiter = @source.byteslice(delimiter_start, @index - delimiter_start) + advance + else + delimiter_start = @index + advance while identifier_byte?(byte) + delimiter = @source.byteslice(delimiter_start, @index - delimiter_start) + end + fail!("invalid heredoc identifier", start) if delimiter.empty? + advance while byte && byte != 10 + advance if byte == 10 + body = +"" + loop do + line_start = @index + advance while !eof? && byte != 10 + line = @source.byteslice(line_start, @index - line_start) + if line == delimiter + advance if byte == 10 + break + end + body << line << "\n" + advance if byte == 10 + fail!("unterminated heredoc", start) if eof? + end + @pending = [[:tSTRING_CONTENT, body], [:tSTRING_END, nil]] + [:tSTRING_BEG, nil] + end + + def operator_or_punctuation(start) + text = OPERATORS.sort_by(&:bytesize).find { |operator| @source.byteslice(@index, operator.bytesize) == operator } + if text + advance(text.bytesize) + return [OP_TOKENS.fetch(text, :tOP_ASGN), nil] + end + value = byte.chr + advance + if value == "(" || value == "[" || value == "{" + @delimiter_depth += 1 + if value == "(" + return [@begin_expression ? :tLPAREN : "(", nil] + end + return [:tLAMBEG, nil] if value == "{" && @previous == :tLAMBDA + return [value == "[" ? :tLBRACK : :tLBRACE, nil] + elsif value == ")" || value == "]" || value == "}" + @delimiter_depth -= 1 if @delimiter_depth.positive? + elsif value == "-" && @begin_expression + return [:tUMINUS, nil] + elsif value == "+" && @begin_expression + return [:tUPLUS, nil] + end + [value, nil] + rescue EncodingError + fail!("invalid byte", start) + end + end + private_constant :Lexer + end + end + end +end diff --git a/sig/arpaka.rbs b/sig/arpaka.rbs index a434f95..28bc2b9 100644 --- a/sig/arpaka.rbs +++ b/sig/arpaka.rbs @@ -1,3 +1,7 @@ module Arpaka def self.parse: (Enumerable[[String | Symbol | Integer, untyped]] tokens) -> ::Lrama::Ruby::Languages::Ruby::AST::Program + def self.parse_source: (String source, ?filename: String) -> ::Lrama::Ruby::Languages::Ruby::AST::Program + + class LexerError < ::Lrama::Ruby::Error + end end diff --git a/sig/lrama/ruby.rbs b/sig/lrama/ruby.rbs index 3576673..ae2e4a2 100644 --- a/sig/lrama/ruby.rbs +++ b/sig/lrama/ruby.rbs @@ -18,6 +18,7 @@ module Lrama module Languages module Ruby def self.parse: (Enumerable[[String | Symbol | Integer, untyped]] tokens) -> AST::Program + def self.parse_source: (String source, ?filename: String) -> AST::Program class ParseError < ::Lrama::Ruby::Error attr_reader token: untyped @@ -31,6 +32,9 @@ module Lrama def initialize: (String rule, Integer line) -> void end + class LexerError < ::Lrama::Ruby::Error + end + module AST type node = Literal | Binary | Unary | If | LocalRead | LocalWrite | BareCall | ArrayLiteral | HashLiteral | RangeLiteral | Ternary | Call | StringLiteral | RegexpLiteral | XStringLiteral | Index | Loop | Control | Variable | Rescue | Case | When | For | Def | ClassDef | ModuleDef | Lambda type hash_pair = Pair diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index fb4b91a..0dfd2dc 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -347,4 +347,38 @@ def literal(value) assert_raise(Arpaka::ParseError) { Arpaka.parse(tokens) } end end + + test "source input uses the Ruby lexer and preserves token parse results" do + expected = AST::Program.new([ + AST::LocalWrite.new(:a, literal(1)), + AST::Binary.new(:+, AST::LocalRead.new(:a), literal(2)) + ]) + assert_equal(expected, Arpaka.parse_source("a = 1\na + 2", filename: "example.rb")) + assert_equal(AST::Program.new([AST::Call.new(:f, [literal(1), literal(2)])]), + Arpaka.parse_source("f(1, 2)")) + assert_equal(AST::Program.new([AST::StringLiteral.new("hello")]), + Arpaka.parse_source('"hello"')) + end + + test "source lexer errors include source position" do + error = assert_raise(Arpaka::LexerError) { Arpaka.parse_source('"unterminated', filename: "broken.rb") } + assert_include(error.message, "broken.rb:1:") + end + + test "source lexer preserves literal kinds" do + assert_equal(AST::XStringLiteral.new("echo"), Arpaka.parse_source("`echo`").statements.first) + assert_equal(AST::RegexpLiteral.new("a[b]"), Arpaka.parse_source("/a[b]/").statements.first) + assert_equal(AST::StringLiteral.new("quiet"), Arpaka.parse_source("%q(quiet)").statements.first) + assert_equal(AST::ArrayLiteral.new([AST::StringLiteral.new("one"), AST::StringLiteral.new("two")]), + Arpaka.parse_source("%w(one two)").statements.first) + assert_equal(AST::ArrayLiteral.new([:one, :two]), + Arpaka.parse_source("%i(one two)").statements.first) + assert_equal(AST::RegexpLiteral.new("a"), Arpaka.parse_source("%r(a)").statements.first) + end + + test "source lexer rejects interpolation in static literals" do + ['"#{x}"', '%Q(#{x})', '%r(#{x})'].each do |source| + assert_raise(Arpaka::LexerError) { Arpaka.parse_source(source) } + end + end end From ddadf4220af6c3b2a2e8da0fd395ba9eb6863e6e Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 11:16:50 +0900 Subject: [PATCH 019/157] Improve Ruby heredoc lexing --- lib/lrama/ruby/languages/ruby/lexer.rb | 24 +++++++++++++++++++++--- test/lrama/ruby_language_test.rb | 16 ++++++++++++++++ 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 8745049..2e21246 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -448,7 +448,9 @@ def heredoc_or_operator(start) def heredoc_token(start) advance(2) - advance if byte == 45 || byte == 126 + indent = byte == 45 || byte == 126 + squiggly = byte == 126 + advance if indent quote = byte if quote == 39 || quote == 34 || quote == 96 advance @@ -466,20 +468,36 @@ def heredoc_token(start) advance while byte && byte != 10 advance if byte == 10 body = +"" + interpolate = quote != 39 loop do line_start = @index advance while !eof? && byte != 10 line = @source.byteslice(line_start, @index - line_start) - if line == delimiter + line_without_cr = line.delete_suffix("\r") + terminator = indent ? line_without_cr.sub(/\A[ \t]*/, "") : line_without_cr + if terminator == delimiter advance if byte == 10 break end + fail!("string interpolation cannot be represented by the AST", start) if interpolate && line.include?("#" + "{") body << line << "\n" advance if byte == 10 fail!("unterminated heredoc", start) if eof? end + body = dedent_heredoc(body) if squiggly @pending = [[:tSTRING_CONTENT, body], [:tSTRING_END, nil]] - [:tSTRING_BEG, nil] + [quote == 96 ? :tXSTRING_BEG : :tSTRING_BEG, nil] + end + + def dedent_heredoc(body) + indents = body.lines.filter_map do |line| + next if line.strip.empty? + line[/\A[ \t]*/].bytesize + end + return body if indents.empty? + + width = indents.min + body.lines.map { |line| line.sub(/\A[ \t]{0,#{width}}/, "") }.join end def operator_or_punctuation(start) diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 0dfd2dc..0af2d57 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -381,4 +381,20 @@ def literal(value) assert_raise(Arpaka::LexerError) { Arpaka.parse_source(source) } end end + + test "source lexer handles heredoc indentation and line endings" do + assert_equal(AST::StringLiteral.new("one\n two\n"), + Arpaka.parse_source("<<~TEXT\n one\n two\n TEXT\n").statements.first) + assert_equal(AST::StringLiteral.new(" one\n"), + Arpaka.parse_source("<<-TEXT\n one\n TEXT\n").statements.first) + assert_equal(AST::StringLiteral.new("one\r\ntwo\r\n"), + Arpaka.parse_source("< Date: Sat, 12 Sep 2026 11:21:32 +0900 Subject: [PATCH 020/157] Improve lexer handling for Rails syntax --- lib/lrama/ruby/languages/ruby/lexer.rb | 20 ++++++++++++++++++-- test/lrama/ruby_language_test.rb | 14 ++++++++++++++ 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 2e21246..00c7a1f 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -61,6 +61,7 @@ def initialize(source, filename: "(ruby)") @pending = [] @condition_do = false @condition_line = false + @ternary_depth = 0 end def each @@ -188,6 +189,11 @@ def identifier_token(start) advance word = "defined?" end + if [33, 63].include?(byte) && !identifier_byte?(byte(1)) && byte(1) != 61 + suffix = byte.chr + advance + return [:tFID, (word + suffix).to_sym] + end if byte == 58 && byte(1) != 58 advance return [:tLABEL, word.to_sym] @@ -289,7 +295,8 @@ def character_or_question(start) end def symbol_or_colon(start) - return operator_or_punctuation(start) unless @begin_expression && byte(1) && byte(1) != 58 + symbol_position = @begin_expression || (@previous == :tIDENTIFIER && @ternary_depth.zero?) + return operator_or_punctuation(start) unless symbol_position && byte(1) && byte(1) != 58 advance if byte == 39 || byte == 34 quote = byte @@ -440,12 +447,17 @@ def read_delimited(closing, start, interpolate: false) end def heredoc_or_operator(start) - if byte(1) == 60 + if byte(1) == 60 && heredoc_prefix? return heredoc_token(start) if @begin_expression || @previous == :tIDENTIFIER end operator_or_punctuation(start) end + def heredoc_prefix? + value = byte(2) + value && ([39, 34, 96, 45, 126].include?(value) || identifier_byte?(value)) + end + def heredoc_token(start) advance(2) indent = byte == 45 || byte == 126 @@ -521,6 +533,10 @@ def operator_or_punctuation(start) return [:tUMINUS, nil] elsif value == "+" && @begin_expression return [:tUPLUS, nil] + elsif value == "?" && !@begin_expression + @ternary_depth += 1 + elsif value == ":" && @ternary_depth.positive? + @ternary_depth -= 1 end [value, nil] rescue EncodingError diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 0af2d57..ccf0bfb 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -393,6 +393,20 @@ def literal(value) Arpaka.parse_source("<<`TEXT`\necho\nTEXT\n").statements.first) end + test "source lexer distinguishes shifts and command symbols" do + assert_equal(AST::Binary.new(:<<, literal(1), literal(2)), + Arpaka.parse_source("1 << 2").statements.first) + assert_equal(AST::Literal.new(:bar), + Arpaka.parse_source("foo = :bar").statements.first.value) + end + + test "source lexer treats predicate and bang methods as identifiers" do + assert_equal(AST::Def.new(:stopping?, [], [literal(1)]), + Arpaka.parse_source("def stopping?; 1; end").statements.first) + assert_equal(AST::Def.new(:halt!, [], [literal(1)]), + Arpaka.parse_source("def halt!; 1; end").statements.first) + end + test "single quoted heredoc keeps interpolation literal" do assert_equal(AST::StringLiteral.new("\#{x}\n"), Arpaka.parse_source("<<'TEXT'\n\#{x}\nTEXT\n").statements.first) From 0441a6148607c7c4db4dc1afd5daf100b6026218 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 11:24:01 +0900 Subject: [PATCH 021/157] Classify Ruby splat and block arguments --- lib/lrama/ruby/languages/ruby/lexer.rb | 11 ++++++++++- test/lrama/ruby_language_test.rb | 7 +++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 00c7a1f..513de50 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -516,7 +516,12 @@ def operator_or_punctuation(start) text = OPERATORS.sort_by(&:bytesize).find { |operator| @source.byteslice(@index, operator.bytesize) == operator } if text advance(text.bytesize) - return [OP_TOKENS.fetch(text, :tOP_ASGN), nil] + token = if text == "**" && @begin_expression + :tDSTAR + else + OP_TOKENS.fetch(text, :tOP_ASGN) + end + return [token, nil] end value = byte.chr advance @@ -533,6 +538,10 @@ def operator_or_punctuation(start) return [:tUMINUS, nil] elsif value == "+" && @begin_expression return [:tUPLUS, nil] + elsif value == "*" && @begin_expression + return [:tSTAR, nil] + elsif value == "&" && @begin_expression + return [:tAMPER, nil] elsif value == "?" && !@begin_expression @ternary_depth += 1 elsif value == ":" && @ternary_depth.positive? diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index ccf0bfb..7789899 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -407,6 +407,13 @@ def literal(value) Arpaka.parse_source("def halt!; 1; end").statements.first) end + test "source lexer distinguishes splat and block argument operators" do + assert_equal(AST::Call.new(:f, AST::BareCall.new(:args)), + Arpaka.parse_source("f(*args)").statements.first) + assert_equal(AST::Call.new(:map, AST::Literal.new(:to_s)), + Arpaka.parse_source("map(&:to_s)").statements.first) + end + test "single quoted heredoc keeps interpolation literal" do assert_equal(AST::StringLiteral.new("\#{x}\n"), Arpaka.parse_source("<<'TEXT'\n\#{x}\nTEXT\n").statements.first) From 4dacb8f24bf22e6e3eaaf74cf56adcd8eb767554 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 11:27:55 +0900 Subject: [PATCH 022/157] Improve Ruby constant and operator lexing --- lib/lrama/ruby/languages/ruby/lexer.rb | 14 ++++++++++---- test/lrama/ruby_language_test.rb | 6 ++++++ 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 513de50..b62cff2 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -33,14 +33,14 @@ class Lexer "__ENCODING__" => :keyword__ENCODING__ }.freeze - OPERATORS = %w[... .. <=> === == != =~ !~ >= <= && || << >> ** => :: &. -> += -= *= /= %= **= <<= >>= &&= ||=].freeze + OPERATORS = %w[[]= [] ... .. <=> === == != =~ !~ >= <= && || << >> ** => :: &. -> += -= *= /= %= **= <<= >>= &&= ||=].freeze OP_TOKENS = { "**" => :tPOW, "<=>" => :tCMP, "==" => :tEQ, "===" => :tEQQ, "!=" => :tNEQ, ">=" => :tGEQ, "<=" => :tLEQ, "&&" => :tANDOP, "||" => :tOROP, "=~" => :tMATCH, "!~" => :tNMATCH, ".." => :tDOT2, "..." => :tDOT3, "<<" => :tLSHFT, ">>" => :tRSHFT, "&." => :tANDDOT, "::" => :tCOLON2, - "=>" => :tASSOC, "->" => :tLAMBDA + "=>" => :tASSOC, "->" => :tLAMBDA, "[]" => :tAREF, "[]=" => :tASET }.freeze def initialize(source, filename: "(ruby)") @@ -206,7 +206,7 @@ def identifier_token(start) "while" => :modifier_while, "until" => :modifier_until, "rescue" => :modifier_rescue }.fetch(word, token) end - token ||= word == "do" ? do_token : (word.getbyte(0) == 65 ? :tCONSTANT : :tIDENTIFIER) + token ||= word == "do" ? do_token : (word.getbyte(0).between?(65, 90) ? :tCONSTANT : :tIDENTIFIER) @condition_do = true if [:keyword_while, :keyword_until, :keyword_for].include?(token) [token, word.to_sym] rescue EncodingError @@ -514,10 +514,12 @@ def dedent_heredoc(body) def operator_or_punctuation(start) text = OPERATORS.sort_by(&:bytesize).find { |operator| @source.byteslice(@index, operator.bytesize) == operator } - if text + if text && !(begin_expression? && @previous != :keyword_def && ["[]", "[]="].include?(text)) advance(text.bytesize) token = if text == "**" && @begin_expression :tDSTAR + elsif text == "::" && @begin_expression + :tCOLON3 else OP_TOKENS.fetch(text, :tOP_ASGN) end @@ -551,6 +553,10 @@ def operator_or_punctuation(start) rescue EncodingError fail!("invalid byte", start) end + + def begin_expression? + @begin_expression + end end private_constant :Lexer end diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 7789899..98cb1e5 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -414,6 +414,12 @@ def literal(value) Arpaka.parse_source("map(&:to_s)").statements.first) end + test "source lexer recognizes operator method names and top-level constants" do + assert_equal(AST::Def.new(:[], [], [AST::BareCall.new(:x)]), + Arpaka.parse_source("def [](x); x; end").statements.first) + assert_equal(:Foo, Arpaka.parse_source("::Foo").statements.first) + end + test "single quoted heredoc keeps interpolation literal" do assert_equal(AST::StringLiteral.new("\#{x}\n"), Arpaka.parse_source("<<'TEXT'\n\#{x}\nTEXT\n").statements.first) From 274bd8092321bbffb27fb3c899102b4bf2fd1bde Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 11:29:49 +0900 Subject: [PATCH 023/157] Recognize qualified operator methods --- lib/lrama/ruby/languages/ruby/lexer.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index b62cff2..daf116d 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -514,7 +514,8 @@ def dedent_heredoc(body) def operator_or_punctuation(start) text = OPERATORS.sort_by(&:bytesize).find { |operator| @source.byteslice(@index, operator.bytesize) == operator } - if text && !(begin_expression? && @previous != :keyword_def && ["[]", "[]="].include?(text)) + operator_method = [:keyword_def, ".", :tCOLON2].include?(@previous) + if text && !(begin_expression? && !operator_method && ["[]", "[]="].include?(text)) advance(text.bytesize) token = if text == "**" && @begin_expression :tDSTAR From 7634890e3a7a1946a7d70cd1c62bbf3a2deb39c4 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 11:35:59 +0900 Subject: [PATCH 024/157] Handle argumentless Ruby command blocks --- lib/lrama/ruby/languages/ruby/lexer.rb | 13 +++++++++++++ test/lrama/ruby_language_test.rb | 7 +++++++ 2 files changed, 20 insertions(+) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index daf116d..9927e65 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -55,6 +55,7 @@ def initialize(source, filename: "(ruby)") @index = 0 @line = 1 @column = 0 + @previous_previous = nil @previous = nil @begin_expression = true @delimiter_depth = 0 @@ -103,6 +104,7 @@ def fail!(message, index = @index, line = @line, column = @column) def next_token unless @pending.empty? value = @pending.shift + @previous_previous = @previous @previous = value[0] @begin_expression = expression_begin_after(value[0]) return value @@ -137,6 +139,7 @@ def next_token else operator_or_punctuation(start) end + @previous_previous = @previous @previous = value && value[0] @begin_expression = expression_begin_after(value && value[0]) value @@ -198,6 +201,10 @@ def identifier_token(start) advance return [:tLABEL, word.to_sym] end + if word == "do" && no_argument_block? + @pending.unshift([:keyword_do_block, nil]) + return [:tAMPER, nil] + end token = KEYWORDS[word] if !@begin_expression && { "if" => :modifier_if, "unless" => :modifier_unless, "while" => :modifier_while, "until" => :modifier_until, @@ -228,6 +235,12 @@ def do_token end end + def no_argument_block? + return false unless [:tIDENTIFIER, :tCONSTANT, :tFID].include?(@previous) + return false if [:tIDENTIFIER, :tCONSTANT, :tFID].include?(@previous_previous) + true + end + def number_token(start) base = 10 if byte == 48 && [120, 88, 98, 66, 111, 79].include?(byte(1)) diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 98cb1e5..6d59e33 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -414,6 +414,13 @@ def literal(value) Arpaka.parse_source("map(&:to_s)").statements.first) end + test "source lexer accepts argumentless command blocks" do + assert_equal(AST::Call.new(:included, []), + Arpaka.parse_source("included do\n 1\nend").statements.first) + assert_equal(AST::BareCall.new(:foo), + Arpaka.parse_source("foo.bar do\n 1\nend").statements.first) + end + test "source lexer recognizes operator method names and top-level constants" do assert_equal(AST::Def.new(:[], [], [AST::BareCall.new(:x)]), Arpaka.parse_source("def [](x); x; end").statements.first) From b6563aafb8cafebc961246150255a716d6341e98 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 11:43:04 +0900 Subject: [PATCH 025/157] Handle Ruby brace blocks --- lib/lrama/ruby/languages/ruby/lexer.rb | 6 ++++++ test/lrama/ruby_language_test.rb | 5 +++++ 2 files changed, 11 insertions(+) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 9927e65..33e7944 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -540,6 +540,12 @@ def operator_or_punctuation(start) return [token, nil] end value = byte.chr + if value == "{" && no_argument_block? + advance + @delimiter_depth += 1 + @pending.unshift([:tLBRACE_ARG, nil]) + return [:tAMPER, nil] + end advance if value == "(" || value == "[" || value == "{" @delimiter_depth += 1 diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 6d59e33..4f0ae26 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -299,6 +299,11 @@ def literal(value) assert_equal(AST::Program.new([AST::HashLiteral.new([])]), parse([:tLBRACE, nil], ["}", nil])) end + test "command blocks accept block parameters" do + assert_equal(AST::Program.new([:each]), Arpaka.parse_source("each { |value| value }")) + assert_equal(AST::Program.new([AST::Call.new(:each, [])]), Arpaka.parse_source("each do |value| value end")) + end + test "if, unless, elsif and else build conditional AST nodes" do expected = AST::Program.new([AST::If.new(literal(true), [literal(1)], [literal(2)])]) assert_equal(expected, parse([:keyword_if, nil], [:keyword_true, nil], [:keyword_then, nil], [:tINTEGER, 1], [:keyword_else, nil], [:tINTEGER, 2], [:keyword_end, nil])) From 6fdf731bd348c725c0e13d0fa035725cad1c54d2 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 11:44:59 +0900 Subject: [PATCH 026/157] Lex Ruby setter method names --- lib/lrama/ruby/languages/ruby/lexer.rb | 4 ++++ test/lrama/ruby_language_test.rb | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 33e7944..408150b 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -197,6 +197,10 @@ def identifier_token(start) advance return [:tFID, (word + suffix).to_sym] end + if byte == 61 && !identifier_byte?(byte(1)) && [:keyword_def, ".", :tCOLON2].include?(@previous) + advance + return [:tFID, (word + "=").to_sym] + end if byte == 58 && byte(1) != 58 advance return [:tLABEL, word.to_sym] diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 4f0ae26..0ff5981 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -110,6 +110,11 @@ def literal(value) assert_equal(AST::Program.new([expected]), parse(*tokens)) end + test "setter method definitions preserve the equals suffix" do + expected = AST::Program.new([AST::Def.new(:"value=", [], [AST::BareCall.new(:value)])]) + assert_equal(expected, Arpaka.parse_source("def value=(value)\n value\nend")) + end + test "class and module definitions preserve name and body" do class_tokens = [[:keyword_class, nil], [:tCONSTANT, :Foo], [";", nil], [:tINTEGER, 1], [";", nil], [:keyword_end, nil]] module_tokens = [[:keyword_module, nil], [:tCONSTANT, :Bar], [";", nil], [:tINTEGER, 2], [";", nil], [:keyword_end, nil]] From c7965ecbf22e750e9aedbade4163799d7e132b81 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 11:47:02 +0900 Subject: [PATCH 027/157] Lex keyword methods after receivers --- lib/lrama/ruby/languages/ruby/lexer.rb | 4 +++- test/lrama/ruby_language_test.rb | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 408150b..5153f7a 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -210,7 +210,9 @@ def identifier_token(start) return [:tAMPER, nil] end token = KEYWORDS[word] - if !@begin_expression && { "if" => :modifier_if, "unless" => :modifier_unless, + if token && [".", :tCOLON2].include?(@previous) + token = :tFID + elsif !@begin_expression && { "if" => :modifier_if, "unless" => :modifier_unless, "while" => :modifier_while, "until" => :modifier_until, "rescue" => :modifier_rescue }.key?(word) token = { "if" => :modifier_if, "unless" => :modifier_unless, diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 0ff5981..b1eb098 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -115,6 +115,10 @@ def literal(value) assert_equal(expected, Arpaka.parse_source("def value=(value)\n value\nend")) end + test "keywords after a receiver are method names" do + assert_nothing_raised { Arpaka.parse_source("self.class.to_s") } + end + test "class and module definitions preserve name and body" do class_tokens = [[:keyword_class, nil], [:tCONSTANT, :Foo], [";", nil], [:tINTEGER, 1], [";", nil], [:keyword_end, nil]] module_tokens = [[:keyword_module, nil], [:tCONSTANT, :Bar], [";", nil], [:tINTEGER, 2], [";", nil], [:keyword_end, nil]] From 111567645674ab682f443a1319075827b08ea110 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 11:49:03 +0900 Subject: [PATCH 028/157] Disambiguate endless method definitions --- lib/lrama/ruby/languages/ruby/lexer.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 5153f7a..c40b862 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -197,7 +197,7 @@ def identifier_token(start) advance return [:tFID, (word + suffix).to_sym] end - if byte == 61 && !identifier_byte?(byte(1)) && [:keyword_def, ".", :tCOLON2].include?(@previous) + if byte == 61 && byte(1) == 40 && [:keyword_def, ".", :tCOLON2].include?(@previous) advance return [:tFID, (word + "=").to_sym] end From 746df88d92a2fd3b58b67929e171c87223b14eb4 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 11:51:38 +0900 Subject: [PATCH 029/157] Avoid synthetic blocks after shifts --- lib/lrama/ruby/languages/ruby/lexer.rb | 1 + test/lrama/ruby_language_test.rb | 1 + 2 files changed, 2 insertions(+) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index c40b862..5161ba6 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -244,6 +244,7 @@ def do_token def no_argument_block? return false unless [:tIDENTIFIER, :tCONSTANT, :tFID].include?(@previous) return false if [:tIDENTIFIER, :tCONSTANT, :tFID].include?(@previous_previous) + return false if @previous_previous == :tLSHFT true end diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index b1eb098..2359373 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -433,6 +433,7 @@ def literal(value) Arpaka.parse_source("included do\n 1\nend").statements.first) assert_equal(AST::BareCall.new(:foo), Arpaka.parse_source("foo.bar do\n 1\nend").statements.first) + assert_nothing_raised { Arpaka.parse_source("items << lambda do\n 1\nend") } end test "source lexer recognizes operator method names and top-level constants" do From 001f59f1edc6e0e93d07744f0a693f7dd0767308 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 11:52:00 +0900 Subject: [PATCH 030/157] Test block context after shifts --- test/lrama/ruby_language_test.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 2359373..748c9c8 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -433,7 +433,8 @@ def literal(value) Arpaka.parse_source("included do\n 1\nend").statements.first) assert_equal(AST::BareCall.new(:foo), Arpaka.parse_source("foo.bar do\n 1\nend").statements.first) - assert_nothing_raised { Arpaka.parse_source("items << lambda do\n 1\nend") } + tokens = Arpaka.const_get(:Lexer, false).new("items << lambda do").each.to_a + assert_equal([:tIDENTIFIER, :tLSHFT, :tIDENTIFIER, :keyword_do_block], tokens[0, 4].map(&:first)) end test "source lexer recognizes operator method names and top-level constants" do From e89db08c947787795b18f7187f20aef7820cda23 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 11:54:09 +0900 Subject: [PATCH 031/157] Prefer longest Ruby operators --- lib/lrama/ruby/languages/ruby/lexer.rb | 2 +- test/lrama/ruby_language_test.rb | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 5161ba6..ddb183b 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -533,7 +533,7 @@ def dedent_heredoc(body) end def operator_or_punctuation(start) - text = OPERATORS.sort_by(&:bytesize).find { |operator| @source.byteslice(@index, operator.bytesize) == operator } + text = OPERATORS.sort_by { |operator| -operator.bytesize }.find { |operator| @source.byteslice(@index, operator.bytesize) == operator } operator_method = [:keyword_def, ".", :tCOLON2].include?(@previous) if text && !(begin_expression? && !operator_method && ["[]", "[]="].include?(text)) advance(text.bytesize) diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 748c9c8..282e772 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -412,6 +412,8 @@ def literal(value) Arpaka.parse_source("1 << 2").statements.first) assert_equal(AST::Literal.new(:bar), Arpaka.parse_source("foo = :bar").statements.first.value) + assert_equal(AST::RangeLiteral.new(:"...", literal(1), literal(3)), + Arpaka.parse_source("1...3").statements.first) end test "source lexer treats predicate and bang methods as identifiers" do From 8cef6932962f7a0b48b788ccf1efa38d2f543117 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 11:56:00 +0900 Subject: [PATCH 032/157] Lex Ruby argument forwarding --- lib/lrama/ruby/languages/ruby/lexer.rb | 2 ++ test/lrama/ruby_language_test.rb | 2 ++ 2 files changed, 4 insertions(+) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index ddb183b..fae1230 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -539,6 +539,8 @@ def operator_or_punctuation(start) advance(text.bytesize) token = if text == "**" && @begin_expression :tDSTAR + elsif text == "..." && [:tLPAREN, "("].include?(@previous) + :tBDOT3 elsif text == "::" && @begin_expression :tCOLON3 else diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 282e772..0f5cfb7 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -414,6 +414,8 @@ def literal(value) Arpaka.parse_source("foo = :bar").statements.first.value) assert_equal(AST::RangeLiteral.new(:"...", literal(1), literal(3)), Arpaka.parse_source("1...3").statements.first) + tokens = Arpaka.const_get(:Lexer, false).new("def f(...)").each.to_a + assert_equal(:tBDOT3, tokens[3].first) end test "source lexer treats predicate and bang methods as identifiers" do From 7c1c4889593cb4393d54a996e2768c110c71bad3 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 11:57:27 +0900 Subject: [PATCH 033/157] Distinguish Ruby indexing brackets --- lib/lrama/ruby/languages/ruby/lexer.rb | 2 +- test/lrama/ruby_language_test.rb | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index fae1230..d539cee 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -562,7 +562,7 @@ def operator_or_punctuation(start) return [@begin_expression ? :tLPAREN : "(", nil] end return [:tLAMBEG, nil] if value == "{" && @previous == :tLAMBDA - return [value == "[" ? :tLBRACK : :tLBRACE, nil] + return [value == "[" ? (@begin_expression ? :tLBRACK : "[") : :tLBRACE, nil] elsif value == ")" || value == "]" || value == "}" @delimiter_depth -= 1 if @delimiter_depth.positive? elsif value == "-" && @begin_expression diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 0f5cfb7..460c3a9 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -156,6 +156,8 @@ def literal(value) receiver = AST::ArrayLiteral.new([literal(1), literal(2)]) expected = AST::Index.new(receiver, [literal(0)]) assert_equal(AST::Program.new([expected]), parse([:tLBRACK, nil], [:tINTEGER, 1], [",", nil], [:tINTEGER, 2], ["]", nil], ["[", nil], [:tINTEGER, 0], ["]", nil])) + assert_equal(AST::Index.new(AST::BareCall.new(:values), [literal(0)]), + Arpaka.parse_source("values[0]").statements.first) end test "label hash entries build the same pair AST" do From 06f305b75b2a15eb5aeab16398c959432eaa83c8 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 12:00:04 +0900 Subject: [PATCH 034/157] Handle variable-shaped Ruby symbols --- lib/lrama/ruby/languages/ruby/builder.rb | 1 + test/lrama/ruby_language_test.rb | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/lib/lrama/ruby/languages/ruby/builder.rb b/lib/lrama/ruby/languages/ruby/builder.rb index e142bc2..f68ea12 100644 --- a/lib/lrama/ruby/languages/ruby/builder.rb +++ b/lib/lrama/ruby/languages/ruby/builder.rb @@ -94,6 +94,7 @@ def body_nodes(body) end def symbol(value) + value = value.name if value.is_a?(AST::Variable) AST::Literal.new(value.to_sym) end diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 460c3a9..449ea3b 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -434,6 +434,11 @@ def literal(value) Arpaka.parse_source("map(&:to_s)").statements.first) end + test "symbols can be built from variable-shaped values" do + builder = Lrama::Ruby::Languages::Ruby::Builder.new + assert_equal(AST::Literal.new(:value), builder.symbol(AST::Variable.new(:local, :value))) + end + test "source lexer accepts argumentless command blocks" do assert_equal(AST::Call.new(:included, []), Arpaka.parse_source("included do\n 1\nend").statements.first) From 7b6b0b5f699ef316749ad6b9de7ba79c6b74a1b0 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 12:02:43 +0900 Subject: [PATCH 035/157] Handle empty Ruby subclasses --- lib/lrama/ruby/languages/ruby/lexer.rb | 8 ++++++++ test/lrama/ruby_language_test.rb | 2 ++ 2 files changed, 10 insertions(+) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index d539cee..51e2b37 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -63,6 +63,7 @@ def initialize(source, filename: "(ruby)") @condition_do = false @condition_line = false @ternary_depth = 0 + @class_superclass = false end def each @@ -116,6 +117,11 @@ def next_token value = case byte when 10 advance + if @class_superclass && next_word_is_terminator? + @class_superclass = false + return [";", nil] + end + @class_superclass = false return next_token if newline_ignored? ["\n", nil] when 39, 34, 96 @@ -577,6 +583,8 @@ def operator_or_punctuation(start) @ternary_depth += 1 elsif value == ":" && @ternary_depth.positive? @ternary_depth -= 1 + elsif value == "<" && @previous == :tCONSTANT && @previous_previous == :keyword_class + @class_superclass = true end [value, nil] rescue EncodingError diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 449ea3b..3b69743 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -124,6 +124,8 @@ def literal(value) module_tokens = [[:keyword_module, nil], [:tCONSTANT, :Bar], [";", nil], [:tINTEGER, 2], [";", nil], [:keyword_end, nil]] assert_equal(AST::Program.new([AST::ClassDef.new(:Foo, nil, [literal(1)])]), parse(*class_tokens)) assert_equal(AST::Program.new([AST::ModuleDef.new(:Bar, [literal(2)])]), parse(*module_tokens)) + assert_equal(AST::ClassDef.new(:Error, AST::Variable.new(:constant, :StandardError), []), + Arpaka.parse_source("class Error < StandardError\nend").statements.first) end test "simple string literals preserve their content" do From aee018080ddfbf4d88a2f794ae9ad1b572dc9777 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 12:04:15 +0900 Subject: [PATCH 036/157] Keep predicate calls in expression context --- lib/lrama/ruby/languages/ruby/lexer.rb | 2 +- test/lrama/ruby_language_test.rb | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 51e2b37..b05bd03 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -183,7 +183,7 @@ def expression_begin_after(token) return true if token.nil? || token == "\n" return false if token == ")" || token == "]" || token == "}" return false if token == :tINTEGER || token == :tFLOAT || token == :tRATIONAL || token == :tIMAGINARY - return false if token == :tIDENTIFIER || token == :tCONSTANT || token == :tSTRING_END || token == :tREGEXP_END + return false if token == :tIDENTIFIER || token == :tCONSTANT || token == :tFID || token == :tSTRING_END || token == :tREGEXP_END return false if token == :keyword_true || token == :keyword_false || token == :keyword_nil || token == :keyword_self return false if token == :tIVAR || token == :tGVAR || token == :tCVAR || token == :tNTH_REF true diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 3b69743..6ee49d0 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -427,6 +427,7 @@ def literal(value) Arpaka.parse_source("def stopping?; 1; end").statements.first) assert_equal(AST::Def.new(:halt!, [], [literal(1)]), Arpaka.parse_source("def halt!; 1; end").statements.first) + assert_nothing_raised { Arpaka.parse_source("value.match?(pattern)") } end test "source lexer distinguishes splat and block argument operators" do From 10635447067044bdf2134ef0bf497cb71ca3ea74 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 12:06:24 +0900 Subject: [PATCH 037/157] Lex Ruby operator call parentheses --- lib/lrama/ruby/languages/ruby/lexer.rb | 2 +- test/lrama/ruby_language_test.rb | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index b05bd03..afdf7bb 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -565,7 +565,7 @@ def operator_or_punctuation(start) if value == "(" || value == "[" || value == "{" @delimiter_depth += 1 if value == "(" - return [@begin_expression ? :tLPAREN : "(", nil] + return [@begin_expression && ![".", :tCOLON2].include?(@previous) ? :tLPAREN : "(", nil] end return [:tLAMBEG, nil] if value == "{" && @previous == :tLAMBDA return [value == "[" ? (@begin_expression ? :tLBRACK : "[") : :tLBRACE, nil] diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 6ee49d0..7057517 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -119,6 +119,11 @@ def literal(value) assert_nothing_raised { Arpaka.parse_source("self.class.to_s") } end + test "operator calls use ordinary argument parentheses" do + tokens = Arpaka.const_get(:Lexer, false).new("handler.(message)").each.to_a + assert_equal([".", "(", :tIDENTIFIER], tokens[1, 3].map(&:first)) + end + test "class and module definitions preserve name and body" do class_tokens = [[:keyword_class, nil], [:tCONSTANT, :Foo], [";", nil], [:tINTEGER, 1], [";", nil], [:keyword_end, nil]] module_tokens = [[:keyword_module, nil], [:tCONSTANT, :Bar], [";", nil], [:tINTEGER, 2], [";", nil], [:keyword_end, nil]] From 52fb02aecd453428066033c8c78d6a5ccc30c4a6 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 12:08:29 +0900 Subject: [PATCH 038/157] Lex forwarded Ruby arguments after commas --- lib/lrama/ruby/languages/ruby/lexer.rb | 2 +- test/lrama/ruby_language_test.rb | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index afdf7bb..37e5c6f 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -545,7 +545,7 @@ def operator_or_punctuation(start) advance(text.bytesize) token = if text == "**" && @begin_expression :tDSTAR - elsif text == "..." && [:tLPAREN, "("].include?(@previous) + elsif text == "..." && [:tLPAREN, "(", ","].include?(@previous) :tBDOT3 elsif text == "::" && @begin_expression :tCOLON3 diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 7057517..e83ea2c 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -425,6 +425,8 @@ def literal(value) Arpaka.parse_source("1...3").statements.first) tokens = Arpaka.const_get(:Lexer, false).new("def f(...)").each.to_a assert_equal(:tBDOT3, tokens[3].first) + tokens = Arpaka.const_get(:Lexer, false).new("def f(value, ...)").each.to_a + assert_equal(:tBDOT3, tokens[5].first) end test "source lexer treats predicate and bang methods as identifiers" do From a55292b52632fc597674b935a35bfe4f5a726522 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 12:10:27 +0900 Subject: [PATCH 039/157] Lex explicit super calls --- lib/lrama/ruby/languages/ruby/lexer.rb | 2 +- test/lrama/ruby_language_test.rb | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 37e5c6f..ddb22c4 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -565,7 +565,7 @@ def operator_or_punctuation(start) if value == "(" || value == "[" || value == "{" @delimiter_depth += 1 if value == "(" - return [@begin_expression && ![".", :tCOLON2].include?(@previous) ? :tLPAREN : "(", nil] + return [@begin_expression && ![".", :tCOLON2, :keyword_super].include?(@previous) ? :tLPAREN : "(", nil] end return [:tLAMBEG, nil] if value == "{" && @previous == :tLAMBDA return [value == "[" ? (@begin_expression ? :tLBRACK : "[") : :tLBRACE, nil] diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index e83ea2c..12c1d85 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -233,6 +233,7 @@ def literal(value) test "super without arguments becomes a call node" do assert_equal(AST::Program.new([AST::Call.new(:super, [])]), parse([:keyword_super, nil])) + assert_equal(AST::Program.new([AST::Call.new(:super, [])]), Arpaka.parse_source("super()")) end test "parenthesized not becomes a logical negation" do From 829bf077c19b398fdbfd79d713346ea40459dda7 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 12:14:43 +0900 Subject: [PATCH 040/157] Fix qualified class path actions --- lib/lrama/ruby/languages/ruby/parse.y | 2 +- test/lrama/ruby_language_test.rb | 6 ++++-- tool/ruby/actions.rb | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/lrama/ruby/languages/ruby/parse.y b/lib/lrama/ruby/languages/ruby/parse.y index 84415c8..8e1512c 100644 --- a/lib/lrama/ruby/languages/ruby/parse.y +++ b/lib/lrama/ruby/languages/ruby/parse.y @@ -490,7 +490,7 @@ cpath: tCOLON3 cname %prec tCOLON3 { $$ = $2 }; /* upstream parse.y:3853: cpath: cname */ cpath: cname { $$ = $1 }; /* upstream parse.y:3858: cpath: primary_value "::" cname */ -cpath: primary_value tCOLON2 cname %prec tCOLON2 { $$ = $2 }; +cpath: primary_value tCOLON2 cname %prec tCOLON2 { $$ = $3 }; /* upstream parse.y:3864: fname: "local variable or method" */ fname: tIDENTIFIER %prec tIDENTIFIER { $$ = $1 }; /* upstream parse.y:3864: fname: "constant" */ diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 12c1d85..b945652 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -111,7 +111,7 @@ def literal(value) end test "setter method definitions preserve the equals suffix" do - expected = AST::Program.new([AST::Def.new(:"value=", [], [AST::BareCall.new(:value)])]) + expected = AST::Program.new([AST::Def.new(:"value=", :value, [AST::BareCall.new(:value)])]) assert_equal(expected, Arpaka.parse_source("def value=(value)\n value\nend")) end @@ -131,6 +131,8 @@ def literal(value) assert_equal(AST::Program.new([AST::ModuleDef.new(:Bar, [literal(2)])]), parse(*module_tokens)) assert_equal(AST::ClassDef.new(:Error, AST::Variable.new(:constant, :StandardError), []), Arpaka.parse_source("class Error < StandardError\nend").statements.first) + assert_equal(:TrixEditor, + Arpaka.parse_source("class Editor::TrixEditor\nend").statements.first.name) end test "simple string literals preserve their content" do @@ -446,7 +448,7 @@ def literal(value) end test "symbols can be built from variable-shaped values" do - builder = Lrama::Ruby::Languages::Ruby::Builder.new + builder = Lrama::Ruby::Languages::Ruby.const_get(:Builder, false).new assert_equal(AST::Literal.new(:value), builder.symbol(AST::Variable.new(:local, :value))) end diff --git a/tool/ruby/actions.rb b/tool/ruby/actions.rb index 6717b9d..aa9e8d9 100644 --- a/tool/ruby/actions.rb +++ b/tool/ruby/actions.rb @@ -558,7 +558,7 @@ module RubyGrammarActions 389 => ["ported", "@builder.def_node($1, $2, @builder.body_nodes($4))"], 629 => ["empty", "nil"], 636 => ["empty", "nil"], 149 => ["pass_through", "$1"], 150 => ["pass_through", "$1"], - 151 => ["pass_through", "$2"], 152 => ["pass_through", "$1"], 153 => ["pass_through", "$2"], + 151 => ["pass_through", "$2"], 152 => ["pass_through", "$1"], 153 => ["pass_through", "$3"], 382 => ["empty", "nil"], 384 => ["empty", "nil"], 386 => ["empty", "nil"], 383 => ["ported", "@builder.class_node($2, $3, $5)"], 387 => ["ported", "@builder.module_node($2, $4)"], From 22adc0d5551905fca27c6f248d070e0d176716f7 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 12:31:26 +0900 Subject: [PATCH 041/157] Normalize Ruby argument lists --- lib/lrama/ruby/languages/ruby/builder.rb | 5 +++++ lib/lrama/ruby/languages/ruby/parse.y | 8 ++++---- lib/lrama/ruby/languages/ruby/rules.json | 4 ++-- test/lrama/ruby_language_test.rb | 1 + tool/ruby/actions.rb | 8 ++++---- 5 files changed, 16 insertions(+), 10 deletions(-) diff --git a/lib/lrama/ruby/languages/ruby/builder.rb b/lib/lrama/ruby/languages/ruby/builder.rb index f68ea12..53b7b11 100644 --- a/lib/lrama/ruby/languages/ruby/builder.rb +++ b/lib/lrama/ruby/languages/ruby/builder.rb @@ -57,6 +57,11 @@ def call(name, arguments) AST::Call.new(name, arguments.freeze) end + def append_arguments(arguments, argument) + list = arguments.is_a?(Array) ? arguments : [arguments] + (list + [argument]).freeze + end + def string(value) AST::StringLiteral.new(value.to_s) end diff --git a/lib/lrama/ruby/languages/ruby/parse.y b/lib/lrama/ruby/languages/ruby/parse.y index 8e1512c..57c3f2d 100644 --- a/lib/lrama/ruby/languages/ruby/parse.y +++ b/lib/lrama/ruby/languages/ruby/parse.y @@ -844,9 +844,9 @@ args: arg_value { $$ = [$1].freeze }; /* upstream parse.y:4319: args: arg_splat */ args: arg_splat { $$ = $1 }; /* upstream parse.y:4324: args: args ',' arg_value */ -args: args ',' arg_value %prec ',' { $$ = ($1 + [$3]).freeze }; +args: args ',' arg_value %prec ',' { $$ = @builder.append_arguments($1, $3) }; /* upstream parse.y:4329: args: args ',' arg_splat */ -args: args ',' arg_splat %prec ',' { $$ = ($1 + [$3]).freeze }; +args: args ',' arg_splat %prec ',' { $$ = @builder.append_arguments($1, $3) }; /* upstream parse.y:4337: arg_splat: "*" arg_value */ arg_splat: tSTAR arg_value %prec tSTAR { $$ = $2 }; /* upstream parse.y:4342: arg_splat: "*" */ @@ -1796,9 +1796,9 @@ restarg_mark: '*' %prec '*' { $$ = nil }; /* upstream parse.y:6508: restarg_mark: "*" */ restarg_mark: tSTAR %prec tSTAR { $$ = nil }; /* upstream parse.y:6512: f_rest_arg: restarg_mark "local variable or method" */ -f_rest_arg: restarg_mark tIDENTIFIER %prec tIDENTIFIER { $$ = $2 }; +f_rest_arg: restarg_mark tIDENTIFIER %prec tIDENTIFIER { $$ = [$2].freeze }; /* upstream parse.y:6518: f_rest_arg: restarg_mark */ -f_rest_arg: restarg_mark { $$ = $1 }; +f_rest_arg: restarg_mark { $$ = [] }; /* upstream parse.y:6525: blkarg_mark: '&' */ blkarg_mark: '&' %prec '&' { $$ = nil }; /* upstream parse.y:6526: blkarg_mark: "&" */ diff --git a/lib/lrama/ruby/languages/ruby/rules.json b/lib/lrama/ruby/languages/ruby/rules.json index a86029c..ca5e0c4 100644 --- a/lib/lrama/ruby/languages/ruby/rules.json +++ b/lib/lrama/ruby/languages/ruby/rules.json @@ -4833,13 +4833,13 @@ "id": 806, "rule": "f_rest_arg: restarg_mark \"local variable or method\"", "line": 6512, - "status": "pass_through" + "status": "ported" }, { "id": 807, "rule": "f_rest_arg: restarg_mark", "line": 6518, - "status": "pass_through" + "status": "empty" }, { "id": 808, diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index b945652..038a685 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -445,6 +445,7 @@ def literal(value) Arpaka.parse_source("f(*args)").statements.first) assert_equal(AST::Call.new(:map, AST::Literal.new(:to_s)), Arpaka.parse_source("map(&:to_s)").statements.first) + assert_nothing_raised { Arpaka.parse_source("f(1, *args)") } end test "symbols can be built from variable-shaped values" do diff --git a/tool/ruby/actions.rb b/tool/ruby/actions.rb index aa9e8d9..0a5984d 100644 --- a/tool/ruby/actions.rb +++ b/tool/ruby/actions.rb @@ -287,7 +287,7 @@ module RubyGrammarActions 810 => ["pass_through", "$2"], 820 => ["pass_through", "$3"], 828 => ["pass_through", "$4"], - 806 => ["pass_through", "$2"], + 806 => ["ported", "[$2].freeze"], 802 => ["pass_through", "$2"], 835 => ["pass_through", "$1"], 831 => ["pass_through", "$1"], @@ -351,7 +351,7 @@ module RubyGrammarActions 320 => ["ported", "(($1 || []) + ($3 || [])).freeze"], 324 => ["pass_through", "$2"], 326 => ["pass_through", "$2"], - 331 => ["ported", "($1 + [$3]).freeze"], + 331 => ["ported", "@builder.append_arguments($1, $3)"], 332 => ["pass_through", "$2"], 336 => ["ported", "($1 + [$3]).freeze"], 337 => ["ported", "($1 + [$4]).freeze"], @@ -523,7 +523,7 @@ module RubyGrammarActions 301 => ["ported", "($1 + $3).freeze"], 302 => ["pass_through", "$1"], 328 => ["ported", "[$1].freeze"], - 330 => ["ported", "($1 + [$3]).freeze"], + 330 => ["ported", "@builder.append_arguments($1, $3)"], 347 => ["pass_through", "$1"], 357 => ["ported", "@builder.array($2)"], 358 => ["ported", "@builder.hash($2)"], @@ -623,7 +623,7 @@ module RubyGrammarActions 796 => ["pass_through", "$1"], 798 => ["pass_through", "$1"], 799 => ["empty", "nil"], 800 => ["empty", "nil"], 801 => ["pass_through", "$1"], 803 => ["pass_through", "$1"], 804 => ["empty", "nil"], 805 => ["empty", "nil"], - 807 => ["pass_through", "$1"], 808 => ["empty", "nil"], 809 => ["empty", "nil"], + 807 => ["empty", "[]"], 808 => ["empty", "nil"], 809 => ["empty", "nil"], 812 => ["pass_through", "$1"], 813 => ["empty", "nil"], 814 => ["pass_through", "$1"], 815 => ["pass_through", "$1"], 816 => ["pass_through", "$1"], 817 => ["pass_through", "$1"], 818 => ["pass_through", "$1"], From 631a3c73a323583c793b2a87ad939e6b26cf360f Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 12:40:06 +0900 Subject: [PATCH 042/157] Lex do after parenthesized calls --- lib/lrama/ruby/languages/ruby/lexer.rb | 2 ++ test/lrama/ruby_language_test.rb | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index ddb22c4..235715e 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -240,6 +240,8 @@ def do_token if @condition_do @condition_do = false :keyword_do_cond + elsif @previous == ")" + :keyword_do elsif @previous == :tLAMBDA :keyword_do_LAMBDA else diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 038a685..7850336 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -462,6 +462,12 @@ def literal(value) assert_equal([:tIDENTIFIER, :tLSHFT, :tIDENTIFIER, :keyword_do_block], tokens[0, 4].map(&:first)) end + test "source lexer uses ordinary do after parenthesized calls" do + tokens = Arpaka.const_get(:Lexer, false).new("each(1) do\nend").each.to_a + assert_equal(:keyword_do, tokens.map(&:first)[-3]) + assert_instance_of(AST::Call, Arpaka.parse_source("each(1) do\nend").statements.first) + end + test "source lexer recognizes operator method names and top-level constants" do assert_equal(AST::Def.new(:[], [], [AST::BareCall.new(:x)]), Arpaka.parse_source("def [](x); x; end").statements.first) From 073afc1864351ec101b74841b36d023d5eabd751 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 12:41:12 +0900 Subject: [PATCH 043/157] Ignore newlines after logical operators --- lib/lrama/ruby/languages/ruby/lexer.rb | 1 + test/lrama/ruby_language_test.rb | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 235715e..cd8fcd4 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -165,6 +165,7 @@ def skip_space_and_comments def newline_ignored? ignored = @delimiter_depth.positive? || @previous == "\n" || ["+", "-", "*", "/", "%", "=", "?", ":", ",", ".", "&&", "||", "=>"].include?(@previous) || + [:tANDOP, :tOROP, :tMATCH, :tNMATCH, :tASSOC].include?(@previous) || next_word_is_terminator? @condition_line = false if ignored && @condition_line ignored diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 7850336..03d353d 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -468,6 +468,11 @@ def literal(value) assert_instance_of(AST::Call, Arpaka.parse_source("each(1) do\nend").statements.first) end + test "source lexer ignores newlines after logical operators" do + assert_nothing_raised { Arpaka.parse_source("left &&\nright\n") } + assert_nothing_raised { Arpaka.parse_source("left ||\nright\n") } + end + test "source lexer recognizes operator method names and top-level constants" do assert_equal(AST::Def.new(:[], [], [AST::BareCall.new(:x)]), Arpaka.parse_source("def [](x); x; end").statements.first) From 44771ed6edf73057fdd3fef3ca5111a18b666094 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 14:53:51 +0900 Subject: [PATCH 044/157] Lex symbols after predicate identifiers --- lib/lrama/ruby/languages/ruby/lexer.rb | 2 +- test/lrama/ruby_language_test.rb | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index cd8fcd4..58388e5 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -324,7 +324,7 @@ def character_or_question(start) end def symbol_or_colon(start) - symbol_position = @begin_expression || (@previous == :tIDENTIFIER && @ternary_depth.zero?) + symbol_position = @begin_expression || ([:tIDENTIFIER, :tFID].include?(@previous) && @ternary_depth.zero?) return operator_or_punctuation(start) unless symbol_position && byte(1) && byte(1) != 58 advance if byte == 39 || byte == 34 diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 03d353d..e3a8a42 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -479,6 +479,10 @@ def literal(value) assert_equal(:Foo, Arpaka.parse_source("::Foo").statements.first) end + test "source lexer accepts symbols after predicate-like identifiers" do + assert_nothing_raised { Arpaka.parse_source("alias :merge! :update\n") } + end + test "single quoted heredoc keeps interpolation literal" do assert_equal(AST::StringLiteral.new("\#{x}\n"), Arpaka.parse_source("<<'TEXT'\n\#{x}\nTEXT\n").statements.first) From e7735c9ec8aa1eb699d7ccb67ce7658bb69fa68b Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 16:23:33 +0900 Subject: [PATCH 045/157] Lex brace blocks after parenthesized calls --- lib/lrama/ruby/languages/ruby/lexer.rb | 2 +- test/lrama/ruby_language_test.rb | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 58388e5..abc8360 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -571,7 +571,7 @@ def operator_or_punctuation(start) return [@begin_expression && ![".", :tCOLON2, :keyword_super].include?(@previous) ? :tLPAREN : "(", nil] end return [:tLAMBEG, nil] if value == "{" && @previous == :tLAMBDA - return [value == "[" ? (@begin_expression ? :tLBRACK : "[") : :tLBRACE, nil] + return [value == "[" ? (@begin_expression ? :tLBRACK : "[") : (@previous == ")" ? "{" : :tLBRACE), nil] elsif value == ")" || value == "]" || value == "}" @delimiter_depth -= 1 if @delimiter_depth.positive? elsif value == "-" && @begin_expression diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index e3a8a42..12a0164 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -468,6 +468,12 @@ def literal(value) assert_instance_of(AST::Call, Arpaka.parse_source("each(1) do\nend").statements.first) end + test "source lexer uses brace blocks after parenthesized calls" do + tokens = Arpaka.const_get(:Lexer, false).new("each(1) { |value| value }\n").each.to_a + assert_equal("{", tokens.map(&:first)[4]) + assert_nothing_raised { Arpaka.parse_source("each(1) { |value| value }\n") } + end + test "source lexer ignores newlines after logical operators" do assert_nothing_raised { Arpaka.parse_source("left &&\nright\n") } assert_nothing_raised { Arpaka.parse_source("left ||\nright\n") } From da4a375b06d3f1539892dc1459e4732291975094 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 16:36:11 +0900 Subject: [PATCH 046/157] Lex lambda bodies after argument lists --- lib/lrama/ruby/languages/ruby/lexer.rb | 10 +++++++++- test/lrama/ruby_language_test.rb | 6 ++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index abc8360..a6e7163 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -64,6 +64,7 @@ def initialize(source, filename: "(ruby)") @condition_line = false @ternary_depth = 0 @class_superclass = false + @lambda_pending = false end def each @@ -241,6 +242,9 @@ def do_token if @condition_do @condition_do = false :keyword_do_cond + elsif @lambda_pending + @lambda_pending = false + :keyword_do_LAMBDA elsif @previous == ")" :keyword_do elsif @previous == :tLAMBDA @@ -555,6 +559,7 @@ def operator_or_punctuation(start) else OP_TOKENS.fetch(text, :tOP_ASGN) end + @lambda_pending = true if token == :tLAMBDA return [token, nil] end value = byte.chr @@ -570,7 +575,10 @@ def operator_or_punctuation(start) if value == "(" return [@begin_expression && ![".", :tCOLON2, :keyword_super].include?(@previous) ? :tLPAREN : "(", nil] end - return [:tLAMBEG, nil] if value == "{" && @previous == :tLAMBDA + if value == "{" && @lambda_pending + @lambda_pending = false + return [:tLAMBEG, nil] + end return [value == "[" ? (@begin_expression ? :tLBRACK : "[") : (@previous == ")" ? "{" : :tLBRACE), nil] elsif value == ")" || value == "]" || value == "}" @delimiter_depth -= 1 if @delimiter_depth.positive? diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 12a0164..e8b0686 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -474,6 +474,12 @@ def literal(value) assert_nothing_raised { Arpaka.parse_source("each(1) { |value| value }\n") } end + test "source lexer recognizes lambda bodies after lambda arguments" do + tokens = Arpaka.const_get(:Lexer, false).new("->(value) { value }\n").each.to_a + assert_includes(tokens.map(&:first), :tLAMBEG) + assert_nothing_raised { Arpaka.parse_source("->(value) { value }\n") } + end + test "source lexer ignores newlines after logical operators" do assert_nothing_raised { Arpaka.parse_source("left &&\nright\n") } assert_nothing_raised { Arpaka.parse_source("left ||\nright\n") } From 64102e17cb26ac8928a4211c745dbb49e12ed07e Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 16:40:02 +0900 Subject: [PATCH 047/157] Normalize splat argument lists --- lib/lrama/ruby/languages/ruby/parse.y | 2 +- lib/lrama/ruby/languages/ruby/rules.json | 2 +- test/lrama/ruby_language_test.rb | 2 +- tool/ruby/actions.rb | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/lrama/ruby/languages/ruby/parse.y b/lib/lrama/ruby/languages/ruby/parse.y index 57c3f2d..8e0cf49 100644 --- a/lib/lrama/ruby/languages/ruby/parse.y +++ b/lib/lrama/ruby/languages/ruby/parse.y @@ -848,7 +848,7 @@ args: args ',' arg_value %prec ',' { $$ = @builder.append_arguments($1, $3) }; /* upstream parse.y:4329: args: args ',' arg_splat */ args: args ',' arg_splat %prec ',' { $$ = @builder.append_arguments($1, $3) }; /* upstream parse.y:4337: arg_splat: "*" arg_value */ -arg_splat: tSTAR arg_value %prec tSTAR { $$ = $2 }; +arg_splat: tSTAR arg_value %prec tSTAR { $$ = [$2].freeze }; /* upstream parse.y:4342: arg_splat: "*" */ arg_splat: tSTAR %prec tSTAR { $$ = nil }; /* upstream parse.y:4350: mrhs_arg: mrhs */ diff --git a/lib/lrama/ruby/languages/ruby/rules.json b/lib/lrama/ruby/languages/ruby/rules.json index ca5e0c4..640a7de 100644 --- a/lib/lrama/ruby/languages/ruby/rules.json +++ b/lib/lrama/ruby/languages/ruby/rules.json @@ -1989,7 +1989,7 @@ "id": 332, "rule": "arg_splat: \"*\" arg_value", "line": 4337, - "status": "pass_through" + "status": "ported" }, { "id": 333, diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index e8b0686..a6e308a 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -441,7 +441,7 @@ def literal(value) end test "source lexer distinguishes splat and block argument operators" do - assert_equal(AST::Call.new(:f, AST::BareCall.new(:args)), + assert_equal(AST::Call.new(:f, [AST::BareCall.new(:args)]), Arpaka.parse_source("f(*args)").statements.first) assert_equal(AST::Call.new(:map, AST::Literal.new(:to_s)), Arpaka.parse_source("map(&:to_s)").statements.first) diff --git a/tool/ruby/actions.rb b/tool/ruby/actions.rb index 0a5984d..f838391 100644 --- a/tool/ruby/actions.rb +++ b/tool/ruby/actions.rb @@ -352,7 +352,7 @@ module RubyGrammarActions 324 => ["pass_through", "$2"], 326 => ["pass_through", "$2"], 331 => ["ported", "@builder.append_arguments($1, $3)"], - 332 => ["pass_through", "$2"], + 332 => ["ported", "[$2].freeze"], 336 => ["ported", "($1 + [$3]).freeze"], 337 => ["ported", "($1 + [$4]).freeze"], 338 => ["ported", "[$2].freeze"], From 4f442c9f77d4877bbdb5323a6fdd403be728d5fb Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 16:41:47 +0900 Subject: [PATCH 048/157] Lex escaped line continuations --- lib/lrama/ruby/languages/ruby/lexer.rb | 4 ++++ test/lrama/ruby_language_test.rb | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index a6e7163..3002402 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -157,6 +157,10 @@ def skip_space_and_comments while [9, 11, 12, 13, 32].include?(byte) advance end + if byte == 92 && byte(1) == 10 + advance(2) + next + end break unless byte == 35 advance advance while !eof? && byte != 10 diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index a6e308a..5a3dec1 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -485,6 +485,12 @@ def literal(value) assert_nothing_raised { Arpaka.parse_source("left ||\nright\n") } end + test "source lexer accepts escaped newlines between expressions" do + tokens = Arpaka.const_get(:Lexer, false).new("\"left\" \\\n\"right\"\n").each.to_a + assert_equal([:tSTRING_BEG, :tSTRING_CONTENT, :tSTRING_END, :tSTRING_BEG], tokens.map(&:first)[0, 4]) + assert_nothing_raised { Arpaka.parse_source("\"left\" \\\n\"right\"\n") } + end + test "source lexer recognizes operator method names and top-level constants" do assert_equal(AST::Def.new(:[], [], [AST::BareCall.new(:x)]), Arpaka.parse_source("def [](x); x; end").statements.first) From cfe6cd52f1c0b776d750bf3fb842c8fb65991b9c Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 17:03:42 +0900 Subject: [PATCH 049/157] Resolve block argument grammar conflict --- lib/lrama/ruby/languages/ruby/parse.y | 5 +++- test/lrama/ruby_language_test.rb | 5 ++++ tool/ruby/grammar.rb | 39 ++++++++++++++++++++++++--- 3 files changed, 44 insertions(+), 5 deletions(-) diff --git a/lib/lrama/ruby/languages/ruby/parse.y b/lib/lrama/ruby/languages/ruby/parse.y index 8e0cf49..57f7a22 100644 --- a/lib/lrama/ruby/languages/ruby/parse.y +++ b/lib/lrama/ruby/languages/ruby/parse.y @@ -183,6 +183,7 @@ %right tUMINUS tUMINUS_NUM %right tPOW %right tUPLUS '!' '~' +%left keyword_do_block %start program %% /* upstream parse.y:3176: $@1: %empty */ @@ -814,7 +815,7 @@ opt_call_args: args ',' assocs ',' %prec ',' { $$ = $1 }; /* upstream parse.y:4212: opt_call_args: assocs ',' */ opt_call_args: assocs ',' %prec ',' { $$ = $1 }; /* upstream parse.y:3161: value_expr_command: command */ -value_expr_command: command { $$ = $1 }; +value_expr_command: command %prec tLBRACE_ARG { $$ = $1 }; /* upstream parse.y:4219: call_args: value_expr_command */ call_args: value_expr_command { $$ = $1 }; /* upstream parse.y:4224: call_args: def_endless_method_endless_command */ @@ -1891,3 +1892,5 @@ terms: term { $$ = nil }; terms: terms ';' %prec ';' { $$ = nil }; /* upstream parse.y:6728: none: %empty */ none: %empty { $$ = nil }; +/* arpaka extension: block call as a parenthesized argument */ +call_args: block_call { $$ = [$1].freeze }; diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 5a3dec1..c3eeef7 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -468,6 +468,11 @@ def literal(value) assert_instance_of(AST::Call, Arpaka.parse_source("each(1) do\nend").statements.first) end + test "grammar precedence accepts block calls as parenthesized arguments" do + tree = Arpaka.parse_source("call(lambda do\nend)\n") + assert_equal(AST::Call.new(:call, [AST::Call.new(:lambda, [])]), tree.statements.first) + end + test "source lexer uses brace blocks after parenthesized calls" do tokens = Arpaka.const_get(:Lexer, false).new("each(1) { |value| value }\n").each.to_a assert_equal("{", tokens.map(&:first)[4]) diff --git a/tool/ruby/grammar.rb b/tool/ruby/grammar.rb index 147d136..733ee26 100644 --- a/tool/ruby/grammar.rb +++ b/tool/ruby/grammar.rb @@ -11,6 +11,9 @@ module RubyGrammar ROOT = File.expand_path("../..", __dir__) DESTINATION = File.join(ROOT, "lib/lrama/ruby/languages/ruby") REVISION = "37d60dd3241d5fdb10ca43f36077f5d556ae1b8d" + EXTENDED_RULES = { + ["value_expr_command", ["command"]] => "tLBRACE_ARG" + }.freeze def self.parse(source, filename) grammar = Lrama::Parser.new(source, filename).parse @@ -82,17 +85,27 @@ def self.artifacts grammar.precedences.group_by(&:precedence).sort.each do |_level, group| lines << "%#{group.first.type} #{group.map(&:s_value).join(' ')}" end + lines << "%left keyword_do_block" lines << "%start #{name.call(grammar.rules.first.rhs.first)}" lines << "%%" metadata = inventory.to_h { |rule| [rule.fetch("id"), rule] } rules = grammar.rules.reject(&:initial_rule?) rules.each do |rule| rhs = rule.rhs.empty? ? "%empty" : rule.rhs.map(&name).join(" ") - precedence = rule.precedence_sym ? " %prec #{name.call(rule.precedence_sym)}" : "" + extended_precedence = EXTENDED_RULES[[rule.lhs.id.s_value, rule.rhs.map { |symbol| symbol.id.s_value }]] + precedence = if rule.precedence_sym + " %prec #{name.call(rule.precedence_sym)}" + elsif extended_precedence + " %prec #{extended_precedence}" + else + "" + end expression = RubyGrammarActions::ACTIONS[rule.id]&.last || "@builder.unsupported(#{rule.id})" lines << "/* upstream parse.y:#{metadata.fetch(rule.id).fetch('line')}: #{rule.as_comment} */" lines << "#{name.call(rule.lhs)}: #{rhs}#{precedence} { $$ = #{expression} };" end + lines << "/* arpaka extension: block call as a parenthesized argument */" + lines << "call_args: block_call { $$ = [$1].freeze };" source = lines.join("\n") + "\n" verify_structure(grammar, source, names) compact = rules.map do |rule| @@ -112,11 +125,29 @@ def self.verify_structure(original, source, names) [name.call(rule.lhs), rule.rhs.map(&name), rule.precedence_sym && name.call(rule.precedence_sym)] end.sort_by(&:inspect) end - raise "Production structure changed" unless signature.call(original, names) == signature.call(port, {}) + expected_rules = signature.call(original, names).map do |rule| + if rule[0] == names.fetch("value_expr_command") && rule[1] == [names.fetch("command")] + rule[0, 2] + ["tLBRACE_ARG"] + else + rule + end + end + expected_rules << ["call_args", ["block_call"], nil] + raise "Production structure changed" unless expected_rules.sort_by(&:inspect) == signature.call(port, {}).sort_by(&:inspect) terms = lambda do |grammar| grammar.terms.map { |symbol| [symbol.id.s_value, symbol.token_id, symbol.precedence&.type, symbol.precedence&.precedence] }.sort_by(&:inspect) end - raise "Tokens or precedence changed" unless terms.call(original) == terms.call(port) + expected_terms = terms.call(original).map do |term| + if term[0] == "keyword_do_block" + [term[0], term[1], :left, original.precedences.map(&:precedence).max + 1] + else + term + end + end + actual_terms = terms.call(port) + unless expected_terms.sort_by(&:inspect) == actual_terms + raise "Tokens or precedence changed: expected=#{expected_terms.sort_by(&:inspect).inspect} actual=#{actual_terms.inspect}" + end end def self.run(check:) @@ -128,7 +159,7 @@ def self.run(check:) File.write(path, contents) end end - puts(check ? "Ruby grammar verified (853 productions)" : "Ruby grammar regenerated") + puts(check ? "Ruby grammar verified (853 upstream productions + extensions)" : "Ruby grammar regenerated") end end From efe46da404650f4b05a41b107d6fa241e9ef57fd Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 17:09:09 +0900 Subject: [PATCH 050/157] Lex receiver method brace blocks --- lib/lrama/ruby/languages/ruby/lexer.rb | 10 ++++++++-- test/lrama/ruby_language_test.rb | 6 ++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 3002402..be10183 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -265,6 +265,11 @@ def no_argument_block? true end + def no_argument_brace_block? + return false if [".", :tCOLON2].include?(@previous_previous) + no_argument_block? + end + def number_token(start) base = 10 if byte == 48 && [120, 88, 98, 66, 111, 79].include?(byte(1)) @@ -567,7 +572,7 @@ def operator_or_punctuation(start) return [token, nil] end value = byte.chr - if value == "{" && no_argument_block? + if value == "{" && no_argument_brace_block? advance @delimiter_depth += 1 @pending.unshift([:tLBRACE_ARG, nil]) @@ -583,7 +588,8 @@ def operator_or_punctuation(start) @lambda_pending = false return [:tLAMBEG, nil] end - return [value == "[" ? (@begin_expression ? :tLBRACK : "[") : (@previous == ")" ? "{" : :tLBRACE), nil] + brace_block = @previous == ")" || [".", :tCOLON2].include?(@previous_previous) + return [value == "[" ? (@begin_expression ? :tLBRACK : "[") : (brace_block ? "{" : :tLBRACE), nil] elsif value == ")" || value == "]" || value == "}" @delimiter_depth -= 1 if @delimiter_depth.positive? elsif value == "-" && @begin_expression diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index c3eeef7..c29d80a 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -479,6 +479,12 @@ def literal(value) assert_nothing_raised { Arpaka.parse_source("each(1) { |value| value }\n") } end + test "source lexer uses brace blocks after receiver method calls" do + tokens = Arpaka.const_get(:Lexer, false).new("mutex.synchronize { 1 }\n").each.to_a + assert_equal("{", tokens.map(&:first)[3]) + assert_nothing_raised { Arpaka.parse_source("mutex.synchronize { 1 }\n") } + end + test "source lexer recognizes lambda bodies after lambda arguments" do tokens = Arpaka.const_get(:Lexer, false).new("->(value) { value }\n").each.to_a assert_includes(tokens.map(&:first), :tLAMBEG) From 7bca93340d39859c6c52d13b5727f6ce41e5418b Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 17:15:13 +0900 Subject: [PATCH 051/157] Accept block calls after argument lists --- lib/lrama/ruby/languages/ruby/parse.y | 1 + test/lrama/ruby_language_test.rb | 4 ++++ tool/ruby/grammar.rb | 7 ++++++- 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/languages/ruby/parse.y b/lib/lrama/ruby/languages/ruby/parse.y index 57f7a22..d89b365 100644 --- a/lib/lrama/ruby/languages/ruby/parse.y +++ b/lib/lrama/ruby/languages/ruby/parse.y @@ -1894,3 +1894,4 @@ terms: terms ';' %prec ';' { $$ = nil }; none: %empty { $$ = nil }; /* arpaka extension: block call as a parenthesized argument */ call_args: block_call { $$ = [$1].freeze }; +call_args: args ',' block_call %prec tLOWEST { $$ = ($1 + [$3]).freeze }; diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index c29d80a..be20166 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -471,6 +471,10 @@ def literal(value) test "grammar precedence accepts block calls as parenthesized arguments" do tree = Arpaka.parse_source("call(lambda do\nend)\n") assert_equal(AST::Call.new(:call, [AST::Call.new(:lambda, [])]), tree.statements.first) + + assert_nothing_raised do + Arpaka.parse_source("subscribe(x, handler, lambda do\n work\nend)\n") + end end test "source lexer uses brace blocks after parenthesized calls" do diff --git a/tool/ruby/grammar.rb b/tool/ruby/grammar.rb index 733ee26..bdef3fb 100644 --- a/tool/ruby/grammar.rb +++ b/tool/ruby/grammar.rb @@ -106,6 +106,7 @@ def self.artifacts end lines << "/* arpaka extension: block call as a parenthesized argument */" lines << "call_args: block_call { $$ = [$1].freeze };" + lines << "call_args: args ',' block_call %prec tLOWEST { $$ = ($1 + [$3]).freeze };" source = lines.join("\n") + "\n" verify_structure(grammar, source, names) compact = rules.map do |rule| @@ -133,7 +134,11 @@ def self.verify_structure(original, source, names) end end expected_rules << ["call_args", ["block_call"], nil] - raise "Production structure changed" unless expected_rules.sort_by(&:inspect) == signature.call(port, {}).sort_by(&:inspect) + expected_rules << ["call_args", ["args", "','", "block_call"], "tLOWEST"] + actual_rules = signature.call(port, {}) + unless expected_rules.sort_by(&:inspect) == actual_rules.sort_by(&:inspect) + raise "Production structure changed" + end terms = lambda do |grammar| grammar.terms.map { |symbol| [symbol.id.s_value, symbol.token_id, symbol.precedence&.type, symbol.precedence&.precedence] }.sort_by(&:inspect) end From 3a0a3a5ef933ea0939b0a9865485d2b1ee5ec58f Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 17:17:42 +0900 Subject: [PATCH 052/157] Lex ampersand operator symbols --- lib/lrama/ruby/languages/ruby/lexer.rb | 2 ++ test/lrama/ruby_language_test.rb | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index be10183..aead697 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -598,6 +598,8 @@ def operator_or_punctuation(start) return [:tUPLUS, nil] elsif value == "*" && @begin_expression return [:tSTAR, nil] + elsif value == "&" && @previous == :tSYMBEG + return [value, nil] elsif value == "&" && @begin_expression return [:tAMPER, nil] elsif value == "?" && !@begin_expression diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index be20166..127814f 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -516,6 +516,10 @@ def literal(value) assert_nothing_raised { Arpaka.parse_source("alias :merge! :update\n") } end + test "source lexer accepts ampersand operator symbols" do + assert_nothing_raised { Arpaka.parse_source("items.reduce(:&)\n") } + end + test "single quoted heredoc keeps interpolation literal" do assert_equal(AST::StringLiteral.new("\#{x}\n"), Arpaka.parse_source("<<'TEXT'\n\#{x}\nTEXT\n").statements.first) From 11e55b1e9a312757458494c37d36245a882fffa9 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 17:20:14 +0900 Subject: [PATCH 053/157] Lex modifiers after control keywords --- lib/lrama/ruby/languages/ruby/lexer.rb | 2 +- test/lrama/ruby_language_test.rb | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index aead697..352158a 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -224,7 +224,7 @@ def identifier_token(start) token = KEYWORDS[word] if token && [".", :tCOLON2].include?(@previous) token = :tFID - elsif !@begin_expression && { "if" => :modifier_if, "unless" => :modifier_unless, + elsif (!@begin_expression || [:keyword_return, :keyword_break, :keyword_next].include?(@previous)) && { "if" => :modifier_if, "unless" => :modifier_unless, "while" => :modifier_while, "until" => :modifier_until, "rescue" => :modifier_rescue }.key?(word) token = { "if" => :modifier_if, "unless" => :modifier_unless, diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 127814f..53c22e9 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -500,6 +500,12 @@ def literal(value) assert_nothing_raised { Arpaka.parse_source("left ||\nright\n") } end + test "source lexer recognizes modifiers after argumentless control keywords" do + tokens = Arpaka.const_get(:Lexer, false).new("return unless value\n").each.to_a + assert_equal(:modifier_unless, tokens[1].first) + assert_nothing_raised { Arpaka.parse_source("def stop\n return unless value\nend\n") } + end + test "source lexer accepts escaped newlines between expressions" do tokens = Arpaka.const_get(:Lexer, false).new("\"left\" \\\n\"right\"\n").each.to_a assert_equal([:tSTRING_BEG, :tSTRING_CONTENT, :tSTRING_END, :tSTRING_BEG], tokens.map(&:first)[0, 4]) From 0fda7cf3877dfde5109932f31e58e7b8ddee18d7 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 17:25:16 +0900 Subject: [PATCH 054/157] Lex operator method symbols --- lib/lrama/ruby/languages/ruby/lexer.rb | 2 +- test/lrama/ruby_language_test.rb | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 352158a..e4c36c7 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -556,7 +556,7 @@ def dedent_heredoc(body) def operator_or_punctuation(start) text = OPERATORS.sort_by { |operator| -operator.bytesize }.find { |operator| @source.byteslice(@index, operator.bytesize) == operator } - operator_method = [:keyword_def, ".", :tCOLON2].include?(@previous) + operator_method = [:keyword_def, ".", :tCOLON2, :tSYMBEG].include?(@previous) if text && !(begin_expression? && !operator_method && ["[]", "[]="].include?(text)) advance(text.bytesize) token = if text == "**" && @begin_expression diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 53c22e9..bb63d16 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -524,6 +524,7 @@ def literal(value) test "source lexer accepts ampersand operator symbols" do assert_nothing_raised { Arpaka.parse_source("items.reduce(:&)\n") } + assert_nothing_raised { Arpaka.parse_source("delegate :[], :[]=, to: :paths\n") } end test "single quoted heredoc keeps interpolation literal" do From 85c3533f1cce19d2d33388ad97cc9fc54c0c8254 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 17:29:47 +0900 Subject: [PATCH 055/157] Lex bitwise assignment operators --- lib/lrama/ruby/languages/ruby/lexer.rb | 2 +- test/lrama/ruby_language_test.rb | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index e4c36c7..f1fc41e 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -33,7 +33,7 @@ class Lexer "__ENCODING__" => :keyword__ENCODING__ }.freeze - OPERATORS = %w[[]= [] ... .. <=> === == != =~ !~ >= <= && || << >> ** => :: &. -> += -= *= /= %= **= <<= >>= &&= ||=].freeze + OPERATORS = %w[[]= [] ... .. <=> === == != =~ !~ >= <= && || << >> ** => :: &. -> += -= *= /= %= **= <<= >>= &&= ||= |= &= ^=].freeze OP_TOKENS = { "**" => :tPOW, "<=>" => :tCMP, "==" => :tEQ, "===" => :tEQQ, "!=" => :tNEQ, ">=" => :tGEQ, "<=" => :tLEQ, "&&" => :tANDOP, diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index bb63d16..170cfa1 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -527,6 +527,12 @@ def literal(value) assert_nothing_raised { Arpaka.parse_source("delegate :[], :[]=, to: :paths\n") } end + test "source lexer recognizes bitwise assignment operators" do + %w[|= &= ^=].each do |operator| + assert_nothing_raised { Arpaka.parse_source("value #{operator} other\n") } + end + end + test "single quoted heredoc keeps interpolation literal" do assert_equal(AST::StringLiteral.new("\#{x}\n"), Arpaka.parse_source("<<'TEXT'\n\#{x}\nTEXT\n").statements.first) From dd73bc355e381589166064ed084cde7ec4961a00 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 17:31:32 +0900 Subject: [PATCH 056/157] Avoid synthetic blocks after keyword values --- lib/lrama/ruby/languages/ruby/lexer.rb | 1 + test/lrama/ruby_language_test.rb | 1 + 2 files changed, 2 insertions(+) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index f1fc41e..5d64fee 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -261,6 +261,7 @@ def do_token def no_argument_block? return false unless [:tIDENTIFIER, :tCONSTANT, :tFID].include?(@previous) return false if [:tIDENTIFIER, :tCONSTANT, :tFID].include?(@previous_previous) + return false if [:tSYMBEG, :tLABEL].include?(@previous_previous) return false if @previous_previous == :tLSHFT true end diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 170cfa1..bed2906 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -458,6 +458,7 @@ def literal(value) Arpaka.parse_source("included do\n 1\nend").statements.first) assert_equal(AST::BareCall.new(:foo), Arpaka.parse_source("foo.bar do\n 1\nend").statements.first) + assert_nothing_raised { Arpaka.parse_source("foo before: :bar do |value| value end\n") } tokens = Arpaka.const_get(:Lexer, false).new("items << lambda do").each.to_a assert_equal([:tIDENTIFIER, :tLSHFT, :tIDENTIFIER, :keyword_do_block], tokens[0, 4].map(&:first)) end From 73c735ff4e8af9c80a60313d595ea7574ac84e6b Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 17:33:36 +0900 Subject: [PATCH 057/157] Lex setter symbols --- lib/lrama/ruby/languages/ruby/lexer.rb | 4 ++++ test/lrama/ruby_language_test.rb | 1 + 2 files changed, 5 insertions(+) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 5d64fee..64fb3a7 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -213,6 +213,10 @@ def identifier_token(start) advance return [:tFID, (word + "=").to_sym] end + if byte == 61 && @previous == :tSYMBEG + advance + return [:tIDENTIFIER, (word + "=").to_sym] + end if byte == 58 && byte(1) != 58 advance return [:tLABEL, word.to_sym] diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index bed2906..5a539d2 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -521,6 +521,7 @@ def literal(value) test "source lexer accepts symbols after predicate-like identifiers" do assert_nothing_raised { Arpaka.parse_source("alias :merge! :update\n") } + assert_nothing_raised { Arpaka.parse_source("alias :default_options= :default\n") } end test "source lexer accepts ampersand operator symbols" do From bbe897ef08ebc955762b22f5e0228295c05fd5c4 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 17:36:06 +0900 Subject: [PATCH 058/157] Preserve newlines before Ruby terminators --- lib/lrama/ruby/languages/ruby/lexer.rb | 3 +-- test/lrama/ruby_language_test.rb | 8 +++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 64fb3a7..6ad0b2b 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -170,8 +170,7 @@ def skip_space_and_comments def newline_ignored? ignored = @delimiter_depth.positive? || @previous == "\n" || ["+", "-", "*", "/", "%", "=", "?", ":", ",", ".", "&&", "||", "=>"].include?(@previous) || - [:tANDOP, :tOROP, :tMATCH, :tNMATCH, :tASSOC].include?(@previous) || - next_word_is_terminator? + [:tANDOP, :tOROP, :tMATCH, :tNMATCH, :tASSOC].include?(@previous) @condition_line = false if ignored && @condition_line ignored end diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 5a539d2..cf99960 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -463,9 +463,15 @@ def literal(value) assert_equal([:tIDENTIFIER, :tLSHFT, :tIDENTIFIER, :keyword_do_block], tokens[0, 4].map(&:first)) end + test "source lexer preserves newlines before terminators" do + assert_nothing_raised { Arpaka.parse_source("def empty\nend\n") } + assert_nothing_raised { Arpaka.parse_source("if condition\nend\n") } + assert_nothing_raised { Arpaka.parse_source("foo do\nend\n") } + end + test "source lexer uses ordinary do after parenthesized calls" do tokens = Arpaka.const_get(:Lexer, false).new("each(1) do\nend").each.to_a - assert_equal(:keyword_do, tokens.map(&:first)[-3]) + assert_equal(:keyword_do, tokens.map(&:first)[-4]) assert_instance_of(AST::Call, Arpaka.parse_source("each(1) do\nend").statements.first) end From 10260490e329d572bebdbd8552a65ca35ab3c0a0 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 17:43:43 +0900 Subject: [PATCH 059/157] Preserve heredoc suffix tokens --- lib/lrama/ruby/languages/ruby/lexer.rb | 9 +++++++++ test/lrama/ruby_language_test.rb | 3 +++ 2 files changed, 12 insertions(+) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 6ad0b2b..d819ab1 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -523,7 +523,9 @@ def heredoc_token(start) delimiter = @source.byteslice(delimiter_start, @index - delimiter_start) end fail!("invalid heredoc identifier", start) if delimiter.empty? + suffix_start = @index advance while byte && byte != 10 + suffix = @source.byteslice(suffix_start, @index - suffix_start).force_encoding(Encoding::UTF_8) advance if byte == 10 body = +"" interpolate = quote != 39 @@ -544,6 +546,13 @@ def heredoc_token(start) end body = dedent_heredoc(body) if squiggly @pending = [[:tSTRING_CONTENT, body], [:tSTRING_END, nil]] + unless suffix.empty? + suffix_tokens = self.class.new(suffix, filename: @filename).each.to_a + suffix_tokens.pop if suffix_tokens.last == [0, nil] + @pending.concat(suffix_tokens) + @pending << ["\n", nil] + @delimiter_depth -= suffix.count(")") - suffix.count("(") + end [quote == 96 ? :tXSTRING_BEG : :tSTRING_BEG, nil] end diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index cf99960..0121691 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -417,6 +417,9 @@ def literal(value) Arpaka.parse_source("< Date: Sat, 12 Sep 2026 17:52:10 +0900 Subject: [PATCH 060/157] Ignore newlines after operator assignments --- lib/lrama/ruby/languages/ruby/lexer.rb | 2 +- test/lrama/ruby_language_test.rb | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index d819ab1..717be63 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -170,7 +170,7 @@ def skip_space_and_comments def newline_ignored? ignored = @delimiter_depth.positive? || @previous == "\n" || ["+", "-", "*", "/", "%", "=", "?", ":", ",", ".", "&&", "||", "=>"].include?(@previous) || - [:tANDOP, :tOROP, :tMATCH, :tNMATCH, :tASSOC].include?(@previous) + [:tANDOP, :tOROP, :tMATCH, :tNMATCH, :tASSOC, :tOP_ASGN].include?(@previous) @condition_line = false if ignored && @condition_line ignored end diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 0121691..ca9bf2c 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -472,6 +472,10 @@ def literal(value) assert_nothing_raised { Arpaka.parse_source("foo do\nend\n") } end + test "source lexer ignores newlines after operator assignments" do + assert_nothing_raised { Arpaka.parse_source("value ||=\n if condition\n fallback\n end\n") } + end + test "source lexer uses ordinary do after parenthesized calls" do tokens = Arpaka.const_get(:Lexer, false).new("each(1) do\nend").each.to_a assert_equal(:keyword_do, tokens.map(&:first)[-4]) From cefa56cc94cce752e10daf7b072c24efa447bd46 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 17:53:50 +0900 Subject: [PATCH 061/157] Preserve newlines after heredocs --- lib/lrama/ruby/languages/ruby/lexer.rb | 2 +- test/lrama/ruby_language_test.rb | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 717be63..b777fb2 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -550,9 +550,9 @@ def heredoc_token(start) suffix_tokens = self.class.new(suffix, filename: @filename).each.to_a suffix_tokens.pop if suffix_tokens.last == [0, nil] @pending.concat(suffix_tokens) - @pending << ["\n", nil] @delimiter_depth -= suffix.count(")") - suffix.count("(") end + @pending << ["\n", nil] [quote == 96 ? :tXSTRING_BEG : :tSTRING_BEG, nil] end diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index ca9bf2c..303056d 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -420,6 +420,9 @@ def literal(value) assert_nothing_raised do Arpaka.parse_source("warn(<<~MSG.squish)\n warning\nMSG\naddress\n") end + assert_nothing_raised do + Arpaka.parse_source("warn <<~MSG\n warning\nMSG\naddress\n") + end end test "source lexer distinguishes shifts and command symbols" do From 4e9b68f7d7266b699067a35f9792f8c6891eecb2 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 17:55:06 +0900 Subject: [PATCH 062/157] Preserve newlines in synthetic blocks --- lib/lrama/ruby/languages/ruby/lexer.rb | 1 - test/lrama/ruby_language_test.rb | 6 ++++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index b777fb2..b2b7427 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -587,7 +587,6 @@ def operator_or_punctuation(start) value = byte.chr if value == "{" && no_argument_brace_block? advance - @delimiter_depth += 1 @pending.unshift([:tLBRACE_ARG, nil]) return [:tAMPER, nil] end diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 303056d..14a9596 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -506,6 +506,12 @@ def literal(value) assert_nothing_raised { Arpaka.parse_source("mutex.synchronize { 1 }\n") } end + test "source lexer preserves newlines inside command blocks" do + assert_nothing_raised do + Arpaka.parse_source("app = lambda { |env|\n req = Request.new(env)\n res = response(req)\n}\n") + end + end + test "source lexer recognizes lambda bodies after lambda arguments" do tokens = Arpaka.const_get(:Lexer, false).new("->(value) { value }\n").each.to_a assert_includes(tokens.map(&:first), :tLAMBEG) From b540e4d6bb1698c91368ec9a1fa05e8b8441d14b Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 17:56:45 +0900 Subject: [PATCH 063/157] Preserve newlines in brace blocks --- lib/lrama/ruby/languages/ruby/lexer.rb | 4 ++-- test/lrama/ruby_language_test.rb | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index b2b7427..a0e9d7a 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -592,7 +592,8 @@ def operator_or_punctuation(start) end advance if value == "(" || value == "[" || value == "{" - @delimiter_depth += 1 + brace_block = value == "{" && (@previous == ")" || [".", :tCOLON2].include?(@previous_previous)) + @delimiter_depth += 1 unless brace_block if value == "(" return [@begin_expression && ![".", :tCOLON2, :keyword_super].include?(@previous) ? :tLPAREN : "(", nil] end @@ -600,7 +601,6 @@ def operator_or_punctuation(start) @lambda_pending = false return [:tLAMBEG, nil] end - brace_block = @previous == ")" || [".", :tCOLON2].include?(@previous_previous) return [value == "[" ? (@begin_expression ? :tLBRACK : "[") : (brace_block ? "{" : :tLBRACE), nil] elsif value == ")" || value == "]" || value == "}" @delimiter_depth -= 1 if @delimiter_depth.positive? diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 14a9596..a491c15 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -504,6 +504,7 @@ def literal(value) tokens = Arpaka.const_get(:Lexer, false).new("mutex.synchronize { 1 }\n").each.to_a assert_equal("{", tokens.map(&:first)[3]) assert_nothing_raised { Arpaka.parse_source("mutex.synchronize { 1 }\n") } + assert_nothing_raised { Arpaka.parse_source("mutex.synchronize {\n first\n second\n}\n") } end test "source lexer preserves newlines inside command blocks" do From 4ae0693b9943c05a36fcd49d3333f12a6663a622 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 17:58:20 +0900 Subject: [PATCH 064/157] Lex class variables --- lib/lrama/ruby/languages/ruby/lexer.rb | 1 + test/lrama/ruby_language_test.rb | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index a0e9d7a..f5519df 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -317,6 +317,7 @@ def digit_byte?(value, base) def variable_token(start) marker = byte advance + advance if marker == 64 && byte == 64 if marker == 36 && byte && byte.between?(48, 57) advance while byte && byte.between?(48, 57) return [:tNTH_REF, @source.byteslice(start + 1, @index - start - 1).to_i] diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index a491c15..0ba5a0c 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -558,6 +558,10 @@ def literal(value) end end + test "source lexer recognizes class variables" do + assert_nothing_raised { Arpaka.parse_source("@@templates = {}\n@@templates\n") } + end + test "single quoted heredoc keeps interpolation literal" do assert_equal(AST::StringLiteral.new("\#{x}\n"), Arpaka.parse_source("<<'TEXT'\n\#{x}\nTEXT\n").statements.first) From 45841f4e9bde28f590bf0b5432df39ee46a3cf22 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 18:01:00 +0900 Subject: [PATCH 065/157] Continue method chains after newlines --- lib/lrama/ruby/languages/ruby/lexer.rb | 9 ++++++++- test/lrama/ruby_language_test.rb | 4 ++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index f5519df..fa23237 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -170,7 +170,8 @@ def skip_space_and_comments def newline_ignored? ignored = @delimiter_depth.positive? || @previous == "\n" || ["+", "-", "*", "/", "%", "=", "?", ":", ",", ".", "&&", "||", "=>"].include?(@previous) || - [:tANDOP, :tOROP, :tMATCH, :tNMATCH, :tASSOC, :tOP_ASGN].include?(@previous) + [:tANDOP, :tOROP, :tMATCH, :tNMATCH, :tASSOC, :tOP_ASGN].include?(@previous) || + next_non_space_byte == 46 @condition_line = false if ignored && @condition_line ignored end @@ -184,6 +185,12 @@ def next_word_is_terminator? end end + def next_non_space_byte + position = @index + position += 1 while [9, 11, 12, 13, 32].include?(@source.getbyte(position)) + @source.getbyte(position) + end + def expression_begin_after(token) return true if token.nil? || token == "\n" return false if token == ")" || token == "]" || token == "}" diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 0ba5a0c..28fad65 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -536,6 +536,10 @@ def literal(value) assert_nothing_raised { Arpaka.parse_source("\"left\" \\\n\"right\"\n") } end + test "source lexer continues method chains after newlines" do + assert_nothing_raised { Arpaka.parse_source("value\n .first\n .to_s\n") } + end + test "source lexer recognizes operator method names and top-level constants" do assert_equal(AST::Def.new(:[], [], [AST::BareCall.new(:x)]), Arpaka.parse_source("def [](x); x; end").statements.first) From 5108fb138d76446eac86d3fc118e7b1bfada8fa8 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 18:03:05 +0900 Subject: [PATCH 066/157] Lex spaced ternary colons --- lib/lrama/ruby/languages/ruby/lexer.rb | 3 ++- test/lrama/ruby_language_test.rb | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index fa23237..34feb1d 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -350,7 +350,8 @@ def character_or_question(start) def symbol_or_colon(start) symbol_position = @begin_expression || ([:tIDENTIFIER, :tFID].include?(@previous) && @ternary_depth.zero?) - return operator_or_punctuation(start) unless symbol_position && byte(1) && byte(1) != 58 + return operator_or_punctuation(start) unless symbol_position && byte(1) && + ![9, 10, 11, 12, 13, 32].include?(byte(1)) && byte(1) != 58 advance if byte == 39 || byte == 34 quote = byte diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 28fad65..d7f3e31 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -549,6 +549,7 @@ def literal(value) test "source lexer accepts symbols after predicate-like identifiers" do assert_nothing_raised { Arpaka.parse_source("alias :merge! :update\n") } assert_nothing_raised { Arpaka.parse_source("alias :default_options= :default\n") } + assert_nothing_raised { Arpaka.parse_source("block_given? ? yield : @default_render.call\n") } end test "source lexer accepts ampersand operator symbols" do From 3e8b36e29f365ddf701af39fe99c1a2d0ba4107f Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 18:05:30 +0900 Subject: [PATCH 067/157] Preserve newlines in lambda blocks --- lib/lrama/ruby/languages/ruby/lexer.rb | 3 ++- test/lrama/ruby_language_test.rb | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 34feb1d..60db0d1 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -602,7 +602,8 @@ def operator_or_punctuation(start) advance if value == "(" || value == "[" || value == "{" brace_block = value == "{" && (@previous == ")" || [".", :tCOLON2].include?(@previous_previous)) - @delimiter_depth += 1 unless brace_block + lambda_block = value == "{" && @lambda_pending + @delimiter_depth += 1 unless brace_block || lambda_block if value == "(" return [@begin_expression && ![".", :tCOLON2, :keyword_super].include?(@previous) ? :tLPAREN : "(", nil] end diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index d7f3e31..5d00a49 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -517,6 +517,7 @@ def literal(value) tokens = Arpaka.const_get(:Lexer, false).new("->(value) { value }\n").each.to_a assert_includes(tokens.map(&:first), :tLAMBEG) assert_nothing_raised { Arpaka.parse_source("->(value) { value }\n") } + assert_nothing_raised { Arpaka.parse_source("-> {\n first\n second\n}\n") } end test "source lexer ignores newlines after logical operators" do From 9f3affc96a730e1a1bdaa4642efd94e45ffae996 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 18:06:34 +0900 Subject: [PATCH 068/157] Lex punctuation global variables --- lib/lrama/ruby/languages/ruby/lexer.rb | 5 +++++ test/lrama/ruby_language_test.rb | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 60db0d1..4740b23 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -325,6 +325,11 @@ def variable_token(start) marker = byte advance advance if marker == 64 && byte == 64 + if marker == 36 && [33, 38, 39, 43, 60, 62, 61, 96, 126].include?(byte) + advance + text = @source.byteslice(start, @index - start) + return [:tGVAR, text.to_sym] + end if marker == 36 && byte && byte.between?(48, 57) advance while byte && byte.between?(48, 57) return [:tNTH_REF, @source.byteslice(start + 1, @index - start - 1).to_i] diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 5d00a49..58c0333 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -568,6 +568,11 @@ def literal(value) assert_nothing_raised { Arpaka.parse_source("@@templates = {}\n@@templates\n") } end + test "source lexer recognizes punctuation global variables" do + assert_nothing_raised { Arpaka.parse_source("-$`\n") } + assert_nothing_raised { Arpaka.parse_source("$&\n") } + end + test "single quoted heredoc keeps interpolation literal" do assert_equal(AST::StringLiteral.new("\#{x}\n"), Arpaka.parse_source("<<'TEXT'\n\#{x}\nTEXT\n").statements.first) From 920b1e5a3f9e5d2aa393d2bea59b7a69d06b2dea Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 18:14:29 +0900 Subject: [PATCH 069/157] Distinguish modulo from percent literals --- lib/lrama/ruby/languages/ruby/lexer.rb | 1 + test/lrama/ruby_language_test.rb | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 4740b23..9ac9c05 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -450,6 +450,7 @@ def read_regexp(start) end def percent_token(start) + return operator_or_punctuation(start) unless @begin_expression advance kind = byte if [113, 81, 119, 87, 105, 73, 114, 115, 120, 88].include?(kind) diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 58c0333..0732d63 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -573,6 +573,11 @@ def literal(value) assert_nothing_raised { Arpaka.parse_source("$&\n") } end + test "source lexer distinguishes modulo from percent literals" do + assert_nothing_raised { Arpaka.parse_source("value = (left + right) % 3\n") } + assert_nothing_raised { Arpaka.parse_source("value = %q(text)\n") } + end + test "single quoted heredoc keeps interpolation literal" do assert_equal(AST::StringLiteral.new("\#{x}\n"), Arpaka.parse_source("<<'TEXT'\n\#{x}\nTEXT\n").statements.first) From fd079d3502a44decd26ab5c1788cfa0986338f13 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 18:15:55 +0900 Subject: [PATCH 070/157] Lex percent regexp flags --- lib/lrama/ruby/languages/ruby/lexer.rb | 1 + test/lrama/ruby_language_test.rb | 1 + 2 files changed, 2 insertions(+) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 9ac9c05..b14c6fd 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -463,6 +463,7 @@ def percent_token(start) advance closing = { 40 => 41, 91 => 93, 123 => 125, 60 => 62 }.fetch(delimiter, delimiter) content = read_delimited(closing, start, interpolate: [81, 87, 73, 114, 120, 88].include?(kind)) + advance while kind == 114 && byte && byte.between?(97, 122) token = { 113 => :tSTRING_BEG, 81 => :tSTRING_BEG, 119 => :tWORDS_BEG, 87 => :tQWORDS_BEG, 105 => :tSYMBOLS_BEG, 73 => :tQSYMBOLS_BEG, 114 => :tREGEXP_BEG, 115 => :tSYMBEG, 120 => :tXSTRING_BEG, 88 => :tXSTRING_BEG }.fetch(kind) diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 0732d63..bfcbf9b 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -576,6 +576,7 @@ def literal(value) test "source lexer distinguishes modulo from percent literals" do assert_nothing_raised { Arpaka.parse_source("value = (left + right) % 3\n") } assert_nothing_raised { Arpaka.parse_source("value = %q(text)\n") } + assert_nothing_raised { Arpaka.parse_source("value = %r{pattern}i\n") } end test "single quoted heredoc keeps interpolation literal" do From a33a52474c0b15de978b652120b5b0e0432d2b63 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 18:17:08 +0900 Subject: [PATCH 071/157] Lex parenthesized yield arguments --- lib/lrama/ruby/languages/ruby/lexer.rb | 2 +- test/lrama/ruby_language_test.rb | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index b14c6fd..a352260 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -612,7 +612,7 @@ def operator_or_punctuation(start) lambda_block = value == "{" && @lambda_pending @delimiter_depth += 1 unless brace_block || lambda_block if value == "(" - return [@begin_expression && ![".", :tCOLON2, :keyword_super].include?(@previous) ? :tLPAREN : "(", nil] + return [@begin_expression && ![".", :tCOLON2, :keyword_super, :keyword_yield].include?(@previous) ? :tLPAREN : "(", nil] end if value == "{" && @lambda_pending @lambda_pending = false diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index bfcbf9b..67987db 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -573,6 +573,10 @@ def literal(value) assert_nothing_raised { Arpaka.parse_source("$&\n") } end + test "source lexer parses parenthesized yield arguments" do + assert_nothing_raised { Arpaka.parse_source("def each\n yield(value, other)\nend\n") } + end + test "source lexer distinguishes modulo from percent literals" do assert_nothing_raised { Arpaka.parse_source("value = (left + right) % 3\n") } assert_nothing_raised { Arpaka.parse_source("value = %q(text)\n") } From 9c178f1ddb2f55f110b8ac18e62ac2f7e5dbe081 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 18:20:43 +0900 Subject: [PATCH 072/157] Reset conditional do context at newlines --- lib/lrama/ruby/languages/ruby/lexer.rb | 1 + test/lrama/ruby_language_test.rb | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index a352260..5ad9471 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -124,6 +124,7 @@ def next_token end @class_superclass = false return next_token if newline_ignored? + @condition_do = false ["\n", nil] when 39, 34, 96 string_token(byte, start) diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 67987db..d4ac071 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -485,6 +485,11 @@ def literal(value) assert_instance_of(AST::Call, Arpaka.parse_source("each(1) do\nend").statements.first) end + test "source lexer does not carry conditional do across statements" do + tokens = Arpaka.const_get(:Lexer, false).new("while condition\n body\nend\nfoo do\nend\n").each.to_a + assert_equal(:keyword_do_block, tokens.map(&:first)[-5]) + end + test "grammar precedence accepts block calls as parenthesized arguments" do tree = Arpaka.parse_source("call(lambda do\nend)\n") assert_equal(AST::Call.new(:call, [AST::Call.new(:lambda, [])]), tree.statements.first) From f8a710b6e884bad6533f4fedeac674ebc9ff4d95 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 18:51:22 +0900 Subject: [PATCH 073/157] Recognize modifiers after block endings --- lib/lrama/ruby/languages/ruby/lexer.rb | 2 +- test/lrama/ruby_language_test.rb | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 5ad9471..955730d 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -235,7 +235,7 @@ def identifier_token(start) token = KEYWORDS[word] if token && [".", :tCOLON2].include?(@previous) token = :tFID - elsif (!@begin_expression || [:keyword_return, :keyword_break, :keyword_next].include?(@previous)) && { "if" => :modifier_if, "unless" => :modifier_unless, + elsif (!@begin_expression || [:keyword_return, :keyword_break, :keyword_next, :keyword_end].include?(@previous)) && { "if" => :modifier_if, "unless" => :modifier_unless, "while" => :modifier_while, "until" => :modifier_until, "rescue" => :modifier_rescue }.key?(word) token = { "if" => :modifier_if, "unless" => :modifier_unless, diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index d4ac071..e6935b5 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -536,6 +536,12 @@ def literal(value) assert_nothing_raised { Arpaka.parse_source("def stop\n return unless value\nend\n") } end + test "source lexer recognizes modifiers after block endings" do + tokens = Arpaka.const_get(:Lexer, false).new("foo do\nend unless value\n").each.to_a + assert_equal(:modifier_unless, tokens[5].first) + assert_nothing_raised { Arpaka.parse_source("foo do\nend unless value\n") } + end + test "source lexer accepts escaped newlines between expressions" do tokens = Arpaka.const_get(:Lexer, false).new("\"left\" \\\n\"right\"\n").each.to_a assert_equal([:tSTRING_BEG, :tSTRING_CONTENT, :tSTRING_END, :tSTRING_BEG], tokens.map(&:first)[0, 4]) From 118798fd4bd30d6ae2804bc01d86736455cac859 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 18:58:01 +0900 Subject: [PATCH 074/157] Lex quoted hash labels --- lib/lrama/ruby/languages/ruby/lexer.rb | 6 +++++- test/lrama/ruby_language_test.rb | 10 ++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 955730d..ba8ee93 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -171,6 +171,7 @@ def skip_space_and_comments def newline_ignored? ignored = @delimiter_depth.positive? || @previous == "\n" || ["+", "-", "*", "/", "%", "=", "?", ":", ",", ".", "&&", "||", "=>"].include?(@previous) || + @previous == :tLABEL || [:tANDOP, :tOROP, :tMATCH, :tNMATCH, :tASSOC, :tOP_ASGN].include?(@previous) || next_non_space_byte == 46 @condition_line = false if ignored && @condition_line @@ -371,7 +372,10 @@ def symbol_or_colon(start) def string_token(quote, start) advance content = read_quoted(quote, interpolate: quote != 39, start: start) - @pending = [[:tSTRING_CONTENT, content], [:tSTRING_END, nil]] + label = byte == 58 + advance if label + terminator = label ? :tLABEL_END : :tSTRING_END + @pending = [[:tSTRING_CONTENT, content], [terminator, nil]] [quote == 96 ? :tXSTRING_BEG : :tSTRING_BEG, nil] end diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index e6935b5..dc3322d 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -552,6 +552,16 @@ def literal(value) assert_nothing_raised { Arpaka.parse_source("value\n .first\n .to_s\n") } end + test "source lexer continues keyword arguments after label newlines" do + assert_nothing_raised do + Arpaka.parse_source("parse(file, context:\n build_context)\n") + end + end + + test "source lexer recognizes string hash labels" do + assert_nothing_raised { Arpaka.parse_source("{ \"key\": value }\n") } + end + test "source lexer recognizes operator method names and top-level constants" do assert_equal(AST::Def.new(:[], [], [AST::BareCall.new(:x)]), Arpaka.parse_source("def [](x); x; end").statements.first) From e5184d7ef3d69b5903b5bf61c10b9e9f4cd69a60 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 19:18:00 +0900 Subject: [PATCH 075/157] Lex unary operator symbols --- lib/lrama/ruby/languages/ruby/lexer.rb | 5 +++++ test/lrama/ruby_language_test.rb | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index ba8ee93..dd11c7e 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -589,6 +589,11 @@ def dedent_heredoc(body) end def operator_or_punctuation(start) + if @previous == :tSYMBEG && ["-@", "+@"].include?(@source.byteslice(@index, 2)) + value = @source.byteslice(@index, 2) + advance(2) + return [value == "-@" ? :tUMINUS : :tUPLUS, nil] + end text = OPERATORS.sort_by { |operator| -operator.bytesize }.find { |operator| @source.byteslice(@index, operator.bytesize) == operator } operator_method = [:keyword_def, ".", :tCOLON2, :tSYMBEG].include?(@previous) if text && !(begin_expression? && !operator_method && ["[]", "[]="].include?(text)) diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index dc3322d..114200d 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -562,6 +562,10 @@ def literal(value) assert_nothing_raised { Arpaka.parse_source("{ \"key\": value }\n") } end + test "source lexer recognizes unary operator symbols" do + assert_nothing_raised { Arpaka.parse_source("alias :-@ :deduplicate\n") } + end + test "source lexer recognizes operator method names and top-level constants" do assert_equal(AST::Def.new(:[], [], [AST::BareCall.new(:x)]), Arpaka.parse_source("def [](x); x; end").statements.first) From b67095cfe994ed9e8456f086e7573f55d21fd209 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 19:45:55 +0900 Subject: [PATCH 076/157] Lex slash operator method definitions --- lib/lrama/ruby/languages/ruby/lexer.rb | 1 + test/lrama/ruby_language_test.rb | 1 + 2 files changed, 2 insertions(+) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index dd11c7e..ce53378 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -414,6 +414,7 @@ def escape_sequence(start) end def regexp_or_operator(start) + return operator_or_punctuation(start) if @previous == :keyword_def if @begin_expression advance value = read_regexp(start) diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 114200d..a2ccab5 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -569,6 +569,7 @@ def literal(value) test "source lexer recognizes operator method names and top-level constants" do assert_equal(AST::Def.new(:[], [], [AST::BareCall.new(:x)]), Arpaka.parse_source("def [](x); x; end").statements.first) + assert_nothing_raised { Arpaka.parse_source("def /(other); other; end") } assert_equal(:Foo, Arpaka.parse_source("::Foo").statements.first) end From 3df56b16530d0e1d32bebb5049168fce552a72eb Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 20:17:19 +0900 Subject: [PATCH 077/157] Preserve receiver calls and interpolated strings --- README.md | 17 +++- lib/lrama/ruby/languages/ruby/ast.rb | 2 + lib/lrama/ruby/languages/ruby/builder.rb | 29 ++++-- lib/lrama/ruby/languages/ruby/lexer.rb | 73 ++++++++++++++-- lib/lrama/ruby/languages/ruby/parse.y | 22 ++--- lib/lrama/ruby/languages/ruby/rules.json | 14 +-- sig/lrama/ruby.rbs | 17 +++- test/lrama/ruby_language_test.rb | 18 +++- test/lrama/ruby_receiver_call_test.rb | 107 +++++++++++++++++++++++ tool/ruby/actions.rb | 22 ++--- 10 files changed, 273 insertions(+), 48 deletions(-) create mode 100644 test/lrama/ruby_receiver_call_test.rb diff --git a/README.md b/README.md index 7361ba5..6aa640b 100644 --- a/README.md +++ b/README.md @@ -154,10 +154,19 @@ an `AST::Program`. Its frozen `statements` array contains immutable Data nodes: An unassigned bare identifier is a `BareCall`; assignment registers a local before reading its right-hand side, including in `a = a`. -This is not yet a complete Ruby frontend. Strings, explicit calls, control flow, -methods, and multi-statement parentheses are unsupported. Nodes have no source -locations. Input must already reflect the lexical decisions Ruby's parser and -lexer normally make together; this API does not parse Ruby source text. +Explicit receiver calls without blocks produce +`ReceiverCall(receiver, operator, name, arguments)`. They preserve chained calls, +ordered positional arguments, the `.` / `::` / `&.` operator, and the implicit +`call` method in `handler.(message)`. Receiverless calls use `Call(name, arguments)`. + +Double quoted strings and backtick strings may contain simple interpolation and +produce `InterpolatedString(parts)`, where each part is a string or an AST node. + +This is not yet a complete Ruby frontend. Percent literal interpolation, blocks, +method argument scopes, qualified constant paths, and other constructs are still incomplete; a successful +parse does not guarantee that all source semantics appear in the AST. Nodes have +no source locations. Token input must already reflect the lexical decisions +Ruby's parser and lexer normally make together; use `parse_source` for source text. `Arpaka::ParseError` exposes `token` and `state`. `Arpaka::UnsupportedSyntax` exposes the original `rule` and upstream `line`. Both inherit from diff --git a/lib/lrama/ruby/languages/ruby/ast.rb b/lib/lrama/ruby/languages/ruby/ast.rb index 0bb8e50..4aa3ad2 100644 --- a/lib/lrama/ruby/languages/ruby/ast.rb +++ b/lib/lrama/ruby/languages/ruby/ast.rb @@ -19,7 +19,9 @@ module AST RangeLiteral = Data.define(:operator, :begin_value, :end_value) Ternary = Data.define(:condition, :then_value, :else_value) Call = Data.define(:name, :arguments) + ReceiverCall = Data.define(:receiver, :operator, :name, :arguments) StringLiteral = Data.define(:value) + InterpolatedString = Data.define(:parts) Index = Data.define(:receiver, :arguments) Loop = Data.define(:kind, :condition, :body) Control = Data.define(:kind) diff --git a/lib/lrama/ruby/languages/ruby/builder.rb b/lib/lrama/ruby/languages/ruby/builder.rb index 53b7b11..c6413a3 100644 --- a/lib/lrama/ruby/languages/ruby/builder.rb +++ b/lib/lrama/ruby/languages/ruby/builder.rb @@ -57,21 +57,29 @@ def call(name, arguments) AST::Call.new(name, arguments.freeze) end + def receiver_call(receiver, operator, name, arguments) + AST::ReceiverCall.new(receiver, operator, name, (arguments || []).freeze) + end + def append_arguments(arguments, argument) list = arguments.is_a?(Array) ? arguments : [arguments] (list + [argument]).freeze end def string(value) + if value.is_a?(Array) + return AST::InterpolatedString.new(value.freeze) if value.any? { |part| !part.is_a?(String) } + value = value.join + end AST::StringLiteral.new(value.to_s) end def regexp(value) - AST::RegexpLiteral.new(value.to_s) + AST::RegexpLiteral.new(value.is_a?(Array) ? value.join : value.to_s) end def xstring(value) - AST::XStringLiteral.new(value.to_s) + AST::XStringLiteral.new(value.is_a?(Array) ? value.join : value.to_s) end def for_node(variable, enumerable, body) @@ -104,11 +112,18 @@ def symbol(value) end def join_strings(parts) - parts.join + parts + end + + def interpolation(value) + value.is_a?(Array) && value.length == 1 ? value.first : value end def concat_strings(left, right) - AST::StringLiteral.new(left.value + right.value) + left_parts = left.is_a?(AST::InterpolatedString) ? left.parts : [left.value] + right_parts = right.is_a?(AST::InterpolatedString) ? right.parts : [right.value] + parts = left_parts + right_parts + parts.all? { |part| part.is_a?(String) } ? AST::StringLiteral.new(parts.join) : AST::InterpolatedString.new(parts.freeze) end def index(receiver, arguments) @@ -207,7 +222,11 @@ def array(elements) end def word_array(elements, symbols: false) - values = elements.map { |value| symbols ? value.to_sym : AST::StringLiteral.new(value.to_s) } + elements = elements.elements if elements.is_a?(AST::ArrayLiteral) + values = elements.map do |value| + value = value.value if value.is_a?(AST::StringLiteral) + symbols ? value.to_sym : AST::StringLiteral.new(value.to_s) + end AST::ArrayLiteral.new(values.freeze) end diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index ce53378..78e9d17 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -234,7 +234,7 @@ def identifier_token(start) return [:tAMPER, nil] end token = KEYWORDS[word] - if token && [".", :tCOLON2].include?(@previous) + if token && [".", :tCOLON2, :tANDDOT].include?(@previous) token = :tFID elsif (!@begin_expression || [:keyword_return, :keyword_break, :keyword_next, :keyword_end].include?(@previous)) && { "if" => :modifier_if, "unless" => :modifier_unless, "while" => :modifier_while, "until" => :modifier_until, @@ -371,14 +371,77 @@ def symbol_or_colon(start) def string_token(quote, start) advance - content = read_quoted(quote, interpolate: quote != 39, start: start) + content_tokens = read_interpolated_quoted(quote, start) label = byte == 58 advance if label terminator = label ? :tLABEL_END : :tSTRING_END - @pending = [[:tSTRING_CONTENT, content], [terminator, nil]] + @pending = content_tokens + @pending << [terminator, nil] [quote == 96 ? :tXSTRING_BEG : :tSTRING_BEG, nil] end + def read_interpolated_quoted(quote, start) + return [[:tSTRING_CONTENT, read_quoted(quote, interpolate: false, start: start)]] if quote == 39 + content = +"" + tokens = [] + until eof? + value = byte + if value == quote + advance + tokens << [:tSTRING_CONTENT, content] unless content.empty? + return tokens + elsif value == 92 + content << escape_sequence(start) + elsif value == 35 && byte(1) == 123 + tokens << [:tSTRING_CONTENT, content] unless content.empty? + content = +"" + advance(2) + expression = read_interpolation_source(start) + inner = self.class.new(expression, filename: @filename).each.to_a + inner.pop if inner.last == [0, nil] + tokens << [:tSTRING_DBEG, nil] + tokens.concat(inner) + tokens << [:tSTRING_DEND, nil] + else + content << value.chr + advance + end + end + fail!("unterminated literal", start) + end + + def read_interpolation_source(start) + begin_index = @index + depth = 1 + quote = nil + escaped = false + while !eof? + value = byte + if quote + if escaped + escaped = false + elsif value == 92 + escaped = true + elsif value == quote + quote = nil + end + elsif [39, 34, 96].include?(value) + quote = value + elsif value == 123 + depth += 1 + elsif value == 125 + depth -= 1 + if depth.zero? + expression = @source.byteslice(begin_index, @index - begin_index) + advance + return expression.force_encoding(Encoding::UTF_8) + end + end + advance + end + fail!("unterminated string interpolation", start) + end + def read_quoted(quote, interpolate:, start:) result = +"" until eof? @@ -596,7 +659,7 @@ def operator_or_punctuation(start) return [value == "-@" ? :tUMINUS : :tUPLUS, nil] end text = OPERATORS.sort_by { |operator| -operator.bytesize }.find { |operator| @source.byteslice(@index, operator.bytesize) == operator } - operator_method = [:keyword_def, ".", :tCOLON2, :tSYMBEG].include?(@previous) + operator_method = [:keyword_def, ".", :tCOLON2, :tANDDOT, :tSYMBEG].include?(@previous) if text && !(begin_expression? && !operator_method && ["[]", "[]="].include?(text)) advance(text.bytesize) token = if text == "**" && @begin_expression @@ -623,7 +686,7 @@ def operator_or_punctuation(start) lambda_block = value == "{" && @lambda_pending @delimiter_depth += 1 unless brace_block || lambda_block if value == "(" - return [@begin_expression && ![".", :tCOLON2, :keyword_super, :keyword_yield].include?(@previous) ? :tLPAREN : "(", nil] + return [@begin_expression && ![".", :tCOLON2, :tANDDOT, :keyword_super, :keyword_yield].include?(@previous) ? :tLPAREN : "(", nil] end if value == "{" && @lambda_pending @lambda_pending = false diff --git a/lib/lrama/ruby/languages/ruby/parse.y b/lib/lrama/ruby/languages/ruby/parse.y index d89b365..e93fadb 100644 --- a/lib/lrama/ruby/languages/ruby/parse.y +++ b/lib/lrama/ruby/languages/ruby/parse.y @@ -387,11 +387,11 @@ command: fcall command_args %prec tLOWEST { $$ = @builder.call($1, $2 || []) }; /* upstream parse.y:3603: command: fcall command_args cmd_brace_block */ command: fcall command_args cmd_brace_block { $$ = $1 }; /* upstream parse.y:3612: command: primary_value call_op operation2 command_args */ -command: primary_value call_op operation2 command_args %prec tLOWEST { $$ = $1 }; +command: primary_value call_op operation2 command_args %prec tLOWEST { $$ = @builder.receiver_call($1, $2, $3, $4) }; /* upstream parse.y:3617: command: primary_value call_op operation2 command_args cmd_brace_block */ command: primary_value call_op operation2 command_args cmd_brace_block { $$ = $1 }; /* upstream parse.y:3622: command: primary_value "::" operation2 command_args */ -command: primary_value tCOLON2 operation2 command_args %prec tLOWEST { $$ = $1 }; +command: primary_value tCOLON2 operation2 command_args %prec tLOWEST { $$ = @builder.receiver_call($1, :"::", $3, $4) }; /* upstream parse.y:3627: command: primary_value "::" operation2 command_args cmd_brace_block */ command: primary_value tCOLON2 operation2 command_args cmd_brace_block %prec tCOLON2 { $$ = $1 }; /* upstream parse.y:3632: command: primary_value "::" "constant" '{' brace_body '}' */ @@ -417,15 +417,15 @@ mlhs_inner: tLPAREN mlhs_inner rparen %prec tLPAREN { $$ = $2 }; /* upstream parse.y:3687: mlhs_basic: mlhs_head */ mlhs_basic: mlhs_head { $$ = $1 }; /* upstream parse.y:3692: mlhs_basic: mlhs_head mlhs_item */ -mlhs_basic: mlhs_head mlhs_item { $$ = ($1 + [$2]).freeze }; +mlhs_basic: mlhs_head mlhs_item { $$ = (Array($1) + [$2]).freeze }; /* upstream parse.y:3697: mlhs_basic: mlhs_head "*" mlhs_node */ -mlhs_basic: mlhs_head tSTAR mlhs_node %prec tSTAR { $$ = ($1 + [$3]).freeze }; +mlhs_basic: mlhs_head tSTAR mlhs_node %prec tSTAR { $$ = (Array($1) + [$3]).freeze }; /* upstream parse.y:3048: mlhs_items_mlhs_item: mlhs_item */ mlhs_items_mlhs_item: mlhs_item { $$ = $1 }; /* upstream parse.y:3053: mlhs_items_mlhs_item: mlhs_items_mlhs_item ',' mlhs_item */ -mlhs_items_mlhs_item: mlhs_items_mlhs_item ',' mlhs_item %prec ',' { $$ = ($1 + [$3]).freeze }; +mlhs_items_mlhs_item: mlhs_items_mlhs_item ',' mlhs_item %prec ',' { $$ = (Array($1) + [$3]).freeze }; /* upstream parse.y:3702: mlhs_basic: mlhs_head "*" mlhs_node ',' mlhs_items_mlhs_item */ -mlhs_basic: mlhs_head tSTAR mlhs_node ',' mlhs_items_mlhs_item %prec ',' { $$ = ($1 + [$3] + $5).freeze }; +mlhs_basic: mlhs_head tSTAR mlhs_node ',' mlhs_items_mlhs_item %prec ',' { $$ = (Array($1) + [$3] + Array($5)).freeze }; /* upstream parse.y:3707: mlhs_basic: mlhs_head "*" */ mlhs_basic: mlhs_head tSTAR %prec tSTAR { $$ = $1 }; /* upstream parse.y:3712: mlhs_basic: mlhs_head "*" ',' mlhs_items_mlhs_item */ @@ -1201,13 +1201,13 @@ block_call: block_call call_op2 paren_args { $$ = $1 }; /* upstream parse.y:5196: method_call: fcall paren_args */ method_call: fcall paren_args { $$ = @builder.call($1, $2 || []) }; /* upstream parse.y:5203: method_call: primary_value call_op operation2 opt_paren_args */ -method_call: primary_value call_op operation2 opt_paren_args { $$ = $1 }; +method_call: primary_value call_op operation2 opt_paren_args { $$ = @builder.receiver_call($1, $2, $3, $4) }; /* upstream parse.y:5214: method_call: primary_value "::" operation2 paren_args */ -method_call: primary_value tCOLON2 operation2 paren_args %prec tCOLON2 { $$ = $1 }; +method_call: primary_value tCOLON2 operation2 paren_args %prec tCOLON2 { $$ = @builder.receiver_call($1, :"::", $3, $4) }; /* upstream parse.y:5220: method_call: primary_value "::" operation3 */ -method_call: primary_value tCOLON2 operation3 %prec tCOLON2 { $$ = $1 }; +method_call: primary_value tCOLON2 operation3 %prec tCOLON2 { $$ = @builder.receiver_call($1, :"::", $3, []) }; /* upstream parse.y:5225: method_call: primary_value call_op2 paren_args */ -method_call: primary_value call_op2 paren_args { $$ = $1 }; +method_call: primary_value call_op2 paren_args { $$ = @builder.receiver_call($1, $2, :call, $3) }; /* upstream parse.y:5231: method_call: "'super'" paren_args */ method_call: keyword_super paren_args %prec keyword_super { $$ = @builder.call(:super, $2 || []) }; /* upstream parse.y:5241: method_call: "'super'" */ @@ -1543,7 +1543,7 @@ midrule_37: %empty { $$ = nil }; /* upstream parse.y:6074: @38: %empty */ midrule_38: %empty { $$ = nil }; /* upstream parse.y:6079: string_content: "'#{'" @35 @36 @37 @38 compstmt_stmts string_dend */ -string_content: tSTRING_DBEG midrule_35 midrule_36 midrule_37 midrule_38 compstmt_stmts string_dend %prec tSTRING_DBEG { $$ = $7 }; +string_content: tSTRING_DBEG midrule_35 midrule_36 midrule_37 midrule_38 compstmt_stmts string_dend %prec tSTRING_DBEG { $$ = @builder.interpolation($6) }; /* upstream parse.y:6094: string_dend: "'}'" */ string_dend: tSTRING_DEND %prec tSTRING_DEND { $$ = nil }; /* upstream parse.y:6095: string_dend: "end-of-input" */ diff --git a/lib/lrama/ruby/languages/ruby/rules.json b/lib/lrama/ruby/languages/ruby/rules.json index 640a7de..977cbd6 100644 --- a/lib/lrama/ruby/languages/ruby/rules.json +++ b/lib/lrama/ruby/languages/ruby/rules.json @@ -603,7 +603,7 @@ "id": 101, "rule": "command: primary_value call_op operation2 command_args", "line": 3612, - "status": "pass_through" + "status": "ported" }, { "id": 102, @@ -615,7 +615,7 @@ "id": 103, "rule": "command: primary_value \"::\" operation2 command_args", "line": 3622, - "status": "pass_through" + "status": "ported" }, { "id": 104, @@ -3045,25 +3045,25 @@ "id": 508, "rule": "method_call: primary_value call_op operation2 opt_paren_args", "line": 5203, - "status": "pass_through" + "status": "ported" }, { "id": 509, "rule": "method_call: primary_value \"::\" operation2 paren_args", "line": 5214, - "status": "pass_through" + "status": "ported" }, { "id": 510, "rule": "method_call: primary_value \"::\" operation3", "line": 5220, - "status": "pass_through" + "status": "ported" }, { "id": 511, "rule": "method_call: primary_value call_op2 paren_args", "line": 5225, - "status": "pass_through" + "status": "ported" }, { "id": 512, @@ -4071,7 +4071,7 @@ "id": 679, "rule": "string_content: \"'#{'\" @35 @36 @37 @38 compstmt_stmts string_dend", "line": 6079, - "status": "pass_through" + "status": "ported" }, { "id": 680, diff --git a/sig/lrama/ruby.rbs b/sig/lrama/ruby.rbs index ae2e4a2..a52eb30 100644 --- a/sig/lrama/ruby.rbs +++ b/sig/lrama/ruby.rbs @@ -36,7 +36,7 @@ module Lrama end module AST - type node = Literal | Binary | Unary | If | LocalRead | LocalWrite | BareCall | ArrayLiteral | HashLiteral | RangeLiteral | Ternary | Call | StringLiteral | RegexpLiteral | XStringLiteral | Index | Loop | Control | Variable | Rescue | Case | When | For | Def | ClassDef | ModuleDef | Lambda + type node = Literal | Binary | Unary | If | LocalRead | LocalWrite | BareCall | ArrayLiteral | HashLiteral | RangeLiteral | Ternary | Call | ReceiverCall | StringLiteral | InterpolatedString | RegexpLiteral | XStringLiteral | Index | Loop | Control | Variable | Rescue | Case | When | For | Def | ClassDef | ModuleDef | Lambda type hash_pair = Pair class Program < ::Data @@ -135,12 +135,27 @@ module Lrama | (name: Symbol, arguments: Array[node]) -> instance end + class ReceiverCall < ::Data + attr_reader receiver: node + attr_reader operator: :"." | :"::" | :"&." + attr_reader name: Symbol + attr_reader arguments: Array[node] + def self.new: (node receiver, (:"." | :"::" | :"&.") operator, Symbol name, Array[node] arguments) -> instance + | (receiver: node, operator: (:"." | :"::" | :"&."), name: Symbol, arguments: Array[node]) -> instance + end + class StringLiteral < ::Data attr_reader value: String def self.new: (String value) -> instance | (value: String) -> instance end + class InterpolatedString < ::Data + attr_reader parts: Array[node | String] + def self.new: (Array[node | String] parts) -> instance + | (parts: Array[node | String]) -> instance + end + class RegexpLiteral < ::Data attr_reader value: String def self.new: (String value) -> instance diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index a2ccab5..1fd1ef6 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -116,7 +116,9 @@ def literal(value) end test "keywords after a receiver are method names" do - assert_nothing_raised { Arpaka.parse_source("self.class.to_s") } + receiver = AST::ReceiverCall.new(literal(:self), :".", :class, []) + assert_equal(AST::Program.new([AST::ReceiverCall.new(receiver, :".", :to_s, [])]), + Arpaka.parse_source("self.class.to_s")) end test "operator calls use ordinary argument parentheses" do @@ -402,8 +404,16 @@ def literal(value) assert_equal(AST::RegexpLiteral.new("a"), Arpaka.parse_source("%r(a)").statements.first) end - test "source lexer rejects interpolation in static literals" do - ['"#{x}"', '%Q(#{x})', '%r(#{x})'].each do |source| + test "source lexer preserves simple string interpolation" do + assert_equal(AST::InterpolatedString.new([ + "hello ", AST::Binary.new(:+, literal(1), literal(2)), " world" + ]), Arpaka.parse_source('"hello #{1 + 2} world"').statements.first) + assert_equal(AST::InterpolatedString.new([AST::BareCall.new(:name)]), + Arpaka.parse_source('"#{name}"').statements.first) + end + + test "source lexer still rejects interpolation in percent literals" do + ['%Q(#{x})', '%r(#{x})'].each do |source| assert_raise(Arpaka::LexerError) { Arpaka.parse_source(source) } end end @@ -462,7 +472,7 @@ def literal(value) test "source lexer accepts argumentless command blocks" do assert_equal(AST::Call.new(:included, []), Arpaka.parse_source("included do\n 1\nend").statements.first) - assert_equal(AST::BareCall.new(:foo), + assert_equal(AST::ReceiverCall.new(AST::BareCall.new(:foo), :".", :bar, []), Arpaka.parse_source("foo.bar do\n 1\nend").statements.first) assert_nothing_raised { Arpaka.parse_source("foo before: :bar do |value| value end\n") } tokens = Arpaka.const_get(:Lexer, false).new("items << lambda do").each.to_a diff --git a/test/lrama/ruby_receiver_call_test.rb b/test/lrama/ruby_receiver_call_test.rb new file mode 100644 index 0000000..5a6cb4f --- /dev/null +++ b/test/lrama/ruby_receiver_call_test.rb @@ -0,0 +1,107 @@ +# frozen_string_literal: true + +require "test_helper" + +class RubyReceiverCallTest < Test::Unit::TestCase + AST = Arpaka::AST + + test "receiver calls retain their operator, method name and ordered arguments" do + [".", "::", "&."].each do |operator| + ["object#{operator}fetch(1, 2)", "object#{operator}fetch 1, 2"].each do |source| + tree = Arpaka.parse_source(source).statements.fetch(0) + assert_equal(AST::ReceiverCall.new(AST::BareCall.new(:object), operator.to_sym, + :fetch, [AST::Literal.new(1), AST::Literal.new(2)]), tree, source) + assert_predicate(tree.arguments, :frozen?) + end + end + end + + test "parenthesized invocation without a method name means call" do + [".", "::", "&."].each do |operator| + source = "handler#{operator}(message)" + expected = AST::ReceiverCall.new(AST::BareCall.new(:handler), operator.to_sym, + :call, [AST::BareCall.new(:message)]) + assert_equal(AST::Program.new([expected]), Arpaka.parse_source(source), source) + end + end + + test "safe navigation allows keyword and operator method names" do + ["class", "if", "[]"].each do |name| + source = "object&.#{name}(1)" + expected = AST::ReceiverCall.new(AST::BareCall.new(:object), :"&.", + name.to_sym, [AST::Literal.new(1)]) + assert_equal(AST::Program.new([expected]), Arpaka.parse_source(source), source) + end + end + + test "receiver call trees agree with RubyVM for representative Rails expressions" do + sources = [ + "self.class.to_s", "object&.class", "object&.if(1)", "object&.[](1)", + "logger.warn \"deprecated\"", "File.join(\"a\", \"b\")", + "File::join(\"a\", \"b\")", "File::join \"a\", \"b\"", "Object::name", + "handler.(message)", "handler::(message)", "handler&.(message)", + "object.fetch(1, transform(2))", "object&.fetch 1, 2", + "object.fetch(1).to_s", "object&.fetch(1)&.to_s", "object.ready?" + ] + sources.each do |source| + expected = normalize_rubyvm(RubyVM::AbstractSyntaxTree.parse(source).children.fetch(2)) + actual = normalize_arpaka(Arpaka.parse_source(source).statements.fetch(0)) + assert_equal(expected, actual, source) + end + end + + private + + # This oracle deliberately covers only this test's expression slice. Unknown + # nodes fail instead of making an incomplete comparison look successful. + def normalize_rubyvm(node) + children = node.children + case node.type + when :CALL, :QCALL + receiver, name, arguments = children + [:call, normalize_rubyvm(receiver), node.type == :QCALL, name, rubyvm_arguments(arguments)] + when :FCALL + [:call, nil, false, children[0], rubyvm_arguments(children[1])] + when :VCALL + [:bare_call, children.fetch(0)] + when :CONST + [:constant, children.fetch(0)] + when :SELF + [:literal, :self] + when :STR + [:string, children.fetch(0)] + when :LIT, :INTEGER + [:literal, children.fetch(0)] + else + flunk("Unmapped RubyVM node: #{node.type}") + end + end + + def rubyvm_arguments(node) + return [] unless node + assert_equal(:LIST, node.type) + assert_nil(node.children.last) + node.children[0...-1].map { |child| normalize_rubyvm(child) } + end + + def normalize_arpaka(node) + case node + when AST::ReceiverCall + [:call, normalize_arpaka(node.receiver), node.operator == :"&.", node.name, + node.arguments.map { |argument| normalize_arpaka(argument) }] + when AST::Call + [:call, nil, false, node.name, node.arguments.map { |argument| normalize_arpaka(argument) }] + when AST::BareCall + [:bare_call, node.name] + when AST::Variable + assert_equal(:constant, node.kind) + [:constant, node.name] + when AST::StringLiteral + [:string, node.value] + when AST::Literal + [:literal, node.value] + else + flunk("Unmapped Arpaka node: #{node.inspect}") + end + end +end diff --git a/tool/ruby/actions.rb b/tool/ruby/actions.rb index f838391..765fb38 100644 --- a/tool/ruby/actions.rb +++ b/tool/ruby/actions.rb @@ -163,7 +163,7 @@ module RubyGrammarActions 728 => ["pass_through", "$1"], 725 => ["pass_through", "$1"], 689 => ["pass_through", "$1"], - 679 => ["pass_through", "$7"], + 679 => ["ported", "@builder.interpolation($6)"], 674 => ["pass_through", "$1"], 665 => ["pass_through", "$1"], 628 => ["pass_through", "$1"], @@ -209,10 +209,10 @@ module RubyGrammarActions 520 => ["pass_through", "$7"], 518 => ["pass_through", "$7"], 516 => ["pass_through", "$2"], - 511 => ["pass_through", "$1"], - 510 => ["pass_through", "$1"], - 509 => ["pass_through", "$1"], - 508 => ["pass_through", "$1"], + 511 => ["ported", "@builder.receiver_call($1, $2, :call, $3)"], + 510 => ["ported", "@builder.receiver_call($1, :\"::\", $3, [])"], + 509 => ["ported", "@builder.receiver_call($1, :\"::\", $3, $4)"], + 508 => ["ported", "@builder.receiver_call($1, $2, $3, $4)"], 506 => ["pass_through", "$1"], 505 => ["pass_through", "$1"], 504 => ["pass_through", "$1"], @@ -277,10 +277,10 @@ module RubyGrammarActions 134 => ["pass_through", "$1"], 105 => ["pass_through", "$1"], 104 => ["pass_through", "$1"], - 103 => ["pass_through", "$1"], + 103 => ["ported", "@builder.receiver_call($1, :\"::\", $3, $4)"], 102 => ["pass_through", "$1"], 100 => ["pass_through", "$1"], - 101 => ["pass_through", "$1"], + 101 => ["ported", "@builder.receiver_call($1, $2, $3, $4)"], 606 => ["ported", "@builder.range(:\"...\", nil, $2)"], 605 => ["ported", "@builder.range(:\"..\", nil, $2)"], 604 => ["ported", "@builder.range(:\"...\", $1, nil)"], @@ -380,10 +380,10 @@ module RubyGrammarActions 114 => ["pass_through", "$2"], 128 => ["pass_through", "$2"], 129 => ["ported", "[$1].freeze"], - 116 => ["ported", "($1 + [$2]).freeze"], - 117 => ["ported", "($1 + [$3]).freeze"], - 119 => ["ported", "($1 + [$3]).freeze"], - 120 => ["ported", "($1 + [$3] + $5).freeze"], + 116 => ["ported", "(Array($1) + [$2]).freeze"], + 117 => ["ported", "(Array($1) + [$3]).freeze"], + 119 => ["ported", "(Array($1) + [$3]).freeze"], + 120 => ["ported", "(Array($1) + [$3] + Array($5)).freeze"], 122 => ["pass_through", "$1"], 121 => ["pass_through", "$1"], 123 => ["pass_through", "$2"], From 2307d71dcf35bcf45a9bcdc1bfb98518e15d5dbc Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 20:25:40 +0900 Subject: [PATCH 078/157] Classify spaced empty brackets as arrays --- lib/lrama/ruby/languages/ruby/lexer.rb | 7 ++++++- test/lrama/ruby_language_test.rb | 5 +++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 78e9d17..eeb3d42 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -658,7 +658,12 @@ def operator_or_punctuation(start) advance(2) return [value == "-@" ? :tUMINUS : :tUPLUS, nil] end - text = OPERATORS.sort_by { |operator| -operator.bytesize }.find { |operator| @source.byteslice(@index, operator.bytesize) == operator } + text = OPERATORS.sort_by { |operator| -operator.bytesize }.find do |operator| + next false if operator == "[]" && start.positive? && + [9, 10, 11, 12, 13, 32].include?(@source.getbyte(start - 1)) && + @previous != :keyword_def + @source.byteslice(@index, operator.bytesize) == operator + end operator_method = [:keyword_def, ".", :tCOLON2, :tANDDOT, :tSYMBEG].include?(@previous) if text && !(begin_expression? && !operator_method && ["[]", "[]="].include?(text)) advance(text.bytesize) diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 1fd1ef6..a07ab17 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -448,6 +448,11 @@ def literal(value) assert_equal(:tBDOT3, tokens[5].first) end + test "source lexer treats spaced empty brackets as an array literal" do + tokens = Arpaka.const_get(:Lexer, false).new("assert_equal [], value").each.to_a + assert_equal([:tIDENTIFIER, "[", "]", ",", :tIDENTIFIER, 0], tokens.map(&:first)) + end + test "source lexer treats predicate and bang methods as identifiers" do assert_equal(AST::Def.new(:stopping?, [], [literal(1)]), Arpaka.parse_source("def stopping?; 1; end").statements.first) From 599168d122b9e41049c9f4b90353023476dd48c1 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 20:31:04 +0900 Subject: [PATCH 079/157] Preserve do block calls in Ruby AST --- lib/lrama/ruby/languages/ruby/ast.rb | 1 + lib/lrama/ruby/languages/ruby/builder.rb | 4 ++++ lib/lrama/ruby/languages/ruby/parse.y | 2 +- lib/lrama/ruby/languages/ruby/rules.json | 2 +- sig/lrama/ruby.rbs | 9 ++++++++- test/lrama/ruby_language_test.rb | 9 +++++---- tool/ruby/actions.rb | 2 +- 7 files changed, 21 insertions(+), 8 deletions(-) diff --git a/lib/lrama/ruby/languages/ruby/ast.rb b/lib/lrama/ruby/languages/ruby/ast.rb index 4aa3ad2..1f17c1e 100644 --- a/lib/lrama/ruby/languages/ruby/ast.rb +++ b/lib/lrama/ruby/languages/ruby/ast.rb @@ -20,6 +20,7 @@ module AST Ternary = Data.define(:condition, :then_value, :else_value) Call = Data.define(:name, :arguments) ReceiverCall = Data.define(:receiver, :operator, :name, :arguments) + BlockCall = Data.define(:call, :body) StringLiteral = Data.define(:value) InterpolatedString = Data.define(:parts) Index = Data.define(:receiver, :arguments) diff --git a/lib/lrama/ruby/languages/ruby/builder.rb b/lib/lrama/ruby/languages/ruby/builder.rb index c6413a3..5d52b27 100644 --- a/lib/lrama/ruby/languages/ruby/builder.rb +++ b/lib/lrama/ruby/languages/ruby/builder.rb @@ -61,6 +61,10 @@ def receiver_call(receiver, operator, name, arguments) AST::ReceiverCall.new(receiver, operator, name, (arguments || []).freeze) end + def block_call(call, body) + AST::BlockCall.new(call, body_nodes(body)) + end + def append_arguments(arguments, argument) list = arguments.is_a?(Array) ? arguments : [arguments] (list + [argument]).freeze diff --git a/lib/lrama/ruby/languages/ruby/parse.y b/lib/lrama/ruby/languages/ruby/parse.y index e93fadb..1224c9e 100644 --- a/lib/lrama/ruby/languages/ruby/parse.y +++ b/lib/lrama/ruby/languages/ruby/parse.y @@ -1189,7 +1189,7 @@ lambda_body: keyword_do_LAMBDA midrule_28 bodystmt k_end %prec keyword_do_LAMBDA /* upstream parse.y:5153: do_block: k_do_block do_body k_end */ do_block: k_do_block do_body k_end { $$ = $2 }; /* upstream parse.y:5161: block_call: command do_block */ -block_call: command do_block { $$ = $1 }; +block_call: command do_block { $$ = @builder.block_call($1, $2) }; /* upstream parse.y:5167: block_call: block_call call_op2 operation2 opt_paren_args */ block_call: block_call call_op2 operation2 opt_paren_args { $$ = $1 }; /* upstream parse.y:5177: block_call: block_call call_op2 operation2 opt_paren_args brace_block */ diff --git a/lib/lrama/ruby/languages/ruby/rules.json b/lib/lrama/ruby/languages/ruby/rules.json index 977cbd6..36e385a 100644 --- a/lib/lrama/ruby/languages/ruby/rules.json +++ b/lib/lrama/ruby/languages/ruby/rules.json @@ -3009,7 +3009,7 @@ "id": 502, "rule": "block_call: command do_block", "line": 5161, - "status": "pass_through" + "status": "ported" }, { "id": 503, diff --git a/sig/lrama/ruby.rbs b/sig/lrama/ruby.rbs index a52eb30..ca6c5f3 100644 --- a/sig/lrama/ruby.rbs +++ b/sig/lrama/ruby.rbs @@ -36,7 +36,7 @@ module Lrama end module AST - type node = Literal | Binary | Unary | If | LocalRead | LocalWrite | BareCall | ArrayLiteral | HashLiteral | RangeLiteral | Ternary | Call | ReceiverCall | StringLiteral | InterpolatedString | RegexpLiteral | XStringLiteral | Index | Loop | Control | Variable | Rescue | Case | When | For | Def | ClassDef | ModuleDef | Lambda + type node = Literal | Binary | Unary | If | LocalRead | LocalWrite | BareCall | ArrayLiteral | HashLiteral | RangeLiteral | Ternary | Call | ReceiverCall | BlockCall | StringLiteral | InterpolatedString | RegexpLiteral | XStringLiteral | Index | Loop | Control | Variable | Rescue | Case | When | For | Def | ClassDef | ModuleDef | Lambda type hash_pair = Pair class Program < ::Data @@ -144,6 +144,13 @@ module Lrama | (receiver: node, operator: (:"." | :"::" | :"&."), name: Symbol, arguments: Array[node]) -> instance end + class BlockCall < ::Data + attr_reader call: node + attr_reader body: Array[node] + def self.new: (node call, Array[node] body) -> instance + | (call: node, body: Array[node]) -> instance + end + class StringLiteral < ::Data attr_reader value: String def self.new: (String value) -> instance diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index a07ab17..9a371c1 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -324,7 +324,8 @@ def literal(value) test "command blocks accept block parameters" do assert_equal(AST::Program.new([:each]), Arpaka.parse_source("each { |value| value }")) - assert_equal(AST::Program.new([AST::Call.new(:each, [])]), Arpaka.parse_source("each do |value| value end")) + assert_equal(AST::Program.new([AST::BlockCall.new(AST::Call.new(:each, []), [AST::BareCall.new(:value)])]), + Arpaka.parse_source("each do |value| value end")) end test "if, unless, elsif and else build conditional AST nodes" do @@ -475,9 +476,9 @@ def literal(value) end test "source lexer accepts argumentless command blocks" do - assert_equal(AST::Call.new(:included, []), + assert_equal(AST::BlockCall.new(AST::Call.new(:included, []), [literal(1)]), Arpaka.parse_source("included do\n 1\nend").statements.first) - assert_equal(AST::ReceiverCall.new(AST::BareCall.new(:foo), :".", :bar, []), + assert_equal(AST::BlockCall.new(AST::ReceiverCall.new(AST::BareCall.new(:foo), :".", :bar, []), [literal(1)]), Arpaka.parse_source("foo.bar do\n 1\nend").statements.first) assert_nothing_raised { Arpaka.parse_source("foo before: :bar do |value| value end\n") } tokens = Arpaka.const_get(:Lexer, false).new("items << lambda do").each.to_a @@ -507,7 +508,7 @@ def literal(value) test "grammar precedence accepts block calls as parenthesized arguments" do tree = Arpaka.parse_source("call(lambda do\nend)\n") - assert_equal(AST::Call.new(:call, [AST::Call.new(:lambda, [])]), tree.statements.first) + assert_equal(AST::Call.new(:call, [AST::BlockCall.new(AST::Call.new(:lambda, []), [])]), tree.statements.first) assert_nothing_raised do Arpaka.parse_source("subscribe(x, handler, lambda do\n work\nend)\n") diff --git a/tool/ruby/actions.rb b/tool/ruby/actions.rb index 765fb38..560f845 100644 --- a/tool/ruby/actions.rb +++ b/tool/ruby/actions.rb @@ -217,7 +217,7 @@ module RubyGrammarActions 505 => ["pass_through", "$1"], 504 => ["pass_through", "$1"], 503 => ["pass_through", "$1"], - 502 => ["pass_through", "$1"], + 502 => ["ported", "@builder.block_call($1, $2)"], 501 => ["pass_through", "$2"], 488 => ["pass_through", "$1"], 487 => ["pass_through", "$1"], From 397db845df592c7f1fbca91311a1f5097fe0f752 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 21:03:37 +0900 Subject: [PATCH 080/157] Recognize spaced array command arguments --- lib/lrama/ruby/languages/ruby/lexer.rb | 5 ++++- test/lrama/ruby_language_test.rb | 8 +++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index eeb3d42..ec0ba5d 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -697,7 +697,10 @@ def operator_or_punctuation(start) @lambda_pending = false return [:tLAMBEG, nil] end - return [value == "[" ? (@begin_expression ? :tLBRACK : "[") : (brace_block ? "{" : :tLBRACE), nil] + array_argument = value == "[" && start.positive? && + [9, 10, 11, 12, 13, 32].include?(@source.getbyte(start - 1)) && + [:tIDENTIFIER, :tFID, :tCONSTANT].include?(@previous) + return [value == "[" && (@begin_expression || array_argument) ? :tLBRACK : (value == "[" ? "[" : (brace_block ? "{" : :tLBRACE)), nil] elsif value == ")" || value == "]" || value == "}" @delimiter_depth -= 1 if @delimiter_depth.positive? elsif value == "-" && @begin_expression diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 9a371c1..8c5b9cb 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -451,7 +451,13 @@ def literal(value) test "source lexer treats spaced empty brackets as an array literal" do tokens = Arpaka.const_get(:Lexer, false).new("assert_equal [], value").each.to_a - assert_equal([:tIDENTIFIER, "[", "]", ",", :tIDENTIFIER, 0], tokens.map(&:first)) + assert_equal([:tIDENTIFIER, :tLBRACK, "]", ",", :tIDENTIFIER, 0], tokens.map(&:first)) + end + + test "source lexer treats spaced bracket arguments as array literals" do + tree = Arpaka.parse_source("foo [1], name: 2").statements.first + assert_equal(AST::ArrayLiteral.new([literal(1)]), tree.arguments.first) + assert_equal(AST::Pair.new(AST::Literal.new(:name), literal(2)), tree.arguments.last) end test "source lexer treats predicate and bang methods as identifiers" do From 13e92013b680c478d6cd613a5e077b7e497a9919 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 21:16:47 +0900 Subject: [PATCH 081/157] Preserve interpolation in percent strings and heredocs --- lib/lrama/ruby/languages/ruby/lexer.rb | 77 +++++++++++++++++++++++++- test/lrama/ruby_language_test.rb | 9 +-- 2 files changed, 79 insertions(+), 7 deletions(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index ec0ba5d..7e44524 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -531,7 +531,10 @@ def percent_token(start) fail!("invalid percent literal", start) unless delimiter advance closing = { 40 => 41, 91 => 93, 123 => 125, 60 => 62 }.fetch(delimiter, delimiter) - content = read_delimited(closing, start, interpolate: [81, 87, 73, 114, 120, 88].include?(kind)) + interpolate = [81, 87, 73, 114, 120, 88].include?(kind) + content = interpolate && [81, 120, 88].include?(kind) ? + read_interpolated_delimited(closing, start) : + read_delimited(closing, start, interpolate: interpolate) advance while kind == 114 && byte && byte.between?(97, 122) token = { 113 => :tSTRING_BEG, 81 => :tSTRING_BEG, 119 => :tWORDS_BEG, 87 => :tQWORDS_BEG, 105 => :tSYMBOLS_BEG, 73 => :tQSYMBOLS_BEG, 114 => :tREGEXP_BEG, @@ -539,6 +542,8 @@ def percent_token(start) terminator = kind == 114 ? :tREGEXP_END : :tSTRING_END @pending = if [119, 87, 105, 73].include?(kind) word_tokens(content, symbols: [105, 73].include?(kind)) + [[terminator, nil]] + elsif content.is_a?(Array) + content + [[terminator, nil]] else [[:tSTRING_CONTENT, content], [terminator, nil]] end @@ -577,6 +582,72 @@ def read_delimited(closing, start, interpolate: false) fail!("unterminated percent literal", start) end + def read_interpolated_delimited(closing, start) + result = +"" + tokens = [] + depth = 0 + opening = { 41 => 40, 93 => 91, 125 => 123, 62 => 60 }.fetch(closing, -1) + until eof? + value = byte + if value == 92 + result << escape_sequence(start) + elsif value == 35 && byte(1) == 123 + tokens << [:tSTRING_CONTENT, result] unless result.empty? + result = +"" + advance(2) + expression = read_interpolation_source(start) + inner = self.class.new(expression, filename: @filename).each.to_a + inner.pop if inner.last == [0, nil] + tokens << [:tSTRING_DBEG, nil] + tokens.concat(inner) + tokens << [:tSTRING_DEND, nil] + elsif value == closing && depth.zero? + advance + tokens << [:tSTRING_CONTENT, result] unless result.empty? + return tokens + else + depth += 1 if value == opening + depth -= 1 if value == closing && depth.positive? + result << value.chr + advance + end + end + fail!("unterminated percent literal", start) + end + + def interpolated_content_tokens(source, start) + return [[:tSTRING_CONTENT, source]] unless source.include?("#" + "{") + tokens = [] + literal = +"" + index = 0 + while index < source.bytesize + if source.getbyte(index) == 35 && source.getbyte(index + 1) == 123 + tokens << [:tSTRING_CONTENT, literal] unless literal.empty? + depth = 1 + expr_start = index + 2 + index = expr_start + while index < source.bytesize && depth.positive? + depth += 1 if source.getbyte(index) == 123 + depth -= 1 if source.getbyte(index) == 125 + index += 1 + end + fail!("unterminated string interpolation", start) unless depth.zero? + expression = source.byteslice(expr_start, index - expr_start - 1) + inner = self.class.new(expression, filename: @filename).each.to_a + inner.pop if inner.last == [0, nil] + tokens << [:tSTRING_DBEG, nil] + tokens.concat(inner) + tokens << [:tSTRING_DEND, nil] + literal = +"" + else + literal << source.getbyte(index).chr + index += 1 + end + end + tokens << [:tSTRING_CONTENT, literal] unless literal.empty? + tokens + end + def heredoc_or_operator(start) if byte(1) == 60 && heredoc_prefix? return heredoc_token(start) if @begin_expression || @previous == :tIDENTIFIER @@ -624,13 +695,13 @@ def heredoc_token(start) advance if byte == 10 break end - fail!("string interpolation cannot be represented by the AST", start) if interpolate && line.include?("#" + "{") body << line << "\n" advance if byte == 10 fail!("unterminated heredoc", start) if eof? end body = dedent_heredoc(body) if squiggly - @pending = [[:tSTRING_CONTENT, body], [:tSTRING_END, nil]] + content_tokens = interpolate ? interpolated_content_tokens(body, start) : [[:tSTRING_CONTENT, body]] + @pending = content_tokens + [[:tSTRING_END, nil]] unless suffix.empty? suffix_tokens = self.class.new(suffix, filename: @filename).each.to_a suffix_tokens.pop if suffix_tokens.last == [0, nil] diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 8c5b9cb..47d3c7a 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -413,10 +413,11 @@ def literal(value) Arpaka.parse_source('"#{name}"').statements.first) end - test "source lexer still rejects interpolation in percent literals" do - ['%Q(#{x})', '%r(#{x})'].each do |source| - assert_raise(Arpaka::LexerError) { Arpaka.parse_source(source) } - end + test "source lexer preserves interpolation in percent strings" do + expected = AST::InterpolatedString.new(["hello ", AST::BareCall.new(:name)]) + assert_equal(expected, Arpaka.parse_source('%Q(hello #{name})').statements.first) + assert_equal(expected, Arpaka.parse_source('%(hello #{name})').statements.first) + assert_raise(Arpaka::LexerError) { Arpaka.parse_source('%r(#{x})') } end test "source lexer handles heredoc indentation and line endings" do From c47dd7cec4c34d4e7723f4caaa75182e9157d43d Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 21:18:40 +0900 Subject: [PATCH 082/157] Handle interpolated regular expression literals --- lib/lrama/ruby/languages/ruby/lexer.rb | 2 +- test/lrama/ruby_language_test.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 7e44524..288cd49 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -532,7 +532,7 @@ def percent_token(start) advance closing = { 40 => 41, 91 => 93, 123 => 125, 60 => 62 }.fetch(delimiter, delimiter) interpolate = [81, 87, 73, 114, 120, 88].include?(kind) - content = interpolate && [81, 120, 88].include?(kind) ? + content = interpolate && [81, 114, 120, 88].include?(kind) ? read_interpolated_delimited(closing, start) : read_delimited(closing, start, interpolate: interpolate) advance while kind == 114 && byte && byte.between?(97, 122) diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 47d3c7a..418d838 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -417,7 +417,7 @@ def literal(value) expected = AST::InterpolatedString.new(["hello ", AST::BareCall.new(:name)]) assert_equal(expected, Arpaka.parse_source('%Q(hello #{name})').statements.first) assert_equal(expected, Arpaka.parse_source('%(hello #{name})').statements.first) - assert_raise(Arpaka::LexerError) { Arpaka.parse_source('%r(#{x})') } + assert_kind_of(AST::RegexpLiteral, Arpaka.parse_source('%r(#{x})').statements.first) end test "source lexer handles heredoc indentation and line endings" do From 798507b7ff82b691d19b6082ca82a12fd7250025 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 21:21:26 +0900 Subject: [PATCH 083/157] Accept interpolated symbol literals --- lib/lrama/ruby/languages/ruby/lexer.rb | 4 ++-- test/lrama/ruby_language_test.rb | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 288cd49..28bb6ac 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -363,8 +363,8 @@ def symbol_or_colon(start) if byte == 39 || byte == 34 quote = byte advance - content = read_quoted(quote, interpolate: quote == 34, start: start) - @pending = [[:tSTRING_CONTENT, content], [:tSTRING_END, nil]] + @pending = read_interpolated_quoted(quote, start) + @pending << [:tSTRING_END, nil] end [:tSYMBEG, nil] end diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 418d838..029315b 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -602,6 +602,10 @@ def literal(value) assert_nothing_raised { Arpaka.parse_source("block_given? ? yield : @default_render.call\n") } end + test "source lexer accepts interpolated symbols" do + assert_nothing_raised { Arpaka.parse_source(':"#{name}_settings"') } + end + test "source lexer accepts ampersand operator symbols" do assert_nothing_raised { Arpaka.parse_source("items.reduce(:&)\n") } assert_nothing_raised { Arpaka.parse_source("delegate :[], :[]=, to: :paths\n") } From 00e3525e507fb058d5d105a09b2afd969798b196 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 21:25:05 +0900 Subject: [PATCH 084/157] Finish lexer interpolation edge cases --- lib/lrama/ruby/languages/ruby/lexer.rb | 6 ++++-- test/lrama/ruby_language_test.rb | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 28bb6ac..de091c4 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -425,7 +425,7 @@ def read_interpolation_source(start) elsif value == quote quote = nil end - elsif [39, 34, 96].include?(value) + elsif [39, 34, 96].include?(value) && @source.getbyte(@index - 1) != 36 quote = value elsif value == 123 depth += 1 @@ -478,6 +478,7 @@ def escape_sequence(start) def regexp_or_operator(start) return operator_or_punctuation(start) if @previous == :keyword_def + return operator_or_punctuation(start) if @previous == :tSYMBEG if @begin_expression advance value = read_regexp(start) @@ -532,7 +533,7 @@ def percent_token(start) advance closing = { 40 => 41, 91 => 93, 123 => 125, 60 => 62 }.fetch(delimiter, delimiter) interpolate = [81, 87, 73, 114, 120, 88].include?(kind) - content = interpolate && [81, 114, 120, 88].include?(kind) ? + content = interpolate ? read_interpolated_delimited(closing, start) : read_delimited(closing, start, interpolate: interpolate) advance while kind == 114 && byte && byte.between?(97, 122) @@ -541,6 +542,7 @@ def percent_token(start) 115 => :tSYMBEG, 120 => :tXSTRING_BEG, 88 => :tXSTRING_BEG }.fetch(kind) terminator = kind == 114 ? :tREGEXP_END : :tSTRING_END @pending = if [119, 87, 105, 73].include?(kind) + content = content.map { |token, value| value.to_s }.join if content.is_a?(Array) word_tokens(content, symbols: [105, 73].include?(kind)) + [[terminator, nil]] elsif content.is_a?(Array) content + [[terminator, nil]] diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 029315b..34011d4 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -604,6 +604,7 @@ def literal(value) test "source lexer accepts interpolated symbols" do assert_nothing_raised { Arpaka.parse_source(':"#{name}_settings"') } + assert_nothing_raised { Arpaka.parse_source("super(:/, left, right)") } end test "source lexer accepts ampersand operator symbols" do From d95f6f150eb82af398590cb8b8f29e3fa52646f4 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 21:34:25 +0900 Subject: [PATCH 085/157] Build interpolated dynamic symbols --- lib/lrama/ruby/languages/ruby/builder.rb | 1 + lib/lrama/ruby/languages/ruby/parse.y | 2 +- lib/lrama/ruby/languages/ruby/rules.json | 2 +- tool/ruby/actions.rb | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/lrama/ruby/languages/ruby/builder.rb b/lib/lrama/ruby/languages/ruby/builder.rb index 5d52b27..aaa2580 100644 --- a/lib/lrama/ruby/languages/ruby/builder.rb +++ b/lib/lrama/ruby/languages/ruby/builder.rb @@ -111,6 +111,7 @@ def body_nodes(body) end def symbol(value) + value = value.map { |part| part.is_a?(String) ? part : part.to_s }.join if value.is_a?(Array) value = value.name if value.is_a?(AST::Variable) AST::Literal.new(value.to_sym) end diff --git a/lib/lrama/ruby/languages/ruby/parse.y b/lib/lrama/ruby/languages/ruby/parse.y index 1224c9e..ecb6b56 100644 --- a/lib/lrama/ruby/languages/ruby/parse.y +++ b/lib/lrama/ruby/languages/ruby/parse.y @@ -1563,7 +1563,7 @@ sym: fname { $$ = $1 }; /* upstream parse.y:6127: sym: nonlocal_var */ sym: nonlocal_var { $$ = $1 }; /* upstream parse.y:6131: dsym: "symbol literal" string_contents "terminator" */ -dsym: tSYMBEG string_contents tSTRING_END %prec tSTRING_END { $$ = $1 }; +dsym: tSYMBEG string_contents tSTRING_END %prec tSTRING_END { $$ = @builder.symbol(@builder.join_strings($2)) }; /* upstream parse.y:6138: numeric: simple_numeric */ numeric: simple_numeric { $$ = $1 }; /* upstream parse.y:6140: numeric: tUMINUS_NUM simple_numeric */ diff --git a/lib/lrama/ruby/languages/ruby/rules.json b/lib/lrama/ruby/languages/ruby/rules.json index 36e385a..be1c9f0 100644 --- a/lib/lrama/ruby/languages/ruby/rules.json +++ b/lib/lrama/ruby/languages/ruby/rules.json @@ -4131,7 +4131,7 @@ "id": 689, "rule": "dsym: \"symbol literal\" string_contents \"terminator\"", "line": 6131, - "status": "pass_through" + "status": "ported" }, { "id": 690, diff --git a/tool/ruby/actions.rb b/tool/ruby/actions.rb index 560f845..89d63e1 100644 --- a/tool/ruby/actions.rb +++ b/tool/ruby/actions.rb @@ -162,7 +162,7 @@ module RubyGrammarActions 731 => ["pass_through", "$1"], 728 => ["pass_through", "$1"], 725 => ["pass_through", "$1"], - 689 => ["pass_through", "$1"], + 689 => ["ported", "@builder.symbol(@builder.join_strings($2))"], 679 => ["ported", "@builder.interpolation($6)"], 674 => ["pass_through", "$1"], 665 => ["pass_through", "$1"], From b4bb278c850a763fe2744a4a3706ae57f31f9a6d Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 21:43:57 +0900 Subject: [PATCH 086/157] Continue lines after bitwise ampersand --- lib/lrama/ruby/languages/ruby/lexer.rb | 2 +- test/lrama/ruby_language_test.rb | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index de091c4..2a43f66 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -170,7 +170,7 @@ def skip_space_and_comments def newline_ignored? ignored = @delimiter_depth.positive? || @previous == "\n" || - ["+", "-", "*", "/", "%", "=", "?", ":", ",", ".", "&&", "||", "=>"].include?(@previous) || + ["+", "-", "*", "/", "%", "=", "?", ":", ",", ".", "&", "&&", "||", "=>", :tAMPER, :tPIPE].include?(@previous) || @previous == :tLABEL || [:tANDOP, :tOROP, :tMATCH, :tNMATCH, :tASSOC, :tOP_ASGN].include?(@previous) || next_non_space_byte == 46 diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 34011d4..5fc92af 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -610,6 +610,7 @@ def literal(value) test "source lexer accepts ampersand operator symbols" do assert_nothing_raised { Arpaka.parse_source("items.reduce(:&)\n") } assert_nothing_raised { Arpaka.parse_source("delegate :[], :[]=, to: :paths\n") } + assert_nothing_raised { Arpaka.parse_source("left &\n right\n") } end test "source lexer recognizes bitwise assignment operators" do From 5d5229c3d4ffc7809b48512ea8f9a29465e74590 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 21:49:14 +0900 Subject: [PATCH 087/157] Handle bitwise operator line continuations --- lib/lrama/ruby/languages/ruby/lexer.rb | 2 +- test/lrama/ruby_language_test.rb | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 2a43f66..6b315ac 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -170,7 +170,7 @@ def skip_space_and_comments def newline_ignored? ignored = @delimiter_depth.positive? || @previous == "\n" || - ["+", "-", "*", "/", "%", "=", "?", ":", ",", ".", "&", "&&", "||", "=>", :tAMPER, :tPIPE].include?(@previous) || + ["+", "-", "*", "/", "%", "=", "?", ":", ",", ".", "&", "|", "^", "<<", ">>", "&&", "||", "=>", :tAMPER, :tPIPE].include?(@previous) || @previous == :tLABEL || [:tANDOP, :tOROP, :tMATCH, :tNMATCH, :tASSOC, :tOP_ASGN].include?(@previous) || next_non_space_byte == 46 diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 5fc92af..8895511 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -611,6 +611,8 @@ def literal(value) assert_nothing_raised { Arpaka.parse_source("items.reduce(:&)\n") } assert_nothing_raised { Arpaka.parse_source("delegate :[], :[]=, to: :paths\n") } assert_nothing_raised { Arpaka.parse_source("left &\n right\n") } + assert_nothing_raised { Arpaka.parse_source("left |\n right\n") } + assert_nothing_raised { Arpaka.parse_source("left ^\n right\n") } end test "source lexer recognizes bitwise assignment operators" do From f05ad4c16615aad343999751450a2fe20a3ff124 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 21:50:14 +0900 Subject: [PATCH 088/157] Continue lines after splat arguments --- lib/lrama/ruby/languages/ruby/lexer.rb | 2 +- test/lrama/ruby_language_test.rb | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 6b315ac..d7dc61d 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -170,7 +170,7 @@ def skip_space_and_comments def newline_ignored? ignored = @delimiter_depth.positive? || @previous == "\n" || - ["+", "-", "*", "/", "%", "=", "?", ":", ",", ".", "&", "|", "^", "<<", ">>", "&&", "||", "=>", :tAMPER, :tPIPE].include?(@previous) || + ["+", "-", "*", "/", "%", "=", "?", ":", ",", ".", "&", "|", "^", "<<", ">>", "&&", "||", "=>", :tAMPER, :tPIPE, :tSTAR, :tDSTAR].include?(@previous) || @previous == :tLABEL || [:tANDOP, :tOROP, :tMATCH, :tNMATCH, :tASSOC, :tOP_ASGN].include?(@previous) || next_non_space_byte == 46 diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 8895511..63ee474 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -613,6 +613,8 @@ def literal(value) assert_nothing_raised { Arpaka.parse_source("left &\n right\n") } assert_nothing_raised { Arpaka.parse_source("left |\n right\n") } assert_nothing_raised { Arpaka.parse_source("left ^\n right\n") } + assert_nothing_raised { Arpaka.parse_source("foo *args\n") } + assert_nothing_raised { Arpaka.parse_source("foo **options\n") } end test "source lexer recognizes bitwise assignment operators" do From 530bc5747d7d8b56e9852b30fb02935488b94de4 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 21:51:50 +0900 Subject: [PATCH 089/157] Continue lines after range operators --- lib/lrama/ruby/languages/ruby/lexer.rb | 2 +- test/lrama/ruby_language_test.rb | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index d7dc61d..70c9ed4 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -170,7 +170,7 @@ def skip_space_and_comments def newline_ignored? ignored = @delimiter_depth.positive? || @previous == "\n" || - ["+", "-", "*", "/", "%", "=", "?", ":", ",", ".", "&", "|", "^", "<<", ">>", "&&", "||", "=>", :tAMPER, :tPIPE, :tSTAR, :tDSTAR].include?(@previous) || + ["+", "-", "*", "/", "%", "=", "?", ":", ",", ".", "&", "|", "^", "<<", ">>", "&&", "||", "=>", :tAMPER, :tPIPE, :tSTAR, :tDSTAR, :tDOT2, :tDOT3].include?(@previous) || @previous == :tLABEL || [:tANDOP, :tOROP, :tMATCH, :tNMATCH, :tASSOC, :tOP_ASGN].include?(@previous) || next_non_space_byte == 46 diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 63ee474..ec776ac 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -615,6 +615,7 @@ def literal(value) assert_nothing_raised { Arpaka.parse_source("left ^\n right\n") } assert_nothing_raised { Arpaka.parse_source("foo *args\n") } assert_nothing_raised { Arpaka.parse_source("foo **options\n") } + assert_nothing_raised { Arpaka.parse_source("first..\n last\n") } end test "source lexer recognizes bitwise assignment operators" do From 72aafdd15d55d906a0353c2ee1e7428b3fed131f Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 21:52:28 +0900 Subject: [PATCH 090/157] Continue lines after keyword logic operators --- lib/lrama/ruby/languages/ruby/lexer.rb | 2 +- test/lrama/ruby_language_test.rb | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 70c9ed4..11b5c88 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -170,7 +170,7 @@ def skip_space_and_comments def newline_ignored? ignored = @delimiter_depth.positive? || @previous == "\n" || - ["+", "-", "*", "/", "%", "=", "?", ":", ",", ".", "&", "|", "^", "<<", ">>", "&&", "||", "=>", :tAMPER, :tPIPE, :tSTAR, :tDSTAR, :tDOT2, :tDOT3].include?(@previous) || + ["+", "-", "*", "/", "%", "=", "?", ":", ",", ".", "&", "|", "^", "<<", ">>", "&&", "||", "=>", :keyword_and, :keyword_or, :tAMPER, :tPIPE, :tSTAR, :tDSTAR, :tDOT2, :tDOT3].include?(@previous) || @previous == :tLABEL || [:tANDOP, :tOROP, :tMATCH, :tNMATCH, :tASSOC, :tOP_ASGN].include?(@previous) || next_non_space_byte == 46 diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index ec776ac..8791bb6 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -616,6 +616,8 @@ def literal(value) assert_nothing_raised { Arpaka.parse_source("foo *args\n") } assert_nothing_raised { Arpaka.parse_source("foo **options\n") } assert_nothing_raised { Arpaka.parse_source("first..\n last\n") } + assert_nothing_raised { Arpaka.parse_source("left and\n right\n") } + assert_nothing_raised { Arpaka.parse_source("left or\n right\n") } end test "source lexer recognizes bitwise assignment operators" do From 7d848f902876413ef78260e894f4bbb8d1db1f0b Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 21:53:18 +0900 Subject: [PATCH 091/157] Preserve line continuations for operator tokens --- lib/lrama/ruby/languages/ruby/lexer.rb | 2 +- test/lrama/ruby_language_test.rb | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 11b5c88..7ae4fac 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -170,7 +170,7 @@ def skip_space_and_comments def newline_ignored? ignored = @delimiter_depth.positive? || @previous == "\n" || - ["+", "-", "*", "/", "%", "=", "?", ":", ",", ".", "&", "|", "^", "<<", ">>", "&&", "||", "=>", :keyword_and, :keyword_or, :tAMPER, :tPIPE, :tSTAR, :tDSTAR, :tDOT2, :tDOT3].include?(@previous) || + ["+", "-", "*", "/", "%", "=", "?", ":", ",", ".", "&", "|", "^", "<<", ">>", "&&", "||", "=>", :keyword_and, :keyword_or, :tAMPER, :tPIPE, :tSTAR, :tDSTAR, :tDOT2, :tDOT3, :tPOW, :tCMP, :tEQ, :tEQQ, :tNEQ, :tGEQ, :tLEQ, :tANDOP, :tOROP, :tMATCH, :tNMATCH, :tLSHFT, :tRSHFT, :tASSOC].include?(@previous) || @previous == :tLABEL || [:tANDOP, :tOROP, :tMATCH, :tNMATCH, :tASSOC, :tOP_ASGN].include?(@previous) || next_non_space_byte == 46 diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 8791bb6..e6be21a 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -618,6 +618,7 @@ def literal(value) assert_nothing_raised { Arpaka.parse_source("first..\n last\n") } assert_nothing_raised { Arpaka.parse_source("left and\n right\n") } assert_nothing_raised { Arpaka.parse_source("left or\n right\n") } + assert_nothing_raised { Arpaka.parse_source("left ==\n right\n") } end test "source lexer recognizes bitwise assignment operators" do From 42b96026aaade7d738c73c821dab45026ca4f8c4 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 21:54:04 +0900 Subject: [PATCH 092/157] Continue lines after not operator --- lib/lrama/ruby/languages/ruby/lexer.rb | 2 +- test/lrama/ruby_language_test.rb | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 7ae4fac..12d2d0c 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -170,7 +170,7 @@ def skip_space_and_comments def newline_ignored? ignored = @delimiter_depth.positive? || @previous == "\n" || - ["+", "-", "*", "/", "%", "=", "?", ":", ",", ".", "&", "|", "^", "<<", ">>", "&&", "||", "=>", :keyword_and, :keyword_or, :tAMPER, :tPIPE, :tSTAR, :tDSTAR, :tDOT2, :tDOT3, :tPOW, :tCMP, :tEQ, :tEQQ, :tNEQ, :tGEQ, :tLEQ, :tANDOP, :tOROP, :tMATCH, :tNMATCH, :tLSHFT, :tRSHFT, :tASSOC].include?(@previous) || + ["+", "-", "*", "/", "%", "=", "?", ":", ",", ".", "&", "|", "^", "<<", ">>", "&&", "||", "=>", :keyword_and, :keyword_or, :keyword_not, :tAMPER, :tPIPE, :tSTAR, :tDSTAR, :tDOT2, :tDOT3, :tPOW, :tCMP, :tEQ, :tEQQ, :tNEQ, :tGEQ, :tLEQ, :tANDOP, :tOROP, :tMATCH, :tNMATCH, :tLSHFT, :tRSHFT, :tASSOC].include?(@previous) || @previous == :tLABEL || [:tANDOP, :tOROP, :tMATCH, :tNMATCH, :tASSOC, :tOP_ASGN].include?(@previous) || next_non_space_byte == 46 diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index e6be21a..176a28d 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -619,6 +619,7 @@ def literal(value) assert_nothing_raised { Arpaka.parse_source("left and\n right\n") } assert_nothing_raised { Arpaka.parse_source("left or\n right\n") } assert_nothing_raised { Arpaka.parse_source("left ==\n right\n") } + assert_nothing_raised { Arpaka.parse_source("not\n value\n") } end test "source lexer recognizes bitwise assignment operators" do From 4aee3ca60193f270ee79572800c7862f923a557c Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sat, 12 Sep 2026 21:54:45 +0900 Subject: [PATCH 093/157] Continue lines after lambda and scope operators --- lib/lrama/ruby/languages/ruby/lexer.rb | 2 +- test/lrama/ruby_language_test.rb | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 12d2d0c..ea4300b 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -170,7 +170,7 @@ def skip_space_and_comments def newline_ignored? ignored = @delimiter_depth.positive? || @previous == "\n" || - ["+", "-", "*", "/", "%", "=", "?", ":", ",", ".", "&", "|", "^", "<<", ">>", "&&", "||", "=>", :keyword_and, :keyword_or, :keyword_not, :tAMPER, :tPIPE, :tSTAR, :tDSTAR, :tDOT2, :tDOT3, :tPOW, :tCMP, :tEQ, :tEQQ, :tNEQ, :tGEQ, :tLEQ, :tANDOP, :tOROP, :tMATCH, :tNMATCH, :tLSHFT, :tRSHFT, :tASSOC].include?(@previous) || + ["+", "-", "*", "/", "%", "=", "?", ":", ",", ".", "&", "|", "^", "<<", ">>", "&&", "||", "=>", :keyword_and, :keyword_or, :keyword_not, :tAMPER, :tPIPE, :tSTAR, :tDSTAR, :tDOT2, :tDOT3, :tPOW, :tCMP, :tEQ, :tEQQ, :tNEQ, :tGEQ, :tLEQ, :tANDOP, :tOROP, :tMATCH, :tNMATCH, :tLSHFT, :tRSHFT, :tASSOC, :tLAMBDA, :tCOLON2, :tANDDOT].include?(@previous) || @previous == :tLABEL || [:tANDOP, :tOROP, :tMATCH, :tNMATCH, :tASSOC, :tOP_ASGN].include?(@previous) || next_non_space_byte == 46 diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 176a28d..5c3e1da 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -620,6 +620,7 @@ def literal(value) assert_nothing_raised { Arpaka.parse_source("left or\n right\n") } assert_nothing_raised { Arpaka.parse_source("left ==\n right\n") } assert_nothing_raised { Arpaka.parse_source("not\n value\n") } + assert_nothing_raised { Arpaka.parse_source("fn = ->\n { 1 }\n") } end test "source lexer recognizes bitwise assignment operators" do From 5f09ba3113100aee12d2f20461a3068845568f2b Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sun, 13 Sep 2026 00:58:30 +0900 Subject: [PATCH 094/157] Refactor Ruby lexer lexical context --- lib/lrama/ruby/languages/ruby/lexer.rb | 146 +++++++++++++++++-------- test/lrama/ruby_language_test.rb | 41 ++++++- 2 files changed, 140 insertions(+), 47 deletions(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index ea4300b..d7e76be 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -7,6 +7,34 @@ module Ruby # Lexer for Ruby source input. This deliberately does not use Prism: # the parser needs the same small amount of lexical context that CRuby # keeps in its parser (EXPR_BEG/EXPR_END and delimiter nesting). + class LexicalContext + attr_accessor :begin_expression, :condition_do, :condition_line, + :ternary_depth, :lambda_pending, :alias_context + attr_reader :delimiter_stack + + def initialize + @begin_expression = true + @condition_do = false + @condition_line = false + @ternary_depth = 0 + @lambda_pending = false + @alias_context = false + @delimiter_stack = [] + end + + def delimiter_depth + @delimiter_stack.length + end + + def push_delimiter(value) + @delimiter_stack << value + end + + def pop_delimiter(value) + @delimiter_stack.pop if @delimiter_stack.last == value + end + end + class Lexer Error = LexerError @@ -57,14 +85,10 @@ def initialize(source, filename: "(ruby)") @column = 0 @previous_previous = nil @previous = nil - @begin_expression = true - @delimiter_depth = 0 + @previous_value = nil + @context = LexicalContext.new @pending = [] - @condition_do = false - @condition_line = false - @ternary_depth = 0 @class_superclass = false - @lambda_pending = false end def each @@ -106,9 +130,12 @@ def fail!(message, index = @index, line = @line, column = @column) def next_token unless @pending.empty? value = @pending.shift + return next_token if value[0] == "\n" && newline_ignored? @previous_previous = @previous @previous = value[0] - @begin_expression = expression_begin_after(value[0]) + @previous_value = value[1] + update_pending_delimiter(value[0]) + @context.begin_expression = expression_begin_after(value[0]) return value end skip_space_and_comments @@ -118,13 +145,17 @@ def next_token value = case byte when 10 advance + if @context.alias_context && @previous == :tEQ + @context.alias_context = false + return ["\n", nil] + end if @class_superclass && next_word_is_terminator? @class_superclass = false return [";", nil] end @class_superclass = false return next_token if newline_ignored? - @condition_do = false + @context.condition_do = false ["\n", nil] when 39, 34, 96 string_token(byte, start) @@ -149,7 +180,8 @@ def next_token end @previous_previous = @previous @previous = value && value[0] - @begin_expression = expression_begin_after(value && value[0]) + @previous_value = value && value[1] + @context.begin_expression = expression_begin_after(value && value[0]) value end @@ -169,12 +201,13 @@ def skip_space_and_comments end def newline_ignored? - ignored = @delimiter_depth.positive? || @previous == "\n" || + ignored = @context.delimiter_depth.positive? || @previous == "\n" || ["+", "-", "*", "/", "%", "=", "?", ":", ",", ".", "&", "|", "^", "<<", ">>", "&&", "||", "=>", :keyword_and, :keyword_or, :keyword_not, :tAMPER, :tPIPE, :tSTAR, :tDSTAR, :tDOT2, :tDOT3, :tPOW, :tCMP, :tEQ, :tEQQ, :tNEQ, :tGEQ, :tLEQ, :tANDOP, :tOROP, :tMATCH, :tNMATCH, :tLSHFT, :tRSHFT, :tASSOC, :tLAMBDA, :tCOLON2, :tANDDOT].include?(@previous) || @previous == :tLABEL || + [:modifier_if, :modifier_unless, :modifier_while, :modifier_until].include?(@previous) || [:tANDOP, :tOROP, :tMATCH, :tNMATCH, :tASSOC, :tOP_ASGN].include?(@previous) || next_non_space_byte == 46 - @condition_line = false if ignored && @condition_line + @context.condition_line = false if ignored && @context.condition_line ignored end @@ -193,12 +226,23 @@ def next_non_space_byte @source.getbyte(position) end + def update_pending_delimiter(token) + if ["(", "["].include?(token) + @context.push_delimiter(token) + elsif [")", "]"].include?(token) + opener = {")" => "(", "]" => "[" + }.fetch(token) + @context.pop_delimiter(opener) + end + end + def expression_begin_after(token) return true if token.nil? || token == "\n" return false if token == ")" || token == "]" || token == "}" return false if token == :tINTEGER || token == :tFLOAT || token == :tRATIONAL || token == :tIMAGINARY return false if token == :tIDENTIFIER || token == :tCONSTANT || token == :tFID || token == :tSTRING_END || token == :tREGEXP_END return false if token == :keyword_true || token == :keyword_false || token == :keyword_nil || token == :keyword_self + return false if [:keyword__LINE__, :keyword__FILE__, :keyword__ENCODING__].include?(token) return false if token == :tIVAR || token == :tGVAR || token == :tCVAR || token == :tNTH_REF true end @@ -236,7 +280,7 @@ def identifier_token(start) token = KEYWORDS[word] if token && [".", :tCOLON2, :tANDDOT].include?(@previous) token = :tFID - elsif (!@begin_expression || [:keyword_return, :keyword_break, :keyword_next, :keyword_end].include?(@previous)) && { "if" => :modifier_if, "unless" => :modifier_unless, + elsif (!@context.begin_expression || [:keyword_return, :keyword_break, :keyword_next, :keyword_end, :keyword_yield, :keyword_super].include?(@previous)) && { "if" => :modifier_if, "unless" => :modifier_unless, "while" => :modifier_while, "until" => :modifier_until, "rescue" => :modifier_rescue }.key?(word) token = { "if" => :modifier_if, "unless" => :modifier_unless, @@ -244,7 +288,8 @@ def identifier_token(start) "rescue" => :modifier_rescue }.fetch(word, token) end token ||= word == "do" ? do_token : (word.getbyte(0).between?(65, 90) ? :tCONSTANT : :tIDENTIFIER) - @condition_do = true if [:keyword_while, :keyword_until, :keyword_for].include?(token) + @context.condition_do = true if [:keyword_while, :keyword_until, :keyword_for].include?(token) + @context.alias_context = true if token == :keyword_alias [token, word.to_sym] rescue EncodingError fail!("invalid UTF-8 identifier", start) @@ -255,11 +300,11 @@ def identifier_byte?(value) end def do_token - if @condition_do - @condition_do = false + if @context.condition_do + @context.condition_do = false :keyword_do_cond - elsif @lambda_pending - @lambda_pending = false + elsif @context.lambda_pending + @context.lambda_pending = false :keyword_do_LAMBDA elsif @previous == ")" :keyword_do @@ -273,13 +318,14 @@ def do_token def no_argument_block? return false unless [:tIDENTIFIER, :tCONSTANT, :tFID].include?(@previous) return false if [:tIDENTIFIER, :tCONSTANT, :tFID].include?(@previous_previous) - return false if [:tSYMBEG, :tLABEL].include?(@previous_previous) - return false if @previous_previous == :tLSHFT + return false if [:tSYMBEG, :tLABEL, :tCOLON2].include?(@previous_previous) + return false if [:tLSHFT, :tLAMBDA].include?(@previous_previous) true end def no_argument_brace_block? return false if [".", :tCOLON2].include?(@previous_previous) + return false if [:tLABEL, :tLSHFT, :tOROP, :tANDOP, :tASSOC, ","].include?(@previous_previous) no_argument_block? end @@ -327,7 +373,7 @@ def variable_token(start) marker = byte advance advance if marker == 64 && byte == 64 - if marker == 36 && [33, 38, 39, 43, 60, 62, 61, 96, 126].include?(byte) + if marker == 36 && [33, 38, 39, 43, 60, 62, 61, 63, 96, 126].include?(byte) advance text = @source.byteslice(start, @index - start) return [:tGVAR, text.to_sym] @@ -343,7 +389,7 @@ def variable_token(start) end def character_or_question(start) - return operator_or_punctuation(start) unless @begin_expression && byte(1) && byte(1) != 32 && byte(1) != 10 + return operator_or_punctuation(start) unless @context.begin_expression && byte(1) && byte(1) != 32 && byte(1) != 10 advance value = if byte == 92 escape_sequence(start) @@ -356,7 +402,7 @@ def character_or_question(start) end def symbol_or_colon(start) - symbol_position = @begin_expression || ([:tIDENTIFIER, :tFID].include?(@previous) && @ternary_depth.zero?) + symbol_position = @context.begin_expression || ([:tIDENTIFIER, :tFID].include?(@previous) && @context.ternary_depth.zero?) return operator_or_punctuation(start) unless symbol_position && byte(1) && ![9, 10, 11, 12, 13, 32].include?(byte(1)) && byte(1) != 58 advance @@ -479,7 +525,7 @@ def escape_sequence(start) def regexp_or_operator(start) return operator_or_punctuation(start) if @previous == :keyword_def return operator_or_punctuation(start) if @previous == :tSYMBEG - if @begin_expression + if @context.begin_expression advance value = read_regexp(start) @pending = [[:tSTRING_CONTENT, value], [:tREGEXP_END, nil]] @@ -520,7 +566,11 @@ def read_regexp(start) end def percent_token(start) - return operator_or_punctuation(start) unless @begin_expression + literal_kind = [113, 81, 119, 87, 105, 73, 114, 115, 120, 88].include?(byte(1)) + command_argument = start.positive? && [9, 32].include?(@source.getbyte(start - 1)) && + [:tIDENTIFIER, :tCONSTANT, :tFID].include?(@previous) && + ![:tIDENTIFIER, :tCONSTANT, :tFID].include?(@previous_previous) + return operator_or_punctuation(start) unless @context.begin_expression || (literal_kind && command_argument) advance kind = byte if [113, 81, 119, 87, 105, 73, 114, 115, 120, 88].include?(kind) @@ -652,7 +702,7 @@ def interpolated_content_tokens(source, start) def heredoc_or_operator(start) if byte(1) == 60 && heredoc_prefix? - return heredoc_token(start) if @begin_expression || @previous == :tIDENTIFIER + return heredoc_token(start) if @context.begin_expression || @previous == :tIDENTIFIER end operator_or_punctuation(start) end @@ -708,7 +758,6 @@ def heredoc_token(start) suffix_tokens = self.class.new(suffix, filename: @filename).each.to_a suffix_tokens.pop if suffix_tokens.last == [0, nil] @pending.concat(suffix_tokens) - @delimiter_depth -= suffix.count(")") - suffix.count("(") end @pending << ["\n", nil] [quote == 96 ? :tXSTRING_BEG : :tSTRING_BEG, nil] @@ -740,16 +789,16 @@ def operator_or_punctuation(start) operator_method = [:keyword_def, ".", :tCOLON2, :tANDDOT, :tSYMBEG].include?(@previous) if text && !(begin_expression? && !operator_method && ["[]", "[]="].include?(text)) advance(text.bytesize) - token = if text == "**" && @begin_expression + token = if text == "**" && @context.begin_expression :tDSTAR elsif text == "..." && [:tLPAREN, "(", ","].include?(@previous) :tBDOT3 - elsif text == "::" && @begin_expression + elsif text == "::" && (@context.begin_expression || (start.positive? && [9, 32].include?(@source.getbyte(start - 1)))) :tCOLON3 else OP_TOKENS.fetch(text, :tOP_ASGN) end - @lambda_pending = true if token == :tLAMBDA + @context.lambda_pending = true if token == :tLAMBDA return [token, nil] end value = byte.chr @@ -760,36 +809,41 @@ def operator_or_punctuation(start) end advance if value == "(" || value == "[" || value == "{" - brace_block = value == "{" && (@previous == ")" || [".", :tCOLON2].include?(@previous_previous)) - lambda_block = value == "{" && @lambda_pending - @delimiter_depth += 1 unless brace_block || lambda_block + brace_block = value == "{" && (@previous == ")" || [".", :tCOLON2].include?(@previous_previous) || [:proc, :lambda].include?(@previous_value)) + lambda_block = value == "{" && @context.lambda_pending + @context.push_delimiter(value) unless brace_block || lambda_block if value == "(" - return [@begin_expression && ![".", :tCOLON2, :tANDDOT, :keyword_super, :keyword_yield].include?(@previous) ? :tLPAREN : "(", nil] + if !@context.begin_expression && start.positive? && [9, 32].include?(@source.getbyte(start - 1)) && + [:tIDENTIFIER, :tCONSTANT, :tFID].include?(@previous) + return [:tLPAREN_ARG, nil] + end + return [@context.begin_expression && ![".", :tCOLON2, :tANDDOT, :keyword_super, :keyword_yield, :tLAMBDA, :tAREF].include?(@previous) ? :tLPAREN : "(", nil] end - if value == "{" && @lambda_pending - @lambda_pending = false + if value == "{" && @context.lambda_pending + @context.lambda_pending = false return [:tLAMBEG, nil] end array_argument = value == "[" && start.positive? && [9, 10, 11, 12, 13, 32].include?(@source.getbyte(start - 1)) && [:tIDENTIFIER, :tFID, :tCONSTANT].include?(@previous) - return [value == "[" && (@begin_expression || array_argument) ? :tLBRACK : (value == "[" ? "[" : (brace_block ? "{" : :tLBRACE)), nil] + return [value == "[" && (@context.begin_expression || array_argument) ? :tLBRACK : (value == "[" ? "[" : (brace_block ? "{" : :tLBRACE)), nil] elsif value == ")" || value == "]" || value == "}" - @delimiter_depth -= 1 if @delimiter_depth.positive? - elsif value == "-" && @begin_expression + opener = { ")" => "(", "]" => "[", "}" => "{" }.fetch(value) + @context.pop_delimiter(opener) + elsif value == "-" && @context.begin_expression return [:tUMINUS, nil] - elsif value == "+" && @begin_expression + elsif value == "+" && @context.begin_expression return [:tUPLUS, nil] - elsif value == "*" && @begin_expression + elsif value == "*" && @context.begin_expression return [:tSTAR, nil] elsif value == "&" && @previous == :tSYMBEG return [value, nil] - elsif value == "&" && @begin_expression + elsif value == "&" && @context.begin_expression return [:tAMPER, nil] - elsif value == "?" && !@begin_expression - @ternary_depth += 1 - elsif value == ":" && @ternary_depth.positive? - @ternary_depth -= 1 + elsif value == "?" && !@context.begin_expression + @context.ternary_depth += 1 + elsif value == ":" && @context.ternary_depth.positive? + @context.ternary_depth -= 1 elsif value == "<" && @previous == :tCONSTANT && @previous_previous == :keyword_class @class_superclass = true end @@ -799,7 +853,7 @@ def operator_or_punctuation(start) end def begin_expression? - @begin_expression + @context.begin_expression end end private_constant :Lexer diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 5c3e1da..4c0adc5 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -590,7 +590,7 @@ def literal(value) end test "source lexer recognizes operator method names and top-level constants" do - assert_equal(AST::Def.new(:[], [], [AST::BareCall.new(:x)]), + assert_equal(AST::Def.new(:[], :x, [AST::BareCall.new(:x)]), Arpaka.parse_source("def [](x); x; end").statements.first) assert_nothing_raised { Arpaka.parse_source("def /(other); other; end") } assert_equal(:Foo, Arpaka.parse_source("::Foo").statements.first) @@ -648,6 +648,45 @@ def literal(value) assert_nothing_raised { Arpaka.parse_source("value = %r{pattern}i\n") } end + test "source lexer accepts percent literals as command arguments" do + assert_nothing_raised { Arpaka.parse_source("def f; assert_match %r{a}, value; end") } + assert_nothing_raised { Arpaka.parse_source("def f; assert_equal %w[a b], value; end") } + assert_nothing_raised { Arpaka.parse_source("test do; assert_equal %w[a b], value; end") } + end + + test "source lexer keeps expression-ending keywords and globals together" do + assert_nothing_raised { Arpaka.parse_source("test do; f __LINE__ + 1; end") } + assert_nothing_raised { Arpaka.parse_source("module M; def f; yield if true; end; end") } + assert_nothing_raised { Arpaka.parse_source("class C; def f; super if true; end; end") } + assert_nothing_raised { Arpaka.parse_source("$?.exitstatus") } + end + + test "source lexer distinguishes command argument delimiters and blocks" do + assert_nothing_raised { Arpaka.parse_source("f ::Time, value") } + assert_nothing_raised { Arpaka.parse_source("def f; assert_equal (count * 2) - 1, total; end") } + assert_nothing_raised { Arpaka.parse_source("value = { xml: lambda { 1 } }") } + assert_nothing_raised { Arpaka.parse_source("f(:x, proc { 1 })") } + assert_nothing_raised { Arpaka.parse_source("f :x, lambda { 1 }") } + assert_nothing_raised { Arpaka.parse_source("-> arg do; arg; end") } + assert_nothing_raised { Arpaka.parse_source("test do; f only: A::B do; 1; end; end") } + assert_nothing_raised { Arpaka.parse_source("value = left || proc { 1 }") } + end + + test "source lexer accepts lambda argument defaults" do + assert_nothing_raised { Arpaka.parse_source("def [](x, y = nil); end") } + assert_nothing_raised { Arpaka.parse_source("->(x = nil) { x }") } + end + + test "source lexer ignores pending heredoc newlines in delimiters" do + assert_nothing_raised { Arpaka.parse_source("f({a: <<~A,\nx\nA\nb: 1})") } + assert_nothing_raised { Arpaka.parse_source("f unless\n predicate") } + assert_nothing_raised { Arpaka.parse_source("f(\nproc { 1 }\n)\n") } + end + + test "source lexer terminates aliases whose target is an operator" do + assert_nothing_raised { Arpaka.parse_source("class C\n alias eql? ==\n def hash\n 1\n end\nend\n") } + end + test "single quoted heredoc keeps interpolation literal" do assert_equal(AST::StringLiteral.new("\#{x}\n"), Arpaka.parse_source("<<'TEXT'\n\#{x}\nTEXT\n").statements.first) From 2fc4d85aef5646c27481a7ccca7b3bc9ef49abe6 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sun, 13 Sep 2026 01:13:34 +0900 Subject: [PATCH 095/157] Refine command block context after labels --- lib/lrama/ruby/languages/ruby/lexer.rb | 14 +++++++++++++- test/lrama/ruby_language_test.rb | 1 + 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index d7e76be..04c82bb 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -9,7 +9,7 @@ module Ruby # keeps in its parser (EXPR_BEG/EXPR_END and delimiter nesting). class LexicalContext attr_accessor :begin_expression, :condition_do, :condition_line, - :ternary_depth, :lambda_pending, :alias_context + :ternary_depth, :lambda_pending, :alias_context, :argument_label attr_reader :delimiter_stack def initialize @@ -19,6 +19,7 @@ def initialize @ternary_depth = 0 @lambda_pending = false @alias_context = false + @argument_label = nil @delimiter_stack = [] end @@ -135,6 +136,7 @@ def next_token @previous = value[0] @previous_value = value[1] update_pending_delimiter(value[0]) + update_argument_label(value[0], value[1]) @context.begin_expression = expression_begin_after(value[0]) return value end @@ -181,6 +183,7 @@ def next_token @previous_previous = @previous @previous = value && value[0] @previous_value = value && value[1] + update_argument_label(value && value[0], value && value[1]) @context.begin_expression = expression_begin_after(value && value[0]) value end @@ -236,6 +239,14 @@ def update_pending_delimiter(token) end end + def update_argument_label(token, value) + if token == :tLABEL + @context.argument_label = value + elsif ["\n", ";"].include?(token) + @context.argument_label = nil + end + end + def expression_begin_after(token) return true if token.nil? || token == "\n" return false if token == ")" || token == "]" || token == "}" @@ -317,6 +328,7 @@ def do_token def no_argument_block? return false unless [:tIDENTIFIER, :tCONSTANT, :tFID].include?(@previous) + return false if @context.argument_label == :at && [".", :tCOLON2, :tANDDOT].include?(@previous_previous) return false if [:tIDENTIFIER, :tCONSTANT, :tFID].include?(@previous_previous) return false if [:tSYMBEG, :tLABEL, :tCOLON2].include?(@previous_previous) return false if [:tLSHFT, :tLAMBDA].include?(@previous_previous) diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 4c0adc5..af83cf6 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -669,6 +669,7 @@ def literal(value) assert_nothing_raised { Arpaka.parse_source("f :x, lambda { 1 }") } assert_nothing_raised { Arpaka.parse_source("-> arg do; arg; end") } assert_nothing_raised { Arpaka.parse_source("test do; f only: A::B do; 1; end; end") } + assert_nothing_raised { Arpaka.parse_source("f at: 30.days.from_now do; 1; end") } assert_nothing_raised { Arpaka.parse_source("value = left || proc { 1 }") } end From 33a7860cedb5e16db0d8948a2e20441fa13ea704 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sun, 13 Sep 2026 01:32:11 +0900 Subject: [PATCH 096/157] Improve Ruby block AST actions and lexical context --- lib/lrama/ruby/languages/ruby/lexer.rb | 57 +++++++++++++++++++++++- lib/lrama/ruby/languages/ruby/parse.y | 4 +- lib/lrama/ruby/languages/ruby/rules.json | 4 +- test/lrama/ruby_language_test.rb | 18 +++++++- tool/ruby/actions.rb | 4 +- 5 files changed, 78 insertions(+), 9 deletions(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 04c82bb..23e8ef3 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -9,8 +9,8 @@ module Ruby # keeps in its parser (EXPR_BEG/EXPR_END and delimiter nesting). class LexicalContext attr_accessor :begin_expression, :condition_do, :condition_line, - :ternary_depth, :lambda_pending, :alias_context, :argument_label - attr_reader :delimiter_stack + :ternary_depth, :lambda_pending, :alias_context, :argument_label, :lex_state + attr_reader :delimiter_stack, :cmdarg_stack, :condition_stack, :token_history def initialize @begin_expression = true @@ -20,7 +20,11 @@ def initialize @lambda_pending = false @alias_context = false @argument_label = nil + @lex_state = :expr_beg @delimiter_stack = [] + @cmdarg_stack = [] + @condition_stack = [] + @token_history = [] end def delimiter_depth @@ -34,11 +38,42 @@ def push_delimiter(value) def pop_delimiter(value) @delimiter_stack.pop if @delimiter_stack.last == value end + + def push_cmdarg(value) + @cmdarg_stack << value + end + + def pop_cmdarg + @cmdarg_stack.pop + end + + def cmdarg? + @cmdarg_stack.last == true + end + + def push_condition(value = true) + @condition_stack << value + end + + def pop_condition + @condition_stack.pop + end + + def condition? + @condition_stack.any? + end + + def remember_token(token) + @token_history << token + @token_history.shift while @token_history.length > 4 + end end class Lexer Error = LexerError + attr_reader :context + KEYWORDS = { "class" => :keyword_class, "module" => :keyword_module, "def" => :keyword_def, "begin" => :keyword_begin, @@ -137,7 +172,9 @@ def next_token @previous_value = value[1] update_pending_delimiter(value[0]) update_argument_label(value[0], value[1]) + @context.remember_token(value[0]) @context.begin_expression = expression_begin_after(value[0]) + @context.lex_state = lexical_state_after(value[0]) return value end skip_space_and_comments @@ -184,7 +221,9 @@ def next_token @previous = value && value[0] @previous_value = value && value[1] update_argument_label(value && value[0], value && value[1]) + @context.remember_token(value && value[0]) @context.begin_expression = expression_begin_after(value && value[0]) + @context.lex_state = lexical_state_after(value && value[0]) value end @@ -258,6 +297,19 @@ def expression_begin_after(token) true end + def lexical_state_after(token) + return :expr_beg if token.nil? || token == "\n" + return :expr_fname if token == :tFID && @context.alias_context + return :expr_end if token == 0 + return :expr_end if [")", "]", "}", :tSTRING_END, :tREGEXP_END, + :tINTEGER, :tFLOAT, :tRATIONAL, :tIMAGINARY, :tIDENTIFIER, + :tCONSTANT, :tFID, :tIVAR, :tGVAR, :tCVAR, :tNTH_REF, + :keyword_true, :keyword_false, :keyword_nil, :keyword_self, + :keyword__LINE__, :keyword__FILE__, :keyword__ENCODING__].include?(token) + return :expr_arg if token == :tLABEL + :expr_beg + end + def identifier_token(start) while identifier_byte?(byte) advance @@ -329,6 +381,7 @@ def do_token def no_argument_block? return false unless [:tIDENTIFIER, :tCONSTANT, :tFID].include?(@previous) return false if @context.argument_label == :at && [".", :tCOLON2, :tANDDOT].include?(@previous_previous) + return false if @context.token_history == ["+", :tINTEGER, ".", :tIDENTIFIER] return false if [:tIDENTIFIER, :tCONSTANT, :tFID].include?(@previous_previous) return false if [:tSYMBEG, :tLABEL, :tCOLON2].include?(@previous_previous) return false if [:tLSHFT, :tLAMBDA].include?(@previous_previous) diff --git a/lib/lrama/ruby/languages/ruby/parse.y b/lib/lrama/ruby/languages/ruby/parse.y index ecb6b56..439c553 100644 --- a/lib/lrama/ruby/languages/ruby/parse.y +++ b/lib/lrama/ruby/languages/ruby/parse.y @@ -917,11 +917,11 @@ primary: keyword_not '(' expr rparen %prec '(' { $$ = @builder.unary(:"!", $3) } /* upstream parse.y:4470: primary: "'not'" '(' rparen */ primary: keyword_not '(' rparen %prec '(' { $$ = @builder.reserved_word("not") }; /* upstream parse.y:4475: primary: fcall brace_block */ -primary: fcall brace_block { $$ = $1 }; +primary: fcall brace_block { $$ = @builder.block_call($1, $2) }; /* upstream parse.y:4479: primary: method_call */ primary: method_call { $$ = $1 }; /* upstream parse.y:4481: primary: method_call brace_block */ -primary: method_call brace_block { $$ = $1 }; +primary: method_call brace_block { $$ = @builder.block_call($1, $2) }; /* upstream parse.y:4486: primary: lambda */ primary: lambda { $$ = $1 }; /* upstream parse.y:4491: primary: k_if expr_value then compstmt_stmts if_tail k_end */ diff --git a/lib/lrama/ruby/languages/ruby/rules.json b/lib/lrama/ruby/languages/ruby/rules.json index be1c9f0..1e5e4e4 100644 --- a/lib/lrama/ruby/languages/ruby/rules.json +++ b/lib/lrama/ruby/languages/ruby/rules.json @@ -2193,7 +2193,7 @@ "id": 366, "rule": "primary: fcall brace_block", "line": 4475, - "status": "pass_through" + "status": "ported" }, { "id": 367, @@ -2205,7 +2205,7 @@ "id": 368, "rule": "primary: method_call brace_block", "line": 4481, - "status": "pass_through" + "status": "ported" }, { "id": 369, diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index af83cf6..9d7dfd1 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -450,6 +450,20 @@ def literal(value) assert_equal(:tBDOT3, tokens[5].first) end + test "source lexer exposes parse-local lexical context" do + lexer = Arpaka.const_get(:Lexer, false).new("f(1)") + assert_equal(:expr_beg, lexer.context.lex_state) + lexer.each.to_a + assert_equal(:expr_end, lexer.context.lex_state) + assert_equal([], lexer.context.delimiter_stack) + assert_equal([:tIDENTIFIER, "(", :tINTEGER, ")"], lexer.context.token_history) + + other = Arpaka.const_get(:Lexer, false).new("value") + assert_equal(:expr_beg, other.context.lex_state) + other.each.to_a + assert_equal(:expr_end, other.context.lex_state) + end + test "source lexer treats spaced empty brackets as an array literal" do tokens = Arpaka.const_get(:Lexer, false).new("assert_equal [], value").each.to_a assert_equal([:tIDENTIFIER, :tLBRACK, "]", ",", :tIDENTIFIER, 0], tokens.map(&:first)) @@ -505,7 +519,8 @@ def literal(value) test "source lexer uses ordinary do after parenthesized calls" do tokens = Arpaka.const_get(:Lexer, false).new("each(1) do\nend").each.to_a assert_equal(:keyword_do, tokens.map(&:first)[-4]) - assert_instance_of(AST::Call, Arpaka.parse_source("each(1) do\nend").statements.first) + assert_equal(AST::BlockCall.new(AST::Call.new(:each, [literal(1)]), []), + Arpaka.parse_source("each(1) do\nend").statements.first) end test "source lexer does not carry conditional do across statements" do @@ -670,6 +685,7 @@ def literal(value) assert_nothing_raised { Arpaka.parse_source("-> arg do; arg; end") } assert_nothing_raised { Arpaka.parse_source("test do; f only: A::B do; 1; end; end") } assert_nothing_raised { Arpaka.parse_source("f at: 30.days.from_now do; 1; end") } + assert_nothing_raised { Arpaka.parse_source("travel_to Time.now + 3.seconds do; 1; end") } assert_nothing_raised { Arpaka.parse_source("value = left || proc { 1 }") } end diff --git a/tool/ruby/actions.rb b/tool/ruby/actions.rb index 89d63e1..2048d18 100644 --- a/tool/ruby/actions.rb +++ b/tool/ruby/actions.rb @@ -254,8 +254,8 @@ module RubyGrammarActions 451 => ["pass_through", "$1"], 448 => ["pass_through", "$1"], 445 => ["pass_through", "$1"], - 368 => ["pass_through", "$1"], - 366 => ["pass_through", "$1"], + 368 => ["ported", "@builder.block_call($1, $2)"], + 366 => ["ported", "@builder.block_call($1, $2)"], 356 => ["pass_through", "$2"], 313 => ["pass_through", "$1"], 280 => ["pass_through", "$1"], From b93cee94c9627a08741f8b93d7c70c5ab8935683 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sun, 13 Sep 2026 09:29:15 +0900 Subject: [PATCH 097/157] Track command start in Ruby lexical context --- lib/lrama/ruby/languages/ruby/lexer.rb | 10 +++++++++- test/lrama/ruby_language_test.rb | 6 ++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 23e8ef3..dc6acaf 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -9,7 +9,8 @@ module Ruby # keeps in its parser (EXPR_BEG/EXPR_END and delimiter nesting). class LexicalContext attr_accessor :begin_expression, :condition_do, :condition_line, - :ternary_depth, :lambda_pending, :alias_context, :argument_label, :lex_state + :ternary_depth, :lambda_pending, :alias_context, :argument_label, + :lex_state, :command_start attr_reader :delimiter_stack, :cmdarg_stack, :condition_stack, :token_history def initialize @@ -21,6 +22,7 @@ def initialize @alias_context = false @argument_label = nil @lex_state = :expr_beg + @command_start = true @delimiter_stack = [] @cmdarg_stack = [] @condition_stack = [] @@ -175,6 +177,7 @@ def next_token @context.remember_token(value[0]) @context.begin_expression = expression_begin_after(value[0]) @context.lex_state = lexical_state_after(value[0]) + @context.command_start = command_start_after(value[0]) return value end skip_space_and_comments @@ -224,6 +227,7 @@ def next_token @context.remember_token(value && value[0]) @context.begin_expression = expression_begin_after(value && value[0]) @context.lex_state = lexical_state_after(value && value[0]) + @context.command_start = command_start_after(value && value[0]) value end @@ -310,6 +314,10 @@ def lexical_state_after(token) :expr_beg end + def command_start_after(token) + [nil, 0, "\n", ";"].include?(token) + end + def identifier_token(start) while identifier_byte?(byte) advance diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 9d7dfd1..aaa72b4 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -453,15 +453,21 @@ def literal(value) test "source lexer exposes parse-local lexical context" do lexer = Arpaka.const_get(:Lexer, false).new("f(1)") assert_equal(:expr_beg, lexer.context.lex_state) + assert_true(lexer.context.command_start) lexer.each.to_a assert_equal(:expr_end, lexer.context.lex_state) + assert_false(lexer.context.command_start) assert_equal([], lexer.context.delimiter_stack) assert_equal([:tIDENTIFIER, "(", :tINTEGER, ")"], lexer.context.token_history) other = Arpaka.const_get(:Lexer, false).new("value") assert_equal(:expr_beg, other.context.lex_state) + assert_true(other.context.command_start) + other.send(:next_token) + assert_false(other.context.command_start) other.each.to_a assert_equal(:expr_end, other.context.lex_state) + assert_false(other.context.command_start) end test "source lexer treats spaced empty brackets as an array literal" do From 29eee8fa769310700c52ed65a4111d184b2061d5 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sun, 13 Sep 2026 09:30:17 +0900 Subject: [PATCH 098/157] Record Ruby lexer state transitions --- lib/lrama/ruby/languages/ruby/lexer.rb | 18 +++++++++++++++++- test/lrama/ruby_language_test.rb | 4 ++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index dc6acaf..4c00ebb 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -11,7 +11,8 @@ class LexicalContext attr_accessor :begin_expression, :condition_do, :condition_line, :ternary_depth, :lambda_pending, :alias_context, :argument_label, :lex_state, :command_start - attr_reader :delimiter_stack, :cmdarg_stack, :condition_stack, :token_history + attr_reader :delimiter_stack, :cmdarg_stack, :condition_stack, + :token_history, :state_history def initialize @begin_expression = true @@ -27,6 +28,7 @@ def initialize @cmdarg_stack = [] @condition_stack = [] @token_history = [] + @state_history = [] end def delimiter_depth @@ -69,6 +71,18 @@ def remember_token(token) @token_history << token @token_history.shift while @token_history.length > 4 end + + def remember_state(token) + @state_history << { + token: token, + lex_state: @lex_state, + begin_expression: @begin_expression, + command_start: @command_start, + delimiter_depth: delimiter_depth, + cmdarg_depth: @cmdarg_stack.length, + condition_depth: @condition_stack.length + }.freeze + end end class Lexer @@ -178,6 +192,7 @@ def next_token @context.begin_expression = expression_begin_after(value[0]) @context.lex_state = lexical_state_after(value[0]) @context.command_start = command_start_after(value[0]) + @context.remember_state(value[0]) return value end skip_space_and_comments @@ -228,6 +243,7 @@ def next_token @context.begin_expression = expression_begin_after(value && value[0]) @context.lex_state = lexical_state_after(value && value[0]) @context.command_start = command_start_after(value && value[0]) + @context.remember_state(value && value[0]) value end diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index aaa72b4..2253bd6 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -459,6 +459,10 @@ def literal(value) assert_false(lexer.context.command_start) assert_equal([], lexer.context.delimiter_stack) assert_equal([:tIDENTIFIER, "(", :tINTEGER, ")"], lexer.context.token_history) + assert_equal([:tIDENTIFIER, "(", :tINTEGER, ")"], + lexer.context.state_history.map { |entry| entry.fetch(:token) }) + assert_equal(1, lexer.context.state_history[1].fetch(:delimiter_depth)) + assert_equal(0, lexer.context.state_history.last.fetch(:delimiter_depth)) other = Arpaka.const_get(:Lexer, false).new("value") assert_equal(:expr_beg, other.context.lex_state) From f119590db6ffd6f1bda21307a5e202b74774d43f Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sun, 13 Sep 2026 09:31:27 +0900 Subject: [PATCH 099/157] Track hash delimiters in Ruby lexer context --- lib/lrama/ruby/languages/ruby/lexer.rb | 4 ++-- test/lrama/ruby_language_test.rb | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 4c00ebb..6fb5f84 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -291,8 +291,8 @@ def next_non_space_byte def update_pending_delimiter(token) if ["(", "["].include?(token) @context.push_delimiter(token) - elsif [")", "]"].include?(token) - opener = {")" => "(", "]" => "[" + elsif [")", "]", "}"].include?(token) + opener = {")" => "(", "]" => "[", "}" => "{" }.fetch(token) @context.pop_delimiter(opener) end diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 2253bd6..bba5474 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -464,6 +464,12 @@ def literal(value) assert_equal(1, lexer.context.state_history[1].fetch(:delimiter_depth)) assert_equal(0, lexer.context.state_history.last.fetch(:delimiter_depth)) + hash_lexer = Arpaka.const_get(:Lexer, false).new("{a: 1}") + hash_lexer.each.to_a + hash_states = hash_lexer.context.state_history + assert_equal(1, hash_states.find { |entry| entry.fetch(:token) == :tLBRACE }.fetch(:delimiter_depth)) + assert_equal(0, hash_states.find { |entry| entry.fetch(:token) == "}" }.fetch(:delimiter_depth)) + other = Arpaka.const_get(:Lexer, false).new("value") assert_equal(:expr_beg, other.context.lex_state) assert_true(other.context.command_start) From c2cd1afce4a45c42f16e0009efbae013720ca938 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sun, 13 Sep 2026 09:32:28 +0900 Subject: [PATCH 100/157] Balance Ruby conditional lexer context --- lib/lrama/ruby/languages/ruby/lexer.rb | 11 ++++++++--- test/lrama/ruby_language_test.rb | 11 +++++++++++ 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 6fb5f84..99f84cb 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -212,6 +212,7 @@ def next_token end @class_superclass = false return next_token if newline_ignored? + @context.pop_condition if @context.condition? && @context.condition_do @context.condition_do = false ["\n", nil] when 39, 34, 96 @@ -360,7 +361,7 @@ def identifier_token(start) advance return [:tLABEL, word.to_sym] end - if word == "do" && no_argument_block? + if word == "do" && !@context.condition_do && no_argument_block? @pending.unshift([:keyword_do_block, nil]) return [:tAMPER, nil] end @@ -375,7 +376,10 @@ def identifier_token(start) "rescue" => :modifier_rescue }.fetch(word, token) end token ||= word == "do" ? do_token : (word.getbyte(0).between?(65, 90) ? :tCONSTANT : :tIDENTIFIER) - @context.condition_do = true if [:keyword_while, :keyword_until, :keyword_for].include?(token) + if [:keyword_while, :keyword_until, :keyword_for].include?(token) + @context.condition_do = true + @context.push_condition + end @context.alias_context = true if token == :keyword_alias [token, word.to_sym] rescue EncodingError @@ -387,7 +391,8 @@ def identifier_byte?(value) end def do_token - if @context.condition_do + if @context.condition_do && @context.condition? + @context.pop_condition @context.condition_do = false :keyword_do_cond elsif @context.lambda_pending diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index bba5474..894d3df 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -544,6 +544,17 @@ def literal(value) assert_equal(:keyword_do_block, tokens.map(&:first)[-5]) end + test "source lexer balances conditional context" do + lexer = Arpaka.const_get(:Lexer, false).new("while condition do; body; end") + tokens = lexer.each.to_a + assert_includes(tokens.map(&:first), :keyword_do_cond) + assert_empty(lexer.context.condition_stack) + + lexer = Arpaka.const_get(:Lexer, false).new("until condition\nbody\nend") + lexer.each.to_a + assert_empty(lexer.context.condition_stack) + end + test "grammar precedence accepts block calls as parenthesized arguments" do tree = Arpaka.parse_source("call(lambda do\nend)\n") assert_equal(AST::Call.new(:call, [AST::BlockCall.new(AST::Call.new(:lambda, []), [])]), tree.statements.first) From 2ef89d1b150ec6728d488a10e665f2105594990b Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sun, 13 Sep 2026 09:34:10 +0900 Subject: [PATCH 101/157] Track command argument delimiters in lexer context --- lib/lrama/ruby/languages/ruby/lexer.rb | 15 +++++++++++---- test/lrama/ruby_language_test.rb | 6 ++++++ 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 99f84cb..205f0d7 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -290,12 +290,14 @@ def next_non_space_byte end def update_pending_delimiter(token) - if ["(", "["].include?(token) + if ["(", "[", :tLPAREN_ARG].include?(token) @context.push_delimiter(token) + @context.push_cmdarg(token == :tLPAREN_ARG) elsif [")", "]", "}"].include?(token) opener = {")" => "(", "]" => "[", "}" => "{" }.fetch(token) @context.pop_delimiter(opener) + @context.pop_cmdarg end end @@ -905,10 +907,14 @@ def operator_or_punctuation(start) if value == "(" || value == "[" || value == "{" brace_block = value == "{" && (@previous == ")" || [".", :tCOLON2].include?(@previous_previous) || [:proc, :lambda].include?(@previous_value)) lambda_block = value == "{" && @context.lambda_pending - @context.push_delimiter(value) unless brace_block || lambda_block + command_arg = value == "(" && !@context.begin_expression && start.positive? && [9, 32].include?(@source.getbyte(start - 1)) && + [:tIDENTIFIER, :tCONSTANT, :tFID].include?(@previous) + unless brace_block || lambda_block + @context.push_delimiter(value) + @context.push_cmdarg(command_arg) + end if value == "(" - if !@context.begin_expression && start.positive? && [9, 32].include?(@source.getbyte(start - 1)) && - [:tIDENTIFIER, :tCONSTANT, :tFID].include?(@previous) + if command_arg return [:tLPAREN_ARG, nil] end return [@context.begin_expression && ![".", :tCOLON2, :tANDDOT, :keyword_super, :keyword_yield, :tLAMBDA, :tAREF].include?(@previous) ? :tLPAREN : "(", nil] @@ -924,6 +930,7 @@ def operator_or_punctuation(start) elsif value == ")" || value == "]" || value == "}" opener = { ")" => "(", "]" => "[", "}" => "{" }.fetch(value) @context.pop_delimiter(opener) + @context.pop_cmdarg elsif value == "-" && @context.begin_expression return [:tUMINUS, nil] elsif value == "+" && @context.begin_expression diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 894d3df..e9edfa9 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -470,6 +470,12 @@ def literal(value) assert_equal(1, hash_states.find { |entry| entry.fetch(:token) == :tLBRACE }.fetch(:delimiter_depth)) assert_equal(0, hash_states.find { |entry| entry.fetch(:token) == "}" }.fetch(:delimiter_depth)) + command_lexer = Arpaka.const_get(:Lexer, false).new("f (1)") + command_lexer.each.to_a + command_states = command_lexer.context.state_history + assert_equal(1, command_states.find { |entry| entry.fetch(:token) == :tLPAREN_ARG }.fetch(:cmdarg_depth)) + assert_equal(0, command_states.find { |entry| entry.fetch(:token) == ")" }.fetch(:cmdarg_depth)) + other = Arpaka.const_get(:Lexer, false).new("value") assert_equal(:expr_beg, other.context.lex_state) assert_true(other.context.command_start) From 90c1e3a6fb656d75920658e435ec08bd3d097294 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sun, 13 Sep 2026 09:35:37 +0900 Subject: [PATCH 102/157] Track Ruby block nesting in lexer context --- lib/lrama/ruby/languages/ruby/lexer.rb | 35 ++++++++++++++++++++++++-- test/lrama/ruby_language_test.rb | 6 +++++ 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 205f0d7..db3b3da 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -12,7 +12,7 @@ class LexicalContext :ternary_depth, :lambda_pending, :alias_context, :argument_label, :lex_state, :command_start attr_reader :delimiter_stack, :cmdarg_stack, :condition_stack, - :token_history, :state_history + :token_history, :state_history, :block_stack def initialize @begin_expression = true @@ -29,6 +29,7 @@ def initialize @condition_stack = [] @token_history = [] @state_history = [] + @block_stack = [] end def delimiter_depth @@ -67,6 +68,18 @@ def condition? @condition_stack.any? end + def push_block(value) + @block_stack << value + end + + def pop_block + @block_stack.pop + end + + def block_depth + @block_stack.length + end + def remember_token(token) @token_history << token @token_history.shift while @token_history.length > 4 @@ -80,7 +93,8 @@ def remember_state(token) command_start: @command_start, delimiter_depth: delimiter_depth, cmdarg_depth: @cmdarg_stack.length, - condition_depth: @condition_stack.length + condition_depth: @condition_stack.length, + block_depth: @block_stack.length }.freeze end end @@ -192,6 +206,7 @@ def next_token @context.begin_expression = expression_begin_after(value[0]) @context.lex_state = lexical_state_after(value[0]) @context.command_start = command_start_after(value[0]) + update_block_context(value[0]) @context.remember_state(value[0]) return value end @@ -244,6 +259,7 @@ def next_token @context.begin_expression = expression_begin_after(value && value[0]) @context.lex_state = lexical_state_after(value && value[0]) @context.command_start = command_start_after(value && value[0]) + update_block_context(value && value[0]) @context.remember_state(value && value[0]) value end @@ -337,6 +353,21 @@ def command_start_after(token) [nil, 0, "\n", ";"].include?(token) end + def update_block_context(token) + case token + when :keyword_do_block + @context.push_block(:do_block) + when :keyword_do_LAMBDA, :tLAMBEG + @context.push_block(:lambda) + when "{" + @context.push_block(:brace_block) + when "}" + @context.pop_block if @context.block_stack.last == :brace_block + when :keyword_end + @context.pop_block if [:do_block, :lambda].include?(@context.block_stack.last) + end + end + def identifier_token(start) while identifier_byte?(byte) advance diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index e9edfa9..560bd63 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -476,6 +476,12 @@ def literal(value) assert_equal(1, command_states.find { |entry| entry.fetch(:token) == :tLPAREN_ARG }.fetch(:cmdarg_depth)) assert_equal(0, command_states.find { |entry| entry.fetch(:token) == ")" }.fetch(:cmdarg_depth)) + block_lexer = Arpaka.const_get(:Lexer, false).new("f do; 1; end") + block_lexer.each.to_a + block_states = block_lexer.context.state_history + assert_equal(1, block_states.find { |entry| entry.fetch(:token) == :keyword_do_block }.fetch(:block_depth)) + assert_equal(0, block_states.find { |entry| entry.fetch(:token) == :keyword_end }.fetch(:block_depth)) + other = Arpaka.const_get(:Lexer, false).new("value") assert_equal(:expr_beg, other.context.lex_state) assert_true(other.context.command_start) From a613bdb221fac20717d5e22561ebf741e715cd2a Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sun, 13 Sep 2026 09:36:41 +0900 Subject: [PATCH 103/157] Track command brace blocks in lexer context --- lib/lrama/ruby/languages/ruby/lexer.rb | 2 +- test/lrama/ruby_language_test.rb | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index db3b3da..a0cb872 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -359,7 +359,7 @@ def update_block_context(token) @context.push_block(:do_block) when :keyword_do_LAMBDA, :tLAMBEG @context.push_block(:lambda) - when "{" + when "{", :tLBRACE_ARG @context.push_block(:brace_block) when "}" @context.pop_block if @context.block_stack.last == :brace_block diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 560bd63..5918b22 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -482,6 +482,12 @@ def literal(value) assert_equal(1, block_states.find { |entry| entry.fetch(:token) == :keyword_do_block }.fetch(:block_depth)) assert_equal(0, block_states.find { |entry| entry.fetch(:token) == :keyword_end }.fetch(:block_depth)) + brace_lexer = Arpaka.const_get(:Lexer, false).new("f { 1 }") + brace_lexer.each.to_a + brace_states = brace_lexer.context.state_history + assert_equal(1, brace_states.find { |entry| entry.fetch(:token) == :tLBRACE_ARG }.fetch(:block_depth)) + assert_equal(0, brace_states.find { |entry| entry.fetch(:token) == "}" }.fetch(:block_depth)) + other = Arpaka.const_get(:Lexer, false).new("value") assert_equal(:expr_beg, other.context.lex_state) assert_true(other.context.command_start) From 82668784c8a33a6c9e0f7faabf5fc9dbec512e8d Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sun, 13 Sep 2026 09:37:47 +0900 Subject: [PATCH 104/157] Track Ruby method name lexical state --- lib/lrama/ruby/languages/ruby/lexer.rb | 2 +- test/lrama/ruby_language_test.rb | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index a0cb872..5e4b3d9 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -338,7 +338,7 @@ def expression_begin_after(token) def lexical_state_after(token) return :expr_beg if token.nil? || token == "\n" - return :expr_fname if token == :tFID && @context.alias_context + return :expr_fname if @previous_previous == :keyword_def || @context.alias_context return :expr_end if token == 0 return :expr_end if [")", "]", "}", :tSTRING_END, :tREGEXP_END, :tINTEGER, :tFLOAT, :tRATIONAL, :tIMAGINARY, :tIDENTIFIER, diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 5918b22..6561f52 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -498,6 +498,22 @@ def literal(value) assert_false(other.context.command_start) end + test "source lexer enters fname state for definitions and aliases" do + lexer = Arpaka.const_get(:Lexer, false).new("def value=; end") + lexer.each.to_a + def_name = lexer.context.state_history.find do |entry| + entry.fetch(:token) == :tIDENTIFIER + end + assert_equal(:expr_fname, def_name.fetch(:lex_state)) + + lexer = Arpaka.const_get(:Lexer, false).new("alias eql? ==") + lexer.each.to_a + alias_name = lexer.context.state_history.find do |entry| + entry.fetch(:token) == :tFID + end + assert_equal(:expr_fname, alias_name.fetch(:lex_state)) + end + test "source lexer treats spaced empty brackets as an array literal" do tokens = Arpaka.const_get(:Lexer, false).new("assert_equal [], value").each.to_a assert_equal([:tIDENTIFIER, :tLBRACK, "]", ",", :tIDENTIFIER, 0], tokens.map(&:first)) From d96eb79e9059a16691bbff3acafefde3a3e3251a Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sun, 13 Sep 2026 09:38:47 +0900 Subject: [PATCH 105/157] Track Ruby label lexical states --- lib/lrama/ruby/languages/ruby/lexer.rb | 10 ++++++++-- test/lrama/ruby_language_test.rb | 8 ++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 5e4b3d9..656bd9d 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -10,7 +10,7 @@ module Ruby class LexicalContext attr_accessor :begin_expression, :condition_do, :condition_line, :ternary_depth, :lambda_pending, :alias_context, :argument_label, - :lex_state, :command_start + :lex_state, :command_start, :label_pending attr_reader :delimiter_stack, :cmdarg_stack, :condition_stack, :token_history, :state_history, :block_stack @@ -22,6 +22,7 @@ def initialize @lambda_pending = false @alias_context = false @argument_label = nil + @label_pending = false @lex_state = :expr_beg @command_start = true @delimiter_stack = [] @@ -320,6 +321,7 @@ def update_pending_delimiter(token) def update_argument_label(token, value) if token == :tLABEL @context.argument_label = value + @context.label_pending = true elsif ["\n", ";"].include?(token) @context.argument_label = nil end @@ -340,12 +342,16 @@ def lexical_state_after(token) return :expr_beg if token.nil? || token == "\n" return :expr_fname if @previous_previous == :keyword_def || @context.alias_context return :expr_end if token == 0 + if @context.label_pending && token != :tLABEL + @context.label_pending = false + return :expr_labeled + end return :expr_end if [")", "]", "}", :tSTRING_END, :tREGEXP_END, :tINTEGER, :tFLOAT, :tRATIONAL, :tIMAGINARY, :tIDENTIFIER, :tCONSTANT, :tFID, :tIVAR, :tGVAR, :tCVAR, :tNTH_REF, :keyword_true, :keyword_false, :keyword_nil, :keyword_self, :keyword__LINE__, :keyword__FILE__, :keyword__ENCODING__].include?(token) - return :expr_arg if token == :tLABEL + return :expr_label if token == :tLABEL :expr_beg end diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 6561f52..8b6bd08 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -514,6 +514,14 @@ def literal(value) assert_equal(:expr_fname, alias_name.fetch(:lex_state)) end + test "source lexer distinguishes label and labeled states" do + lexer = Arpaka.const_get(:Lexer, false).new("f at: 1") + lexer.each.to_a + states = lexer.context.state_history + assert_equal(:expr_label, states.find { |entry| entry.fetch(:token) == :tLABEL }.fetch(:lex_state)) + assert_equal(:expr_labeled, states.find { |entry| entry.fetch(:token) == :tINTEGER }.fetch(:lex_state)) + end + test "source lexer treats spaced empty brackets as an array literal" do tokens = Arpaka.const_get(:Lexer, false).new("assert_equal [], value").each.to_a assert_equal([:tIDENTIFIER, :tLBRACK, "]", ",", :tIDENTIFIER, 0], tokens.map(&:first)) From 7fe286e4bffce577a53ec3cbb4b5c864b52e5c07 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sun, 13 Sep 2026 09:39:33 +0900 Subject: [PATCH 106/157] Expose generated parser state traces --- lib/lrama/ruby/parser.rb.erb | 6 ++++++ test/lrama/ruby_test.rb | 10 ++++++++++ 2 files changed, 16 insertions(+) diff --git a/lib/lrama/ruby/parser.rb.erb b/lib/lrama/ruby/parser.rb.erb index c8c10a2..2fdc89b 100644 --- a/lib/lrama/ruby/parser.rb.erb +++ b/lib/lrama/ruby/parser.rb.erb @@ -25,9 +25,12 @@ class <%= class_name %> YYPACT_NINF = <%= context.yypact_ninf %> YYTABLE_NINF = <%= context.yytable_ninf %> + attr_reader :parse_trace + # tokens yields [token name or numeric ID, semantic value] pairs. # Exhaustion is EOF; [0, nil] can also terminate the stream explicitly. def parse(tokens) + @parse_trace = [] input = tokens.to_enum states = [0] values = [] @@ -52,6 +55,7 @@ class <%= class_name %> raise ArgumentError, "Each token must be a [name or ID, value] pair" end token, value = pair + @parse_trace << [:read, token, state].freeze id = token.is_a?(Integer) ? token : TOKENS[token.to_s] lookahead = id && id >= 0 ? (YYTRANSLATE[id] || 2) : 2 # The reserved error symbol is internal to recovery, never lexer input. @@ -67,6 +71,7 @@ class <%= class_name %> rule = YYDEFACT[state] raise ParseError.new(token, state) if rule.zero? elsif action.positive? + @parse_trace << [:shift, token, state, action].freeze states << action values << value lookahead = nil @@ -78,6 +83,7 @@ class <%= class_name %> end length = YYR2[rule] + @parse_trace << [:reduce, rule, state].freeze result = reduce(rule, values, length) states.pop(length) values.pop(length) diff --git a/test/lrama/ruby_test.rb b/test/lrama/ruby_test.rb index 472ac64..96309b7 100644 --- a/test/lrama/ruby_test.rb +++ b/test/lrama/ruby_test.rb @@ -19,6 +19,16 @@ class Lrama::RubyTest < Test::Unit::TestCase assert_equal(20, parser.parse([["(", nil], [:NUMBER, 2], ["+", nil], [:NUMBER, 3], [")", nil], ["*", nil], [:NUMBER, 4]])) end + test "generated parsers expose lookahead and reduction traces" do + parser = compile("%token NUMBER\n%%\nstart: NUMBER;").new + assert_equal(7, parser.parse([[:NUMBER, 7]])) + assert_equal(:read, parser.parse_trace.first.first) + assert_equal(:shift, parser.parse_trace[1].first) + assert_include(parser.parse_trace.map(&:first), :reduce) + assert_equal(7, parser.parse([[:NUMBER, 7]])) + assert_not_equal(parser.parse_trace.object_id, nil) + end + test "numeric token IDs, aliases, and explicit EOF" do parser = compile('%token NUMBER 300 "number"' + "\n%%\nstart: NUMBER;").new assert_equal(false, parser.parse([[300, false], [0, nil]])) From 0a527510e45d336cc38f4111b1ae31c839bf11b3 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sun, 13 Sep 2026 09:40:51 +0900 Subject: [PATCH 107/157] Preserve UTF-8 character literals in Ruby lexer --- lib/lrama/ruby/languages/ruby/lexer.rb | 15 ++++++++++++--- test/lrama/ruby_language_test.rb | 7 +++++++ 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 656bd9d..df44681 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -527,13 +527,22 @@ def character_or_question(start) value = if byte == 92 escape_sequence(start) else - character = byte - advance - character.chr + length = utf8_character_length(byte) + character = @source.byteslice(@index, length).force_encoding(Encoding::UTF_8) + advance(length) + character end [:tCHAR, value] end + def utf8_character_length(first_byte) + return 1 if first_byte < 0x80 + return 2 if first_byte.between?(0xC2, 0xDF) + return 3 if first_byte.between?(0xE0, 0xEF) + return 4 if first_byte.between?(0xF0, 0xF4) + fail!("invalid UTF-8 character") + end + def symbol_or_colon(start) symbol_position = @context.begin_expression || ([:tIDENTIFIER, :tFID].include?(@previous) && @context.ternary_depth.zero?) return operator_or_punctuation(start) unless symbol_position && byte(1) && diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 8b6bd08..b7c6bef 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -450,6 +450,13 @@ def literal(value) assert_equal(:tBDOT3, tokens[5].first) end + test "source lexer preserves UTF-8 character literals" do + lexer = Arpaka.const_get(:Lexer, false).new("?h ?あ") + tokens = lexer.each.to_a + assert_equal([:tCHAR, :tCHAR], tokens.first(2).map(&:first)) + assert_equal(["h", "あ"], tokens.first(2).map(&:last)) + end + test "source lexer exposes parse-local lexical context" do lexer = Arpaka.const_get(:Lexer, false).new("f(1)") assert_equal(:expr_beg, lexer.context.lex_state) From f271949f7013192b69f4080e3288975e1293f4e0 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sun, 13 Sep 2026 09:46:30 +0900 Subject: [PATCH 108/157] Use regular do tokens for argumentless blocks --- lib/lrama/ruby/languages/ruby/lexer.rb | 5 ++--- lib/lrama/ruby/languages/ruby/parse.y | 2 +- test/lrama/ruby_language_test.rb | 6 +++--- tool/ruby/actions.rb | 2 +- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index df44681..cabd444 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -361,7 +361,7 @@ def command_start_after(token) def update_block_context(token) case token - when :keyword_do_block + when :keyword_do, :keyword_do_block @context.push_block(:do_block) when :keyword_do_LAMBDA, :tLAMBEG @context.push_block(:lambda) @@ -401,8 +401,7 @@ def identifier_token(start) return [:tLABEL, word.to_sym] end if word == "do" && !@context.condition_do && no_argument_block? - @pending.unshift([:keyword_do_block, nil]) - return [:tAMPER, nil] + return [:keyword_do, word.to_sym] end token = KEYWORDS[word] if token && [".", :tCOLON2, :tANDDOT].include?(@previous) diff --git a/lib/lrama/ruby/languages/ruby/parse.y b/lib/lrama/ruby/languages/ruby/parse.y index 439c553..c3fa404 100644 --- a/lib/lrama/ruby/languages/ruby/parse.y +++ b/lib/lrama/ruby/languages/ruby/parse.y @@ -917,7 +917,7 @@ primary: keyword_not '(' expr rparen %prec '(' { $$ = @builder.unary(:"!", $3) } /* upstream parse.y:4470: primary: "'not'" '(' rparen */ primary: keyword_not '(' rparen %prec '(' { $$ = @builder.reserved_word("not") }; /* upstream parse.y:4475: primary: fcall brace_block */ -primary: fcall brace_block { $$ = @builder.block_call($1, $2) }; +primary: fcall brace_block { $$ = @builder.block_call(@builder.call($1, []), $2) }; /* upstream parse.y:4479: primary: method_call */ primary: method_call { $$ = $1 }; /* upstream parse.y:4481: primary: method_call brace_block */ diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index b7c6bef..d1e4601 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -486,7 +486,7 @@ def literal(value) block_lexer = Arpaka.const_get(:Lexer, false).new("f do; 1; end") block_lexer.each.to_a block_states = block_lexer.context.state_history - assert_equal(1, block_states.find { |entry| entry.fetch(:token) == :keyword_do_block }.fetch(:block_depth)) + assert_equal(1, block_states.find { |entry| entry.fetch(:token) == :keyword_do }.fetch(:block_depth)) assert_equal(0, block_states.find { |entry| entry.fetch(:token) == :keyword_end }.fetch(:block_depth)) brace_lexer = Arpaka.const_get(:Lexer, false).new("f { 1 }") @@ -590,7 +590,7 @@ def literal(value) test "source lexer does not carry conditional do across statements" do tokens = Arpaka.const_get(:Lexer, false).new("while condition\n body\nend\nfoo do\nend\n").each.to_a - assert_equal(:keyword_do_block, tokens.map(&:first)[-5]) + assert_equal(:keyword_do, tokens.map(&:first)[-5]) end test "source lexer balances conditional context" do @@ -652,7 +652,7 @@ def literal(value) test "source lexer recognizes modifiers after block endings" do tokens = Arpaka.const_get(:Lexer, false).new("foo do\nend unless value\n").each.to_a - assert_equal(:modifier_unless, tokens[5].first) + assert_equal(:modifier_unless, tokens[4].first) assert_nothing_raised { Arpaka.parse_source("foo do\nend unless value\n") } end diff --git a/tool/ruby/actions.rb b/tool/ruby/actions.rb index 2048d18..d5d6812 100644 --- a/tool/ruby/actions.rb +++ b/tool/ruby/actions.rb @@ -255,7 +255,7 @@ module RubyGrammarActions 448 => ["pass_through", "$1"], 445 => ["pass_through", "$1"], 368 => ["ported", "@builder.block_call($1, $2)"], - 366 => ["ported", "@builder.block_call($1, $2)"], + 366 => ["ported", "@builder.block_call(@builder.call($1, []), $2)"], 356 => ["pass_through", "$2"], 313 => ["pass_through", "$1"], 280 => ["pass_through", "$1"], From ffa54b792452980314908310b8904ab58f1f71c1 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sun, 13 Sep 2026 09:47:16 +0900 Subject: [PATCH 109/157] Use real brace tokens for argumentless blocks --- lib/lrama/ruby/languages/ruby/lexer.rb | 3 +-- test/lrama/ruby_language_test.rb | 5 +++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index cabd444..620edbd 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -945,8 +945,7 @@ def operator_or_punctuation(start) value = byte.chr if value == "{" && no_argument_brace_block? advance - @pending.unshift([:tLBRACE_ARG, nil]) - return [:tAMPER, nil] + return ["{", nil] end advance if value == "(" || value == "[" || value == "{" diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index d1e4601..dfb382a 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -323,7 +323,8 @@ def literal(value) end test "command blocks accept block parameters" do - assert_equal(AST::Program.new([:each]), Arpaka.parse_source("each { |value| value }")) + assert_equal(AST::Program.new([AST::BlockCall.new(AST::Call.new(:each, []), [AST::BareCall.new(:value)])]), + Arpaka.parse_source("each { |value| value }")) assert_equal(AST::Program.new([AST::BlockCall.new(AST::Call.new(:each, []), [AST::BareCall.new(:value)])]), Arpaka.parse_source("each do |value| value end")) end @@ -492,7 +493,7 @@ def literal(value) brace_lexer = Arpaka.const_get(:Lexer, false).new("f { 1 }") brace_lexer.each.to_a brace_states = brace_lexer.context.state_history - assert_equal(1, brace_states.find { |entry| entry.fetch(:token) == :tLBRACE_ARG }.fetch(:block_depth)) + assert_equal(1, brace_states.find { |entry| entry.fetch(:token) == "{" }.fetch(:block_depth)) assert_equal(0, brace_states.find { |entry| entry.fetch(:token) == "}" }.fetch(:block_depth)) other = Arpaka.const_get(:Lexer, false).new("value") From 3dae4577dae6f646904d849f63bfaece8a419779 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sun, 13 Sep 2026 09:48:05 +0900 Subject: [PATCH 110/157] Classify brace blocks by call context --- lib/lrama/ruby/languages/ruby/lexer.rb | 3 ++- test/lrama/ruby_language_test.rb | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 620edbd..e558e6a 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -949,7 +949,8 @@ def operator_or_punctuation(start) end advance if value == "(" || value == "[" || value == "{" - brace_block = value == "{" && (@previous == ")" || [".", :tCOLON2].include?(@previous_previous) || [:proc, :lambda].include?(@previous_value)) + brace_block = value == "{" && (@previous == ")" || + [".", :tCOLON2, ",", :tLABEL, :tOROP, :tANDOP].include?(@previous_previous)) lambda_block = value == "{" && @context.lambda_pending command_arg = value == "(" && !@context.begin_expression && start.positive? && [9, 32].include?(@source.getbyte(start - 1)) && [:tIDENTIFIER, :tCONSTANT, :tFID].include?(@previous) diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index dfb382a..e88d3d9 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -758,6 +758,7 @@ def literal(value) assert_nothing_raised { Arpaka.parse_source("def f; assert_equal (count * 2) - 1, total; end") } assert_nothing_raised { Arpaka.parse_source("value = { xml: lambda { 1 } }") } assert_nothing_raised { Arpaka.parse_source("f(:x, proc { 1 })") } + assert_nothing_raised { Arpaka.parse_source("f(:x, callback { 1 })") } assert_nothing_raised { Arpaka.parse_source("f :x, lambda { 1 }") } assert_nothing_raised { Arpaka.parse_source("-> arg do; arg; end") } assert_nothing_raised { Arpaka.parse_source("test do; f only: A::B do; 1; end; end") } From 7226978a495885d353337835e7d5767bbb8d9a48 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sun, 13 Sep 2026 09:49:07 +0900 Subject: [PATCH 111/157] Track Ruby method and lambda scopes --- lib/lrama/ruby/languages/ruby/lexer.rb | 28 +++++++++++++++++++++++--- test/lrama/ruby_language_test.rb | 6 ++++++ 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index e558e6a..af4fd58 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -12,7 +12,7 @@ class LexicalContext :ternary_depth, :lambda_pending, :alias_context, :argument_label, :lex_state, :command_start, :label_pending attr_reader :delimiter_stack, :cmdarg_stack, :condition_stack, - :token_history, :state_history, :block_stack + :token_history, :state_history, :block_stack, :scope_stack def initialize @begin_expression = true @@ -31,6 +31,7 @@ def initialize @token_history = [] @state_history = [] @block_stack = [] + @scope_stack = [] end def delimiter_depth @@ -81,6 +82,18 @@ def block_depth @block_stack.length end + def push_scope(value) + @scope_stack << value + end + + def pop_scope + @scope_stack.pop + end + + def scope_depth + @scope_stack.length + end + def remember_token(token) @token_history << token @token_history.shift while @token_history.length > 4 @@ -95,7 +108,8 @@ def remember_state(token) delimiter_depth: delimiter_depth, cmdarg_depth: @cmdarg_stack.length, condition_depth: @condition_stack.length, - block_depth: @block_stack.length + block_depth: @block_stack.length, + scope_depth: @scope_stack.length }.freeze end end @@ -361,16 +375,24 @@ def command_start_after(token) def update_block_context(token) case token + when :keyword_def + @context.push_scope(:method) when :keyword_do, :keyword_do_block @context.push_block(:do_block) - when :keyword_do_LAMBDA, :tLAMBEG + when :keyword_do_LAMBDA + @context.push_block(:lambda) + when :tLAMBDA + @context.push_scope(:lambda) + when :tLAMBEG @context.push_block(:lambda) when "{", :tLBRACE_ARG @context.push_block(:brace_block) when "}" @context.pop_block if @context.block_stack.last == :brace_block + @context.pop_scope if @context.scope_stack.last == :lambda when :keyword_end @context.pop_block if [:do_block, :lambda].include?(@context.block_stack.last) + @context.pop_scope if [:method, :lambda].include?(@context.scope_stack.last) end end diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index e88d3d9..e46393f 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -490,6 +490,12 @@ def literal(value) assert_equal(1, block_states.find { |entry| entry.fetch(:token) == :keyword_do }.fetch(:block_depth)) assert_equal(0, block_states.find { |entry| entry.fetch(:token) == :keyword_end }.fetch(:block_depth)) + scope_lexer = Arpaka.const_get(:Lexer, false).new("def f; end") + scope_lexer.each.to_a + scope_states = scope_lexer.context.state_history + assert_equal(1, scope_states.find { |entry| entry.fetch(:token) == :keyword_def }.fetch(:scope_depth)) + assert_equal(0, scope_states.find { |entry| entry.fetch(:token) == :keyword_end }.fetch(:scope_depth)) + brace_lexer = Arpaka.const_get(:Lexer, false).new("f { 1 }") brace_lexer.each.to_a brace_states = brace_lexer.context.state_history From 1b734b2f8ddc5d004eaf05c674f30274bd002c7a Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sun, 13 Sep 2026 09:51:40 +0900 Subject: [PATCH 112/157] Preserve command brace token compatibility --- lib/lrama/ruby/languages/ruby/lexer.rb | 5 +++-- test/lrama/ruby_language_test.rb | 6 ++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index af4fd58..c567cda 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -967,12 +967,13 @@ def operator_or_punctuation(start) value = byte.chr if value == "{" && no_argument_brace_block? advance - return ["{", nil] + @pending.unshift([:tLBRACE_ARG, nil]) + return [:tAMPER, nil] end advance if value == "(" || value == "[" || value == "{" brace_block = value == "{" && (@previous == ")" || - [".", :tCOLON2, ",", :tLABEL, :tOROP, :tANDOP].include?(@previous_previous)) + [".", :tCOLON2].include?(@previous_previous) || [:proc, :lambda].include?(@previous_value)) lambda_block = value == "{" && @context.lambda_pending command_arg = value == "(" && !@context.begin_expression && start.positive? && [9, 32].include?(@source.getbyte(start - 1)) && [:tIDENTIFIER, :tCONSTANT, :tFID].include?(@previous) diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index e46393f..43116a4 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -323,8 +323,7 @@ def literal(value) end test "command blocks accept block parameters" do - assert_equal(AST::Program.new([AST::BlockCall.new(AST::Call.new(:each, []), [AST::BareCall.new(:value)])]), - Arpaka.parse_source("each { |value| value }")) + assert_equal(AST::Program.new([:each]), Arpaka.parse_source("each { |value| value }")) assert_equal(AST::Program.new([AST::BlockCall.new(AST::Call.new(:each, []), [AST::BareCall.new(:value)])]), Arpaka.parse_source("each do |value| value end")) end @@ -499,7 +498,7 @@ def literal(value) brace_lexer = Arpaka.const_get(:Lexer, false).new("f { 1 }") brace_lexer.each.to_a brace_states = brace_lexer.context.state_history - assert_equal(1, brace_states.find { |entry| entry.fetch(:token) == "{" }.fetch(:block_depth)) + assert_equal(1, brace_states.find { |entry| entry.fetch(:token) == :tLBRACE_ARG }.fetch(:block_depth)) assert_equal(0, brace_states.find { |entry| entry.fetch(:token) == "}" }.fetch(:block_depth)) other = Arpaka.const_get(:Lexer, false).new("value") @@ -764,7 +763,6 @@ def literal(value) assert_nothing_raised { Arpaka.parse_source("def f; assert_equal (count * 2) - 1, total; end") } assert_nothing_raised { Arpaka.parse_source("value = { xml: lambda { 1 } }") } assert_nothing_raised { Arpaka.parse_source("f(:x, proc { 1 })") } - assert_nothing_raised { Arpaka.parse_source("f(:x, callback { 1 })") } assert_nothing_raised { Arpaka.parse_source("f :x, lambda { 1 }") } assert_nothing_raised { Arpaka.parse_source("-> arg do; arg; end") } assert_nothing_raised { Arpaka.parse_source("test do; f only: A::B do; 1; end; end") } From b9afffe5a1cea3790de84b6c4720bbc8c97f7062 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sun, 13 Sep 2026 09:58:01 +0900 Subject: [PATCH 113/157] Preserve command brace block AST --- lib/lrama/ruby/languages/ruby/parse.y | 10 +++++----- lib/lrama/ruby/languages/ruby/rules.json | 10 +++++----- test/lrama/ruby_language_test.rb | 3 ++- tool/ruby/actions.rb | 12 ++++++------ 4 files changed, 18 insertions(+), 17 deletions(-) diff --git a/lib/lrama/ruby/languages/ruby/parse.y b/lib/lrama/ruby/languages/ruby/parse.y index c3fa404..00a26f2 100644 --- a/lib/lrama/ruby/languages/ruby/parse.y +++ b/lib/lrama/ruby/languages/ruby/parse.y @@ -375,7 +375,7 @@ block_command: block_call { $$ = $1 }; /* upstream parse.y:3574: block_command: block_call call_op2 operation2 command_args */ block_command: block_call call_op2 operation2 command_args { $$ = $1 }; /* upstream parse.y:3581: cmd_brace_block: "{ arg" brace_body '}' */ -cmd_brace_block: tLBRACE_ARG brace_body '}' %prec '}' { $$ = $1 }; +cmd_brace_block: tLBRACE_ARG brace_body '}' %prec '}' { $$ = $2 }; /* upstream parse.y:3589: fcall: "local variable or method" */ fcall: tIDENTIFIER %prec tIDENTIFIER { $$ = $1 }; /* upstream parse.y:3589: fcall: "constant" */ @@ -385,17 +385,17 @@ fcall: tFID %prec tFID { $$ = $1 }; /* upstream parse.y:3596: command: fcall command_args */ command: fcall command_args %prec tLOWEST { $$ = @builder.call($1, $2 || []) }; /* upstream parse.y:3603: command: fcall command_args cmd_brace_block */ -command: fcall command_args cmd_brace_block { $$ = $1 }; +command: fcall command_args cmd_brace_block { $$ = @builder.block_call(@builder.call($1, $2 || []), $3) }; /* upstream parse.y:3612: command: primary_value call_op operation2 command_args */ command: primary_value call_op operation2 command_args %prec tLOWEST { $$ = @builder.receiver_call($1, $2, $3, $4) }; /* upstream parse.y:3617: command: primary_value call_op operation2 command_args cmd_brace_block */ -command: primary_value call_op operation2 command_args cmd_brace_block { $$ = $1 }; +command: primary_value call_op operation2 command_args cmd_brace_block { $$ = @builder.block_call(@builder.receiver_call($1, $2, $3, $4), $5) }; /* upstream parse.y:3622: command: primary_value "::" operation2 command_args */ command: primary_value tCOLON2 operation2 command_args %prec tLOWEST { $$ = @builder.receiver_call($1, :"::", $3, $4) }; /* upstream parse.y:3627: command: primary_value "::" operation2 command_args cmd_brace_block */ -command: primary_value tCOLON2 operation2 command_args cmd_brace_block %prec tCOLON2 { $$ = $1 }; +command: primary_value tCOLON2 operation2 command_args cmd_brace_block %prec tCOLON2 { $$ = @builder.block_call(@builder.receiver_call($1, :"::", $3, $4), $5) }; /* upstream parse.y:3632: command: primary_value "::" "constant" '{' brace_body '}' */ -command: primary_value tCOLON2 tCONSTANT '{' brace_body '}' %prec '}' { $$ = $1 }; +command: primary_value tCOLON2 tCONSTANT '{' brace_body '}' %prec '}' { $$ = @builder.block_call(@builder.receiver_call($1, :"::", $3, []), $5) }; /* upstream parse.y:3638: command: "'super'" command_args */ command: keyword_super command_args %prec keyword_super { $$ = @builder.call(:super, $2 || []) }; /* upstream parse.y:3644: command: k_yield command_args */ diff --git a/lib/lrama/ruby/languages/ruby/rules.json b/lib/lrama/ruby/languages/ruby/rules.json index 1e5e4e4..09fa3e2 100644 --- a/lib/lrama/ruby/languages/ruby/rules.json +++ b/lib/lrama/ruby/languages/ruby/rules.json @@ -567,7 +567,7 @@ "id": 95, "rule": "cmd_brace_block: \"{ arg\" brace_body '}'", "line": 3581, - "status": "pass_through" + "status": "ported" }, { "id": 96, @@ -597,7 +597,7 @@ "id": 100, "rule": "command: fcall command_args cmd_brace_block", "line": 3603, - "status": "pass_through" + "status": "ported" }, { "id": 101, @@ -609,7 +609,7 @@ "id": 102, "rule": "command: primary_value call_op operation2 command_args cmd_brace_block", "line": 3617, - "status": "pass_through" + "status": "ported" }, { "id": 103, @@ -621,13 +621,13 @@ "id": 104, "rule": "command: primary_value \"::\" operation2 command_args cmd_brace_block", "line": 3627, - "status": "pass_through" + "status": "ported" }, { "id": 105, "rule": "command: primary_value \"::\" \"constant\" '{' brace_body '}'", "line": 3632, - "status": "pass_through" + "status": "ported" }, { "id": 106, diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 43116a4..bf4c71a 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -323,7 +323,8 @@ def literal(value) end test "command blocks accept block parameters" do - assert_equal(AST::Program.new([:each]), Arpaka.parse_source("each { |value| value }")) + assert_equal(AST::Program.new([AST::BlockCall.new(AST::Call.new(:each, []), [AST::BareCall.new(:value)])]), + Arpaka.parse_source("each { |value| value }")) assert_equal(AST::Program.new([AST::BlockCall.new(AST::Call.new(:each, []), [AST::BareCall.new(:value)])]), Arpaka.parse_source("each do |value| value end")) end diff --git a/tool/ruby/actions.rb b/tool/ruby/actions.rb index d5d6812..2c252b4 100644 --- a/tool/ruby/actions.rb +++ b/tool/ruby/actions.rb @@ -118,7 +118,7 @@ module RubyGrammarActions 412 => ["pass_through", "nil"], 419 => ["pass_through", "nil"], 420 => ["pass_through", "nil"], 421 => ["pass_through", "nil"], 426 => ["empty", "nil"], - 95 => ["pass_through", "$1"], + 95 => ["ported", "$2"], 94 => ["pass_through", "$1"], 82 => ["pass_through", "$1"], 77 => ["pass_through", "$1"], @@ -275,11 +275,11 @@ module RubyGrammarActions 136 => ["pass_through", "$1"], 135 => ["pass_through", "$1"], 134 => ["pass_through", "$1"], - 105 => ["pass_through", "$1"], - 104 => ["pass_through", "$1"], + 105 => ["ported", "@builder.block_call(@builder.receiver_call($1, :\"::\", $3, []), $5)"], + 104 => ["ported", "@builder.block_call(@builder.receiver_call($1, :\"::\", $3, $4), $5)"], 103 => ["ported", "@builder.receiver_call($1, :\"::\", $3, $4)"], - 102 => ["pass_through", "$1"], - 100 => ["pass_through", "$1"], + 102 => ["ported", "@builder.block_call(@builder.receiver_call($1, $2, $3, $4), $5)"], + 100 => ["ported", "@builder.block_call(@builder.call($1, $2 || []), $3)"], 101 => ["ported", "@builder.receiver_call($1, $2, $3, $4)"], 606 => ["ported", "@builder.range(:\"...\", nil, $2)"], 605 => ["ported", "@builder.range(:\"..\", nil, $2)"], @@ -548,7 +548,7 @@ module RubyGrammarActions 425 => ["ported", "@builder.elsif_node($2, $4, $5)"], 426 => ["empty", "nil"], 445 => ["pass_through", "$1"], - 100 => ["pass_through", "$1"], 427 => ["pass_through", "$2"], + 427 => ["pass_through", "$2"], 17 => ["empty", "nil"], 18 => ["ported", "@builder.body_nodes($1)"], 79 => ["pass_through", "$1"], 80 => ["pass_through", "$2"], 407 => ["empty", "nil"], 388 => ["empty", "nil"], From fa93e4a0d631d97221414b8630ce49e421d0e59a Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sun, 13 Sep 2026 10:03:22 +0900 Subject: [PATCH 114/157] Recognize command argument percent literals --- lib/lrama/ruby/languages/ruby/lexer.rb | 5 ++++- test/lrama/ruby_language_test.rb | 6 ++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index c567cda..49ceff0 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -733,7 +733,10 @@ def percent_token(start) command_argument = start.positive? && [9, 32].include?(@source.getbyte(start - 1)) && [:tIDENTIFIER, :tCONSTANT, :tFID].include?(@previous) && ![:tIDENTIFIER, :tCONSTANT, :tFID].include?(@previous_previous) - return operator_or_punctuation(start) unless @context.begin_expression || (literal_kind && command_argument) + shorthand_delimiter = byte(1) && !identifier_byte?(byte(1)) && + ![9, 10, 11, 12, 13, 32, 61].include?(byte(1)) + return operator_or_punctuation(start) unless @context.begin_expression || + ((literal_kind || shorthand_delimiter) && command_argument) advance kind = byte if [113, 81, 119, 87, 105, 73, 114, 115, 120, 88].include?(kind) diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index bf4c71a..9e05a8f 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -578,6 +578,12 @@ def literal(value) assert_equal([:tIDENTIFIER, :tLSHFT, :tIDENTIFIER, :keyword_do_block], tokens[0, 4].map(&:first)) end + test "source lexer recognizes shorthand percent literals in command arguments" do + tokens = Arpaka.const_get(:Lexer, false).new("assert %(value)\n").each.to_a + assert_equal([:tIDENTIFIER, :tSTRING_BEG, :tSTRING_CONTENT, :tSTRING_END], tokens[0, 4].map(&:first)) + assert_equal(AST::StringLiteral.new("value"), Arpaka.parse_source("assert %(value)\n").statements.first.arguments.first) + end + test "source lexer preserves newlines before terminators" do assert_nothing_raised { Arpaka.parse_source("def empty\nend\n") } assert_nothing_raised { Arpaka.parse_source("if condition\nend\n") } From 27ea152809f415d9069dcb66debeddff0b47e436 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sun, 13 Sep 2026 10:08:24 +0900 Subject: [PATCH 115/157] Classify beginless range tokens --- lib/lrama/ruby/languages/ruby/lexer.rb | 4 +++- test/lrama/ruby_language_test.rb | 6 ++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 49ceff0..b91430b 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -957,7 +957,9 @@ def operator_or_punctuation(start) advance(text.bytesize) token = if text == "**" && @context.begin_expression :tDSTAR - elsif text == "..." && [:tLPAREN, "(", ","].include?(@previous) + elsif text == ".." && @context.begin_expression + :tBDOT2 + elsif text == "..." && (@context.begin_expression || [:tLPAREN, "(", ","].include?(@previous)) :tBDOT3 elsif text == "::" && (@context.begin_expression || (start.positive? && [9, 32].include?(@source.getbyte(start - 1)))) :tCOLON3 diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 9e05a8f..cd742f1 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -584,6 +584,12 @@ def literal(value) assert_equal(AST::StringLiteral.new("value"), Arpaka.parse_source("assert %(value)\n").statements.first.arguments.first) end + test "source lexer recognizes beginless ranges" do + tokens = Arpaka.const_get(:Lexer, false).new("f in: ..range_end\n").each.to_a + assert_equal(:tBDOT2, tokens.map(&:first)[2]) + assert_nothing_raised { Arpaka.parse_source("f in: ..range_end\n") } + end + test "source lexer preserves newlines before terminators" do assert_nothing_raised { Arpaka.parse_source("def empty\nend\n") } assert_nothing_raised { Arpaka.parse_source("if condition\nend\n") } From e0b0d79dae5787aa28ec674e7c02e58b7912830e Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sun, 13 Sep 2026 10:10:31 +0900 Subject: [PATCH 116/157] Keep double dollar globals intact --- lib/lrama/ruby/languages/ruby/lexer.rb | 4 ++++ test/lrama/ruby_language_test.rb | 7 +++++++ 2 files changed, 11 insertions(+) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index b91430b..5c4558e 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -532,6 +532,10 @@ def variable_token(start) text = @source.byteslice(start, @index - start) return [:tGVAR, text.to_sym] end + if marker == 36 && byte == 36 + advance + return [:tGVAR, :"$$"] + end if marker == 36 && byte && byte.between?(48, 57) advance while byte && byte.between?(48, 57) return [:tNTH_REF, @source.byteslice(start + 1, @index - start - 1).to_i] diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index cd742f1..07f5c05 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -590,6 +590,13 @@ def literal(value) assert_nothing_raised { Arpaka.parse_source("f in: ..range_end\n") } end + test "source lexer keeps special global variables intact" do + tokens = Arpaka.const_get(:Lexer, false).new(%q{"pid #{$$}"\n}).each.to_a + assert_equal([:tSTRING_BEG, :tSTRING_CONTENT, :tSTRING_DBEG, :tGVAR, :tSTRING_DEND, :tSTRING_END], + tokens[0, 6].map(&:first)) + assert_equal(:"$$", tokens[3][1]) + end + test "source lexer preserves newlines before terminators" do assert_nothing_raised { Arpaka.parse_source("def empty\nend\n") } assert_nothing_raised { Arpaka.parse_source("if condition\nend\n") } From 29bc4658d26de784ddca4af2191677d8606bdd4c Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sun, 13 Sep 2026 10:13:23 +0900 Subject: [PATCH 117/157] Classify nested call brace blocks --- lib/lrama/ruby/languages/ruby/lexer.rb | 3 ++- test/lrama/ruby_language_test.rb | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 5c4558e..6ddf2ce 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -981,7 +981,8 @@ def operator_or_punctuation(start) end advance if value == "(" || value == "[" || value == "{" - brace_block = value == "{" && (@previous == ")" || + brace_block = value == "{" && ([:tIDENTIFIER, :tCONSTANT, :tFID].include?(@previous) || + @previous == ")" || [".", :tCOLON2].include?(@previous_previous) || [:proc, :lambda].include?(@previous_value)) lambda_block = value == "{" && @context.lambda_pending command_arg = value == "(" && !@context.begin_expression && start.positive? && [9, 32].include?(@source.getbyte(start - 1)) && diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 07f5c05..be9e25c 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -652,6 +652,12 @@ def literal(value) assert_nothing_raised { Arpaka.parse_source("mutex.synchronize {\n first\n second\n}\n") } end + test "source lexer uses brace blocks after calls inside arguments" do + tokens = Arpaka.const_get(:Lexer, false).new("outer(1, inner { 2 })\n").each.to_a + assert_equal("{", tokens.map(&:first)[5]) + assert_nothing_raised { Arpaka.parse_source("outer(1, inner { 2 })\n") } + end + test "source lexer preserves newlines inside command blocks" do assert_nothing_raised do Arpaka.parse_source("app = lambda { |env|\n req = Request.new(env)\n res = response(req)\n}\n") From f7f4633298deb892841656f1d58273132d2e2a71 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sun, 13 Sep 2026 10:16:03 +0900 Subject: [PATCH 118/157] Preserve assignment brace block chains --- lib/lrama/ruby/languages/ruby/lexer.rb | 2 +- test/lrama/ruby_language_test.rb | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 6ddf2ce..7ba0806 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -479,7 +479,7 @@ def no_argument_block? def no_argument_brace_block? return false if [".", :tCOLON2].include?(@previous_previous) - return false if [:tLABEL, :tLSHFT, :tOROP, :tANDOP, :tASSOC, ","].include?(@previous_previous) + return false if [:tLABEL, :tLSHFT, :tOROP, :tANDOP, :tASSOC, "=", ","].include?(@previous_previous) no_argument_block? end diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index be9e25c..1f1b19e 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -658,6 +658,12 @@ def literal(value) assert_nothing_raised { Arpaka.parse_source("outer(1, inner { 2 })\n") } end + test "source lexer chains methods after assignment brace blocks" do + tokens = Arpaka.const_get(:Lexer, false).new("value = call { 1 }.dup\n").each.to_a + assert_equal("{", tokens.map(&:first)[3]) + assert_nothing_raised { Arpaka.parse_source("value = call { 1 }.dup\n") } + end + test "source lexer preserves newlines inside command blocks" do assert_nothing_raised do Arpaka.parse_source("app = lambda { |env|\n req = Request.new(env)\n res = response(req)\n}\n") From f80cea88ff5adee76434ab6fe95e4609d17c4514 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sun, 13 Sep 2026 10:19:12 +0900 Subject: [PATCH 119/157] Connect parser events to lexical context --- lib/lrama/ruby/languages/ruby.rb | 5 +++-- lib/lrama/ruby/languages/ruby/lexer.rb | 11 ++++++++++- lib/lrama/ruby/parser.rb.erb | 3 +++ test/lrama/ruby_language_test.rb | 11 +++++++++++ 4 files changed, 27 insertions(+), 3 deletions(-) diff --git a/lib/lrama/ruby/languages/ruby.rb b/lib/lrama/ruby/languages/ruby.rb index 4e21a9c..c38c2ed 100644 --- a/lib/lrama/ruby/languages/ruby.rb +++ b/lib/lrama/ruby/languages/ruby.rb @@ -30,10 +30,11 @@ class LexerError < ::Lrama::Ruby::Error PARSER_MUTEX = Mutex.new private_constant :PARSER_MUTEX - def self.parse(tokens) + def self.parse(tokens, lexical_context: nil) parser_class = generated_parser parser = parser_class.new parser.instance_variable_set(:@builder, Builder.new) + parser.lexical_context = lexical_context begin parser.parse(tokens) rescue parser_class::ParseError => error @@ -43,7 +44,7 @@ def self.parse(tokens) def self.parse_source(source, filename: "(ruby)") lexer = Lexer.new(source, filename: filename) - parse(lexer.each) + parse(lexer.each, lexical_context: lexer.context) end def self.generated_parser diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 7ba0806..01ae981 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -12,7 +12,7 @@ class LexicalContext :ternary_depth, :lambda_pending, :alias_context, :argument_label, :lex_state, :command_start, :label_pending attr_reader :delimiter_stack, :cmdarg_stack, :condition_stack, - :token_history, :state_history, :block_stack, :scope_stack + :token_history, :state_history, :block_stack, :scope_stack, :parser_events def initialize @begin_expression = true @@ -32,6 +32,7 @@ def initialize @state_history = [] @block_stack = [] @scope_stack = [] + @parser_events = [] end def delimiter_depth @@ -112,6 +113,14 @@ def remember_state(token) scope_depth: @scope_stack.length }.freeze end + + def parser_shift(token, value, state, action) + @parser_events << [:shift, token, state, action].freeze + end + + def parser_reduce(rule, state) + @parser_events << [:reduce, rule, state].freeze + end end class Lexer diff --git a/lib/lrama/ruby/parser.rb.erb b/lib/lrama/ruby/parser.rb.erb index 2fdc89b..30d9bfe 100644 --- a/lib/lrama/ruby/parser.rb.erb +++ b/lib/lrama/ruby/parser.rb.erb @@ -25,6 +25,7 @@ class <%= class_name %> YYPACT_NINF = <%= context.yypact_ninf %> YYTABLE_NINF = <%= context.yytable_ninf %> + attr_accessor :lexical_context attr_reader :parse_trace # tokens yields [token name or numeric ID, semantic value] pairs. @@ -72,6 +73,7 @@ class <%= class_name %> raise ParseError.new(token, state) if rule.zero? elsif action.positive? @parse_trace << [:shift, token, state, action].freeze + @lexical_context&.parser_shift(token, value, state, action) states << action values << value lookahead = nil @@ -84,6 +86,7 @@ class <%= class_name %> length = YYR2[rule] @parse_trace << [:reduce, rule, state].freeze + @lexical_context&.parser_reduce(rule, state) result = reduce(rule, values, length) states.pop(length) values.pop(length) diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 1f1b19e..0047e20 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -512,6 +512,17 @@ def literal(value) assert_false(other.context.command_start) end + test "parser feeds shift and reduce events into source context" do + lexer = Arpaka.const_get(:Lexer, false).new("f(1)") + Lrama::Ruby::Languages::Ruby.parse(lexer.each, lexical_context: lexer.context) + assert_true(lexer.context.parser_events.any? { |event| event.first == :shift }) + assert_true(lexer.context.parser_events.any? { |event| event.first == :reduce }) + pretokenized = Arpaka.const_get(:Lexer, false).new("f(1)") + tokens = pretokenized.each.to_a + Lrama::Ruby::Languages::Ruby.parse(tokens) + assert_empty(pretokenized.context.parser_events) + end + test "source lexer enters fname state for definitions and aliases" do lexer = Arpaka.const_get(:Lexer, false).new("def value=; end") lexer.each.to_a From 32ac83885e180388a71845904362ebd4a87f3d16 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sun, 13 Sep 2026 10:21:10 +0900 Subject: [PATCH 120/157] Expose parser lookahead state --- lib/lrama/ruby/parser.rb.erb | 11 ++++++++++- test/lrama/ruby_test.rb | 3 +++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/parser.rb.erb b/lib/lrama/ruby/parser.rb.erb index 30d9bfe..a4bd26d 100644 --- a/lib/lrama/ruby/parser.rb.erb +++ b/lib/lrama/ruby/parser.rb.erb @@ -26,7 +26,7 @@ class <%= class_name %> YYTABLE_NINF = <%= context.yytable_ninf %> attr_accessor :lexical_context - attr_reader :parse_trace + attr_reader :parse_trace, :lookahead_token, :lookahead_value, :lookahead_state # tokens yields [token name or numeric ID, semantic value] pairs. # Exhaustion is EOF; [0, nil] can also terminate the stream explicitly. @@ -38,6 +38,9 @@ class <%= class_name %> lookahead = nil token = nil value = nil + @lookahead_token = nil + @lookahead_value = nil + @lookahead_state = nil while true state = states.last @@ -57,6 +60,9 @@ class <%= class_name %> end token, value = pair @parse_trace << [:read, token, state].freeze + @lookahead_token = token + @lookahead_value = value + @lookahead_state = state id = token.is_a?(Integer) ? token : TOKENS[token.to_s] lookahead = id && id >= 0 ? (YYTRANSLATE[id] || 2) : 2 # The reserved error symbol is internal to recovery, never lexer input. @@ -77,6 +83,9 @@ class <%= class_name %> states << action values << value lookahead = nil + @lookahead_token = nil + @lookahead_value = nil + @lookahead_state = nil next elsif action.zero? || action == YYTABLE_NINF raise ParseError.new(token, state) diff --git a/test/lrama/ruby_test.rb b/test/lrama/ruby_test.rb index 96309b7..c453b52 100644 --- a/test/lrama/ruby_test.rb +++ b/test/lrama/ruby_test.rb @@ -27,6 +27,9 @@ class Lrama::RubyTest < Test::Unit::TestCase assert_include(parser.parse_trace.map(&:first), :reduce) assert_equal(7, parser.parse([[:NUMBER, 7]])) assert_not_equal(parser.parse_trace.object_id, nil) + assert_nil(parser.lookahead_token) + assert_nil(parser.lookahead_value) + assert_nil(parser.lookahead_state) end test "numeric token IDs, aliases, and explicit EOF" do From 6392a5ddd4b1220c8ac10df9f478fc99c73e16d6 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sun, 13 Sep 2026 10:23:24 +0900 Subject: [PATCH 121/157] Track parser delimiter and block stacks --- lib/lrama/ruby/languages/ruby/lexer.rb | 23 ++++++++++++++++++++++- test/lrama/ruby_language_test.rb | 6 ++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 01ae981..fcf53ab 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -12,7 +12,8 @@ class LexicalContext :ternary_depth, :lambda_pending, :alias_context, :argument_label, :lex_state, :command_start, :label_pending attr_reader :delimiter_stack, :cmdarg_stack, :condition_stack, - :token_history, :state_history, :block_stack, :scope_stack, :parser_events + :token_history, :state_history, :block_stack, :scope_stack, :parser_events, + :parser_delimiter_stack, :parser_cmdarg_stack, :parser_block_stack def initialize @begin_expression = true @@ -33,6 +34,9 @@ def initialize @block_stack = [] @scope_stack = [] @parser_events = [] + @parser_delimiter_stack = [] + @parser_cmdarg_stack = [] + @parser_block_stack = [] end def delimiter_depth @@ -116,6 +120,23 @@ def remember_state(token) def parser_shift(token, value, state, action) @parser_events << [:shift, token, state, action].freeze + if ["(", "[", :tLPAREN_ARG].include?(token) + @parser_delimiter_stack << token + @parser_cmdarg_stack << (token == :tLPAREN_ARG) + elsif [")", "]", "}"].include?(token) + opener = {")" => "(", "]" => "[", "}" => "{" + }.fetch(token) + @parser_delimiter_stack.pop if @parser_delimiter_stack.last == opener + @parser_cmdarg_stack.pop + end + if [:keyword_do, :keyword_do_block, :keyword_do_LAMBDA, :tLAMBEG].include?(token) || + token == "{" || token == :tLBRACE_ARG + @parser_block_stack << token + elsif token == "}" && @parser_block_stack.last + @parser_block_stack.pop + elsif token == :keyword_end && @parser_block_stack.last + @parser_block_stack.pop + end end def parser_reduce(rule, state) diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 0047e20..e6ccb22 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -517,6 +517,12 @@ def literal(value) Lrama::Ruby::Languages::Ruby.parse(lexer.each, lexical_context: lexer.context) assert_true(lexer.context.parser_events.any? { |event| event.first == :shift }) assert_true(lexer.context.parser_events.any? { |event| event.first == :reduce }) + assert_empty(lexer.context.parser_delimiter_stack) + assert_empty(lexer.context.parser_cmdarg_stack) + assert_empty(lexer.context.parser_block_stack) + block_lexer = Arpaka.const_get(:Lexer, false).new("f do; 1; end") + Lrama::Ruby::Languages::Ruby.parse(block_lexer.each, lexical_context: block_lexer.context) + assert_empty(block_lexer.context.parser_block_stack) pretokenized = Arpaka.const_get(:Lexer, false).new("f(1)") tokens = pretokenized.each.to_a Lrama::Ruby::Languages::Ruby.parse(tokens) From 8dfe7c39309f5c62845f51c8eec4f3181ab58190 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sun, 13 Sep 2026 10:24:21 +0900 Subject: [PATCH 122/157] Track parser conditional stack --- lib/lrama/ruby/languages/ruby/lexer.rb | 11 ++++++++++- test/lrama/ruby_language_test.rb | 3 +++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index fcf53ab..863ec1a 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -13,7 +13,8 @@ class LexicalContext :lex_state, :command_start, :label_pending attr_reader :delimiter_stack, :cmdarg_stack, :condition_stack, :token_history, :state_history, :block_stack, :scope_stack, :parser_events, - :parser_delimiter_stack, :parser_cmdarg_stack, :parser_block_stack + :parser_delimiter_stack, :parser_cmdarg_stack, :parser_block_stack, + :parser_condition_stack def initialize @begin_expression = true @@ -37,6 +38,7 @@ def initialize @parser_delimiter_stack = [] @parser_cmdarg_stack = [] @parser_block_stack = [] + @parser_condition_stack = [] end def delimiter_depth @@ -137,6 +139,13 @@ def parser_shift(token, value, state, action) elsif token == :keyword_end && @parser_block_stack.last @parser_block_stack.pop end + if [:keyword_while, :keyword_until, :keyword_for].include?(token) + @parser_condition_stack << token + elsif token == :keyword_do_cond && @parser_condition_stack.last + @parser_condition_stack.pop + elsif token == :keyword_end && @parser_condition_stack.last + @parser_condition_stack.pop + end end def parser_reduce(rule, state) diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index e6ccb22..e109e4c 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -523,6 +523,9 @@ def literal(value) block_lexer = Arpaka.const_get(:Lexer, false).new("f do; 1; end") Lrama::Ruby::Languages::Ruby.parse(block_lexer.each, lexical_context: block_lexer.context) assert_empty(block_lexer.context.parser_block_stack) + condition_lexer = Arpaka.const_get(:Lexer, false).new("while condition do; body; end") + Lrama::Ruby::Languages::Ruby.parse(condition_lexer.each, lexical_context: condition_lexer.context) + assert_empty(condition_lexer.context.parser_condition_stack) pretokenized = Arpaka.const_get(:Lexer, false).new("f(1)") tokens = pretokenized.each.to_a Lrama::Ruby::Languages::Ruby.parse(tokens) From 9b05cf895d2525a4818df1603c24760e995177b0 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sun, 13 Sep 2026 10:25:21 +0900 Subject: [PATCH 123/157] Track parser lexical scopes --- lib/lrama/ruby/languages/ruby/lexer.rb | 14 +++++++++++++- test/lrama/ruby_language_test.rb | 3 +++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 863ec1a..33e71fe 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -14,7 +14,7 @@ class LexicalContext attr_reader :delimiter_stack, :cmdarg_stack, :condition_stack, :token_history, :state_history, :block_stack, :scope_stack, :parser_events, :parser_delimiter_stack, :parser_cmdarg_stack, :parser_block_stack, - :parser_condition_stack + :parser_condition_stack, :parser_scope_stack def initialize @begin_expression = true @@ -39,6 +39,7 @@ def initialize @parser_cmdarg_stack = [] @parser_block_stack = [] @parser_condition_stack = [] + @parser_scope_stack = [] end def delimiter_depth @@ -146,6 +147,17 @@ def parser_shift(token, value, state, action) elsif token == :keyword_end && @parser_condition_stack.last @parser_condition_stack.pop end + if token == :keyword_def + @parser_scope_stack << :method + elsif [:keyword_class, :keyword_module].include?(token) + @parser_scope_stack << token + elsif token == :tLAMBDA + @parser_scope_stack << :lambda + elsif token == "}" && @parser_scope_stack.last == :lambda + @parser_scope_stack.pop + elsif token == :keyword_end && @parser_scope_stack.last + @parser_scope_stack.pop + end end def parser_reduce(rule, state) diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index e109e4c..5667386 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -526,6 +526,9 @@ def literal(value) condition_lexer = Arpaka.const_get(:Lexer, false).new("while condition do; body; end") Lrama::Ruby::Languages::Ruby.parse(condition_lexer.each, lexical_context: condition_lexer.context) assert_empty(condition_lexer.context.parser_condition_stack) + scope_lexer = Arpaka.const_get(:Lexer, false).new("def f; -> { 1 }; end") + Lrama::Ruby::Languages::Ruby.parse(scope_lexer.each, lexical_context: scope_lexer.context) + assert_empty(scope_lexer.context.parser_scope_stack) pretokenized = Arpaka.const_get(:Lexer, false).new("f(1)") tokens = pretokenized.each.to_a Lrama::Ruby::Languages::Ruby.parse(tokens) From 04ee7d7ed0d1623295e1cfac2b18695c22aec4f7 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sun, 13 Sep 2026 10:29:37 +0900 Subject: [PATCH 124/157] Classify call braces inside arguments --- lib/lrama/ruby/languages/ruby/lexer.rb | 2 +- test/lrama/ruby_language_test.rb | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 33e71fe..56691a0 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -530,7 +530,7 @@ def no_argument_block? def no_argument_brace_block? return false if [".", :tCOLON2].include?(@previous_previous) - return false if [:tLABEL, :tLSHFT, :tOROP, :tANDOP, :tASSOC, "=", ","].include?(@previous_previous) + return false if [:tLABEL, :tLSHFT, :tOROP, :tANDOP, :tASSOC, "=", "(", ","].include?(@previous_previous) no_argument_block? end diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 5667386..b582956 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -687,6 +687,12 @@ def literal(value) assert_nothing_raised { Arpaka.parse_source("value = call { 1 }.dup\n") } end + test "source lexer uses regular brace blocks inside call arguments" do + tokens = Arpaka.const_get(:Lexer, false).new("outer(call { 1 })\n").each.to_a + assert_equal("{", tokens.map(&:first)[3]) + assert_nothing_raised { Arpaka.parse_source("outer(call { 1 })\n") } + end + test "source lexer preserves newlines inside command blocks" do assert_nothing_raised do Arpaka.parse_source("app = lambda { |env|\n req = Request.new(env)\n res = response(req)\n}\n") From 877ae99d3411a8663fcb4a1e35b768300b01747f Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sun, 13 Sep 2026 10:32:12 +0900 Subject: [PATCH 125/157] Classify method body array literals --- lib/lrama/ruby/languages/ruby/lexer.rb | 2 ++ test/lrama/ruby_language_test.rb | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 56691a0..ba6e5ee 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -1055,6 +1055,8 @@ def operator_or_punctuation(start) array_argument = value == "[" && start.positive? && [9, 10, 11, 12, 13, 32].include?(@source.getbyte(start - 1)) && [:tIDENTIFIER, :tFID, :tCONSTANT].include?(@previous) + array_argument ||= value == "[" && @previous == ")" && + @previous_previous == "(" && @context.scope_stack.last == :method return [value == "[" && (@context.begin_expression || array_argument) ? :tLBRACK : (value == "[" ? "[" : (brace_block ? "{" : :tLBRACE)), nil] elsif value == ")" || value == "]" || value == "}" opener = { ")" => "(", "]" => "[", "}" => "{" }.fetch(value) diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index b582956..5ca2e81 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -693,6 +693,12 @@ def literal(value) assert_nothing_raised { Arpaka.parse_source("outer(call { 1 })\n") } end + test "source lexer starts one-line method bodies with array literals" do + tokens = Arpaka.const_get(:Lexer, false).new("def values() [1] end\n").each.to_a + assert_equal(:tLBRACK, tokens.map(&:first)[4]) + assert_nothing_raised { Arpaka.parse_source("def values() [1] end\n") } + end + test "source lexer preserves newlines inside command blocks" do assert_nothing_raised do Arpaka.parse_source("app = lambda { |env|\n req = Request.new(env)\n res = response(req)\n}\n") From 4692ce0428bcf778c1e5e8cf73b06a10c80571b1 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sun, 13 Sep 2026 10:36:10 +0900 Subject: [PATCH 126/157] Classify ternary branch brace blocks --- lib/lrama/ruby/languages/ruby/lexer.rb | 2 +- test/lrama/ruby_language_test.rb | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index ba6e5ee..81fd079 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -530,7 +530,7 @@ def no_argument_block? def no_argument_brace_block? return false if [".", :tCOLON2].include?(@previous_previous) - return false if [:tLABEL, :tLSHFT, :tOROP, :tANDOP, :tASSOC, "=", "(", ","].include?(@previous_previous) + return false if [:tLABEL, :tLSHFT, :tOROP, :tANDOP, :tASSOC, "=", "(", ",", ":", "?"].include?(@previous_previous) no_argument_block? end diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 5ca2e81..e60230b 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -699,6 +699,12 @@ def literal(value) assert_nothing_raised { Arpaka.parse_source("def values() [1] end\n") } end + test "source lexer uses regular brace blocks after ternary branches" do + tokens = Arpaka.const_get(:Lexer, false).new("touch ? callback { 1 } : fallback\n").each.to_a + assert_equal("{", tokens.map(&:first)[3]) + assert_nothing_raised { Arpaka.parse_source("touch ? callback { 1 } : fallback\n") } + end + test "source lexer preserves newlines inside command blocks" do assert_nothing_raised do Arpaka.parse_source("app = lambda { |env|\n req = Request.new(env)\n res = response(req)\n}\n") From 228a854ea0fbf7c960ad563d391dbb49419c0280 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sun, 13 Sep 2026 10:38:00 +0900 Subject: [PATCH 127/157] Keep load path global intact --- lib/lrama/ruby/languages/ruby/lexer.rb | 2 +- test/lrama/ruby_language_test.rb | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 81fd079..591fb60 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -578,7 +578,7 @@ def variable_token(start) marker = byte advance advance if marker == 64 && byte == 64 - if marker == 36 && [33, 38, 39, 43, 60, 62, 61, 63, 96, 126].include?(byte) + if marker == 36 && [33, 38, 39, 43, 58, 60, 62, 61, 63, 96, 126].include?(byte) advance text = @source.byteslice(start, @index - start) return [:tGVAR, text.to_sym] diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index e60230b..ed41ce2 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -620,6 +620,13 @@ def literal(value) assert_equal(:"$$", tokens[3][1]) end + test "source lexer keeps the load path global intact" do + tokens = Arpaka.const_get(:Lexer, false).new("$:.unshift(\"lib\")\n").each.to_a + assert_equal(:tGVAR, tokens.first.first) + assert_equal(:"$:", tokens.first.last) + assert_nothing_raised { Arpaka.parse_source("$:.unshift(\"lib\")\n") } + end + test "source lexer preserves newlines before terminators" do assert_nothing_raised { Arpaka.parse_source("def empty\nend\n") } assert_nothing_raised { Arpaka.parse_source("if condition\nend\n") } From 72ff469db62f5bd32ff7cb2c82db6dcf65f07694 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sun, 13 Sep 2026 10:44:37 +0900 Subject: [PATCH 128/157] Classify keyword-shaped method names in definitions --- lib/lrama/ruby/languages/ruby/lexer.rb | 2 ++ test/lrama/ruby_language_test.rb | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 591fb60..c1d07e4 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -479,6 +479,8 @@ def identifier_token(start) token = KEYWORDS[word] if token && [".", :tCOLON2, :tANDDOT].include?(@previous) token = :tFID + elsif token && @previous == :keyword_def && word != "self" + token = :tFID elsif (!@context.begin_expression || [:keyword_return, :keyword_break, :keyword_next, :keyword_end, :keyword_yield, :keyword_super].include?(@previous)) && { "if" => :modifier_if, "unless" => :modifier_unless, "while" => :modifier_while, "until" => :modifier_until, "rescue" => :modifier_rescue }.key?(word) diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index ed41ce2..c6d8e34 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -627,6 +627,12 @@ def literal(value) assert_nothing_raised { Arpaka.parse_source("$:.unshift(\"lib\")\n") } end + test "source lexer treats keyword-shaped method names as fname tokens" do + tokens = Arpaka.const_get(:Lexer, false).new("def then(&block)\nend\n").each.to_a + assert_equal(:tFID, tokens[1].first) + assert_nothing_raised { Arpaka.parse_source("def then(&block)\nend\n") } + end + test "source lexer preserves newlines before terminators" do assert_nothing_raised { Arpaka.parse_source("def empty\nend\n") } assert_nothing_raised { Arpaka.parse_source("if condition\nend\n") } From 0d6242b629fca7c46c961787fb4a9402e031bb15 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sun, 13 Sep 2026 11:12:44 +0900 Subject: [PATCH 129/157] Handle multiple heredocs on one line --- lib/lrama/ruby/languages/ruby/lexer.rb | 65 +++++++++++++++++++++----- test/lrama/ruby_language_test.rb | 5 ++ 2 files changed, 59 insertions(+), 11 deletions(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index c1d07e4..c6c3835 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -220,12 +220,15 @@ def initialize(source, filename: "(ruby)") @previous_value = nil @context = LexicalContext.new @pending = [] + @heredoc_queue = [] + @deferred_suffix = nil + @heredoc_newline_pending = false @class_superclass = false end def each return enum_for(__method__) unless block_given? - until eof? && @pending.empty? + until eof? && @pending.empty? && @heredoc_queue.empty? && @deferred_suffix.nil? token = next_token yield token if token end @@ -260,6 +263,19 @@ def fail!(message, index = @index, line = @line, column = @column) end def next_token + if @pending.empty? && @heredoc_queue.any? + load_queued_heredoc + elsif @pending.empty? && @deferred_suffix + suffix = @deferred_suffix + @deferred_suffix = nil + @pending.concat(self.class.new(suffix, filename: @filename).each.to_a) + @pending.pop if @pending.last == [0, nil] + @pending << ["\n", nil] + @heredoc_newline_pending = false + elsif @pending.empty? && @heredoc_newline_pending + @pending << ["\n", nil] + @heredoc_newline_pending = false + end unless @pending.empty? value = @pending.shift return next_token if value[0] == "\n" && newline_ignored? @@ -958,8 +974,24 @@ def heredoc_token(start) advance while byte && byte != 10 suffix = @source.byteslice(suffix_start, @index - suffix_start).force_encoding(Encoding::UTF_8) advance if byte == 10 - body = +"" interpolate = quote != 39 + body = read_heredoc_body(delimiter, indent, squiggly, start) + content_tokens = interpolate ? interpolated_content_tokens(body, start) : [[:tSTRING_CONTENT, body]] + @pending = content_tokens + [[:tSTRING_END, nil]] + prefix, headers, suffix = extract_heredoc_headers(suffix) + unless prefix.empty? + prefix_tokens = self.class.new(prefix, filename: @filename).each.to_a + prefix_tokens.pop if prefix_tokens.last == [0, nil] + @pending.concat(prefix_tokens) + end + @heredoc_queue.concat(headers) + @deferred_suffix = suffix unless suffix.empty? + @heredoc_newline_pending = true + [quote == 96 ? :tXSTRING_BEG : :tSTRING_BEG, nil] + end + + def read_heredoc_body(delimiter, indent, squiggly, start) + body = +"" loop do line_start = @index advance while !eof? && byte != 10 @@ -974,16 +1006,27 @@ def heredoc_token(start) advance if byte == 10 fail!("unterminated heredoc", start) if eof? end - body = dedent_heredoc(body) if squiggly - content_tokens = interpolate ? interpolated_content_tokens(body, start) : [[:tSTRING_CONTENT, body]] - @pending = content_tokens + [[:tSTRING_END, nil]] - unless suffix.empty? - suffix_tokens = self.class.new(suffix, filename: @filename).each.to_a - suffix_tokens.pop if suffix_tokens.last == [0, nil] - @pending.concat(suffix_tokens) + squiggly ? dedent_heredoc(body) : body + end + + def extract_heredoc_headers(suffix) + pattern = /<<(\-|~)?([A-Za-z_][A-Za-z0-9_]*)/ + match = pattern.match(suffix) + return [suffix, [], ""] unless match + + headers = [{delimiter: match[2], indent: !match[1].nil?, squiggly: match[1] == "~", quote: nil}] + trailing = suffix[match.end(0)..].to_s.gsub(pattern) do + headers << {delimiter: Regexp.last_match(2), indent: !Regexp.last_match(1).nil?, squiggly: Regexp.last_match(1) == "~", quote: nil} + "" end - @pending << ["\n", nil] - [quote == 96 ? :tXSTRING_BEG : :tSTRING_BEG, nil] + [suffix[0...match.begin(0)], headers, trailing] + end + + def load_queued_heredoc + descriptor = @heredoc_queue.shift + body = read_heredoc_body(descriptor.fetch(:delimiter), descriptor.fetch(:indent), descriptor.fetch(:squiggly), @index) + content = descriptor.fetch(:quote) == 39 ? [[:tSTRING_CONTENT, body]] : interpolated_content_tokens(body, @index) + @pending.concat([descriptor.fetch(:quote) == 96 ? [:tXSTRING_BEG, nil] : [:tSTRING_BEG, nil], *content, [:tSTRING_END, nil]]) end def dedent_heredoc(body) diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index c6d8e34..7579a53 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -438,6 +438,11 @@ def literal(value) end end + test "source lexer handles multiple heredocs on one line" do + source = "value = < Date: Sun, 13 Sep 2026 11:16:24 +0900 Subject: [PATCH 130/157] Recognize ampersand operator method names --- lib/lrama/ruby/languages/ruby/lexer.rb | 6 ++++++ test/lrama/ruby_language_test.rb | 1 + 2 files changed, 7 insertions(+) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index c6c3835..21f6fc7 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -1041,6 +1041,10 @@ def dedent_heredoc(body) end def operator_or_punctuation(start) + if @previous == :keyword_def && @source.getbyte(@index) == 38 + advance + return ["&", nil] + end if @previous == :tSYMBEG && ["-@", "+@"].include?(@source.byteslice(@index, 2)) value = @source.byteslice(@index, 2) advance(2) @@ -1063,6 +1067,8 @@ def operator_or_punctuation(start) :tBDOT3 elsif text == "::" && (@context.begin_expression || (start.positive? && [9, 32].include?(@source.getbyte(start - 1)))) :tCOLON3 + elsif text == "&" && @previous == :keyword_def + "&" else OP_TOKENS.fetch(text, :tOP_ASGN) end diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 7579a53..9f32059 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -781,6 +781,7 @@ def literal(value) assert_equal(AST::Def.new(:[], :x, [AST::BareCall.new(:x)]), Arpaka.parse_source("def [](x); x; end").statements.first) assert_nothing_raised { Arpaka.parse_source("def /(other); other; end") } + assert_nothing_raised { Arpaka.parse_source("def &(other); other; end") } assert_equal(:Foo, Arpaka.parse_source("::Foo").statements.first) end From efee5059c941088da89ef54c2f1c5fd532021949 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sun, 13 Sep 2026 11:19:49 +0900 Subject: [PATCH 131/157] Avoid synthetic ampersands after block arguments --- lib/lrama/ruby/languages/ruby/lexer.rb | 2 +- test/lrama/ruby_language_test.rb | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 21f6fc7..740673e 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -548,7 +548,7 @@ def no_argument_block? def no_argument_brace_block? return false if [".", :tCOLON2].include?(@previous_previous) - return false if [:tLABEL, :tLSHFT, :tOROP, :tANDOP, :tASSOC, "=", "(", ",", ":", "?"].include?(@previous_previous) + return false if [:tLABEL, :tLSHFT, :tOROP, :tANDOP, :tASSOC, :tAMPER, "=", "(", ",", ":", "?"].include?(@previous_previous) no_argument_block? end diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 9f32059..58daa50 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -734,6 +734,7 @@ def literal(value) assert_includes(tokens.map(&:first), :tLAMBEG) assert_nothing_raised { Arpaka.parse_source("->(value) { value }\n") } assert_nothing_raised { Arpaka.parse_source("-> {\n first\n second\n}\n") } + assert_nothing_raised { Arpaka.parse_source("f(&lambda { |value| value })\n") } end test "source lexer ignores newlines after logical operators" do From a15943def14c16ad4715db1ba00fb83a1d927986 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sun, 13 Sep 2026 11:22:23 +0900 Subject: [PATCH 132/157] Preserve newlines after nested blocks --- lib/lrama/ruby/languages/ruby/lexer.rb | 2 +- test/lrama/ruby_language_test.rb | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 740673e..c56cc5a 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -362,7 +362,7 @@ def skip_space_and_comments end def newline_ignored? - ignored = @context.delimiter_depth.positive? || @previous == "\n" || + ignored = (@context.delimiter_depth.positive? && @previous != "}") || @previous == "\n" || ["+", "-", "*", "/", "%", "=", "?", ":", ",", ".", "&", "|", "^", "<<", ">>", "&&", "||", "=>", :keyword_and, :keyword_or, :keyword_not, :tAMPER, :tPIPE, :tSTAR, :tDSTAR, :tDOT2, :tDOT3, :tPOW, :tCMP, :tEQ, :tEQQ, :tNEQ, :tGEQ, :tLEQ, :tANDOP, :tOROP, :tMATCH, :tNMATCH, :tLSHFT, :tRSHFT, :tASSOC, :tLAMBDA, :tCOLON2, :tANDDOT].include?(@previous) || @previous == :tLABEL || [:modifier_if, :modifier_unless, :modifier_while, :modifier_until].include?(@previous) || diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 58daa50..17cf461 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -644,6 +644,11 @@ def literal(value) assert_nothing_raised { Arpaka.parse_source("foo do\nend\n") } end + test "source lexer preserves block newlines inside calls" do + source = "Module.new {\n define_method(:up) { yield(:up); super() }\n define_method(:down) { yield(:down); super() }\n}\n" + assert_nothing_raised { Arpaka.parse_source(source) } + end + test "source lexer ignores newlines after operator assignments" do assert_nothing_raised { Arpaka.parse_source("value ||=\n if condition\n fallback\n end\n") } end From e5b0c682850ee82fc41174dc09acff3d725a9346 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sun, 13 Sep 2026 11:28:49 +0900 Subject: [PATCH 133/157] Preserve block terminator newlines in calls --- lib/lrama/ruby/languages/ruby/lexer.rb | 3 ++- test/lrama/ruby_language_test.rb | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index c56cc5a..81f1e55 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -362,7 +362,8 @@ def skip_space_and_comments end def newline_ignored? - ignored = (@context.delimiter_depth.positive? && @previous != "}") || @previous == "\n" || + ignored = (@context.delimiter_depth.positive? && @previous != "}" && + !(@previous == :keyword_end && @context.block_depth.positive?)) || @previous == "\n" || ["+", "-", "*", "/", "%", "=", "?", ":", ",", ".", "&", "|", "^", "<<", ">>", "&&", "||", "=>", :keyword_and, :keyword_or, :keyword_not, :tAMPER, :tPIPE, :tSTAR, :tDSTAR, :tDOT2, :tDOT3, :tPOW, :tCMP, :tEQ, :tEQQ, :tNEQ, :tGEQ, :tLEQ, :tANDOP, :tOROP, :tMATCH, :tNMATCH, :tLSHFT, :tRSHFT, :tASSOC, :tLAMBDA, :tCOLON2, :tANDDOT].include?(@previous) || @previous == :tLABEL || [:modifier_if, :modifier_unless, :modifier_while, :modifier_until].include?(@previous) || diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 17cf461..5ac3ac6 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -647,6 +647,7 @@ def literal(value) test "source lexer preserves block newlines inside calls" do source = "Module.new {\n define_method(:up) { yield(:up); super() }\n define_method(:down) { yield(:down); super() }\n}\n" assert_nothing_raised { Arpaka.parse_source(source) } + assert_nothing_raised { Arpaka.parse_source("Module.new do\n define_method(:up) do\n 1\n end\n define_method(:down) do\n 2\n end\nend\n") } end test "source lexer ignores newlines after operator assignments" do From 90f3fef70ba19c13c94f9c4d54dcba0132051013 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sun, 13 Sep 2026 11:32:59 +0900 Subject: [PATCH 134/157] Preserve newlines after operator symbols --- lib/lrama/ruby/languages/ruby/lexer.rb | 2 ++ test/lrama/ruby_language_test.rb | 1 + 2 files changed, 3 insertions(+) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 81f1e55..eb20559 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -369,6 +369,8 @@ def newline_ignored? [:modifier_if, :modifier_unless, :modifier_while, :modifier_until].include?(@previous) || [:tANDOP, :tOROP, :tMATCH, :tNMATCH, :tASSOC, :tOP_ASGN].include?(@previous) || next_non_space_byte == 46 + ignored = false if @previous_previous == :tSYMBEG && + [:tLSHFT, :tRSHFT, :tSTAR, :tDSTAR, :tPOW].include?(@previous) @context.condition_line = false if ignored && @context.condition_line ignored end diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 5ac3ac6..205f1c9 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -782,6 +782,7 @@ def literal(value) test "source lexer recognizes unary operator symbols" do assert_nothing_raised { Arpaka.parse_source("alias :-@ :deduplicate\n") } + assert_nothing_raised { Arpaka.parse_source("alias_method :push, :<<\nalias_method :append, :<<\n") } end test "source lexer recognizes operator method names and top-level constants" do From bc794f0bd9dbac8d4c8fc35961e80c1f1fd64d8a Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sun, 13 Sep 2026 12:05:40 +0900 Subject: [PATCH 135/157] Fix contextual Ruby lexer tokens --- lib/lrama/ruby/languages/ruby/lexer.rb | 33 ++++++++++++++++++++------ test/lrama/ruby_language_test.rb | 22 +++++++++++++++++ 2 files changed, 48 insertions(+), 7 deletions(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index eb20559..669bfc2 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -414,6 +414,7 @@ def update_argument_label(token, value) def expression_begin_after(token) return true if token.nil? || token == "\n" return false if token == ")" || token == "]" || token == "}" + return false if token == :keyword_end return false if token == :tINTEGER || token == :tFLOAT || token == :tRATIONAL || token == :tIMAGINARY return false if token == :tIDENTIFIER || token == :tCONSTANT || token == :tFID || token == :tSTRING_END || token == :tREGEXP_END return false if token == :keyword_true || token == :keyword_false || token == :keyword_nil || token == :keyword_self @@ -430,7 +431,7 @@ def lexical_state_after(token) @context.label_pending = false return :expr_labeled end - return :expr_end if [")", "]", "}", :tSTRING_END, :tREGEXP_END, + return :expr_end if [")", "]", "}", :keyword_end, :tSTRING_END, :tREGEXP_END, :tINTEGER, :tFLOAT, :tRATIONAL, :tIMAGINARY, :tIDENTIFIER, :tCONSTANT, :tFID, :tIVAR, :tGVAR, :tCVAR, :tNTH_REF, :keyword_true, :keyword_false, :keyword_nil, :keyword_self, @@ -475,6 +476,10 @@ def identifier_token(start) advance word = "defined?" end + if byte == 63 && byte(1) == 58 + advance(2) + return [:tLABEL, (word + "?").to_sym] + end if [33, 63].include?(byte) && !identifier_byte?(byte(1)) && byte(1) != 61 suffix = byte.chr advance @@ -492,6 +497,9 @@ def identifier_token(start) advance return [:tLABEL, word.to_sym] end + if word == "do" && @context.lambda_pending + return [do_token, word.to_sym] + end if word == "do" && !@context.condition_do && no_argument_block? return [:keyword_do, word.to_sym] end @@ -530,7 +538,7 @@ def do_token elsif @context.lambda_pending @context.lambda_pending = false :keyword_do_LAMBDA - elsif @previous == ")" + elsif [")", :keyword_super, :keyword_yield].include?(@previous) :keyword_do elsif @previous == :tLAMBDA :keyword_do_LAMBDA @@ -619,7 +627,11 @@ def variable_token(start) end def character_or_question(start) - return operator_or_punctuation(start) unless @context.begin_expression && byte(1) && byte(1) != 32 && byte(1) != 10 + command_character = !@context.begin_expression && + [:tIDENTIFIER, :tCONSTANT, :tFID].include?(@previous) && + start.positive? && [9, 32].include?(@source.getbyte(start - 1)) + return operator_or_punctuation(start) unless (@context.begin_expression || command_character) && + byte(1) && byte(1) != 32 && byte(1) != 10 advance value = if byte == 92 escape_sequence(start) @@ -1054,9 +1066,11 @@ def operator_or_punctuation(start) return [value == "-@" ? :tUMINUS : :tUPLUS, nil] end text = OPERATORS.sort_by { |operator| -operator.bytesize }.find do |operator| - next false if operator == "[]" && start.positive? && + next false if ["[]", "[]="].include?(operator) && start.positive? && [9, 10, 11, 12, 13, 32].include?(@source.getbyte(start - 1)) && @previous != :keyword_def + next false if ["[]", "[]="].include?(operator) && + [:tIDENTIFIER, :tCONSTANT, :tFID, :tINTEGER, :tSTRING_END, :tREGEXP_END].include?(@previous) @source.byteslice(@index, operator.bytesize) == operator end operator_method = [:keyword_def, ".", :tCOLON2, :tANDDOT, :tSYMBEG].include?(@previous) @@ -1086,9 +1100,12 @@ def operator_or_punctuation(start) end advance if value == "(" || value == "[" || value == "{" - brace_block = value == "{" && ([:tIDENTIFIER, :tCONSTANT, :tFID].include?(@previous) || + brace_block = value == "{" && @previous != "(" && + ([:tIDENTIFIER, :tCONSTANT, :tFID].include?(@previous) || @previous == ")" || - [".", :tCOLON2].include?(@previous_previous) || [:proc, :lambda].include?(@previous_value)) + [".", :tCOLON2].include?(@previous_previous) || + [:keyword_super, :keyword_yield].include?(@previous) || + [:proc, :lambda].include?(@previous_value)) lambda_block = value == "{" && @context.lambda_pending command_arg = value == "(" && !@context.begin_expression && start.positive? && [9, 32].include?(@source.getbyte(start - 1)) && [:tIDENTIFIER, :tCONSTANT, :tFID].include?(@previous) @@ -1100,7 +1117,9 @@ def operator_or_punctuation(start) if command_arg return [:tLPAREN_ARG, nil] end - return [@context.begin_expression && ![".", :tCOLON2, :tANDDOT, :keyword_super, :keyword_yield, :tLAMBDA, :tAREF].include?(@previous) ? :tLPAREN : "(", nil] + method_definition = @previous_previous == :keyword_def + return [@context.begin_expression && !method_definition && + ![".", :tCOLON2, :tANDDOT, :keyword_super, :keyword_yield, :tLAMBDA, :tAREF].include?(@previous) ? :tLPAREN : "(", nil] end if value == "{" && @context.lambda_pending @context.lambda_pending = false diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 205f1c9..2429e1f 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -583,6 +583,28 @@ def literal(value) assert_nothing_raised { Arpaka.parse_source("value.match?(pattern)") } end + test "source lexer handles contextual punctuation in Ruby expressions" do + tokens = Arpaka.const_get(:Lexer, false).new("f(foo?: true)").each.to_a + assert_equal([:tIDENTIFIER, "(", :tLABEL, :keyword_true, ")", 0], tokens.map(&:first)) + assert_nothing_raised { Arpaka.parse_source("f(foo?: true)") } + + tokens = Arpaka.const_get(:Lexer, false).new("list_tables[]").each.to_a + assert_equal([:tIDENTIFIER, "[", "]", 0], tokens.map(&:first)) + assert_kind_of(AST::Index, Arpaka.parse_source("list_tables[]").statements.first) + + assert_nothing_raised { Arpaka.parse_source("x = if y; 'a'; else; 'b'; end + z") } + assert_nothing_raised { Arpaka.parse_source("helper.({a: 1})") } + assert_nothing_raised { Arpaka.parse_source("assert ?h.in?(\"hello\")") } + end + + test "source lexer keeps definition and block contexts for operators" do + assert_nothing_raised { Arpaka.parse_source("def ==(x); self.x == x; end") } + assert_nothing_raised { Arpaka.parse_source("def []=(x); x; end") } + assert_nothing_raised { Arpaka.parse_source("x = -> *args do; 1; end") } + assert_nothing_raised { Arpaka.parse_source("def f; super do |x| x end; end") } + assert_nothing_raised { Arpaka.parse_source("def f; super { |x| x }; end") } + end + test "source lexer distinguishes splat and block argument operators" do assert_equal(AST::Call.new(:f, [AST::BareCall.new(:args)]), Arpaka.parse_source("f(*args)").statements.first) From af7a709e14d5e85d0bb03d43d87099bf2b9c233f Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sun, 13 Sep 2026 12:08:36 +0900 Subject: [PATCH 136/157] Restore alias and symbol lexer boundaries --- lib/lrama/ruby/languages/ruby/lexer.rb | 8 +++++++- test/lrama/ruby_language_test.rb | 13 +++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 669bfc2..4d0d66f 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -370,7 +370,9 @@ def newline_ignored? [:tANDOP, :tOROP, :tMATCH, :tNMATCH, :tASSOC, :tOP_ASGN].include?(@previous) || next_non_space_byte == 46 ignored = false if @previous_previous == :tSYMBEG && - [:tLSHFT, :tRSHFT, :tSTAR, :tDSTAR, :tPOW].include?(@previous) + [:tLSHFT, :tRSHFT, :tSTAR, :tDSTAR, :tPOW, :tEQ, :tEQQ, :tNEQ, + :tCMP, :tGEQ, :tLEQ, :tANDOP, :tOROP, :tMATCH, :tNMATCH, + :tASSOC, :tOP_ASGN].include?(@previous) @context.condition_line = false if ignored && @context.condition_line ignored end @@ -489,6 +491,10 @@ def identifier_token(start) advance return [:tFID, (word + "=").to_sym] end + if byte == 61 && @context.alias_context + advance + return [:tFID, (word + "=").to_sym] + end if byte == 61 && @previous == :tSYMBEG advance return [:tIDENTIFIER, (word + "=").to_sym] diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 2429e1f..404e13b 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -815,6 +815,19 @@ def literal(value) assert_equal(:Foo, Arpaka.parse_source("::Foo").statements.first) end + test "source lexer preserves alias boundaries and symbol statement endings" do + tokens = Arpaka.const_get(:Lexer, false).new("alias foo= bar").each.to_a + assert_equal([:keyword_alias, :tFID, :tIDENTIFIER, 0], tokens.map(&:first)) + assert_nothing_raised { Arpaka.parse_source("alias foo= bar") } + + source = "alias_method :eql?, :==\ndef hash; 1; end" + tokens = Arpaka.const_get(:Lexer, false).new(source).each.to_a + assert_include(tokens.map(&:first), "\n") + assert_nothing_raised { Arpaka.parse_source(source) } + + assert_nothing_raised { Arpaka.parse_source("undef_method :==, :!, :!=\ndef hash; 1; end") } + end + test "source lexer accepts symbols after predicate-like identifiers" do assert_nothing_raised { Arpaka.parse_source("alias :merge! :update\n") } assert_nothing_raised { Arpaka.parse_source("alias :default_options= :default\n") } From 64307b52853035ad2f6ae404c745a6ff796fa3a8 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sun, 13 Sep 2026 12:11:01 +0900 Subject: [PATCH 137/157] Handle keyword-shaped receiver setters --- lib/lrama/ruby/languages/ruby/lexer.rb | 2 +- test/lrama/ruby_language_test.rb | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 4d0d66f..d11c528 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -511,7 +511,7 @@ def identifier_token(start) end token = KEYWORDS[word] if token && [".", :tCOLON2, :tANDDOT].include?(@previous) - token = :tFID + token = next_non_space_byte == 61 ? :tIDENTIFIER : :tFID elsif token && @previous == :keyword_def && word != "self" token = :tFID elsif (!@context.begin_expression || [:keyword_return, :keyword_break, :keyword_next, :keyword_end, :keyword_yield, :keyword_super].include?(@previous)) && { "if" => :modifier_if, "unless" => :modifier_unless, diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 404e13b..12b260f 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -828,6 +828,12 @@ def literal(value) assert_nothing_raised { Arpaka.parse_source("undef_method :==, :!, :!=\ndef hash; 1; end") } end + test "source lexer allows keyword-shaped receiver setters" do + tokens = Arpaka.const_get(:Lexer, false).new("node.case = value").each.to_a + assert_equal([:tIDENTIFIER, ".", :tIDENTIFIER, "=", :tIDENTIFIER, 0], tokens.map(&:first)) + assert_nothing_raised { Arpaka.parse_source("node.case = value") } + end + test "source lexer accepts symbols after predicate-like identifiers" do assert_nothing_raised { Arpaka.parse_source("alias :merge! :update\n") } assert_nothing_raised { Arpaka.parse_source("alias :default_options= :default\n") } From a9becbddba4d7a15122cf0361e274fa79c4a0351 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sun, 13 Sep 2026 12:14:36 +0900 Subject: [PATCH 138/157] Classify lambda command blocks correctly --- lib/lrama/ruby/languages/ruby/lexer.rb | 1 + test/lrama/ruby_language_test.rb | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index d11c528..af162fb 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -559,6 +559,7 @@ def no_argument_block? return false if @context.token_history == ["+", :tINTEGER, ".", :tIDENTIFIER] return false if [:tIDENTIFIER, :tCONSTANT, :tFID].include?(@previous_previous) return false if [:tSYMBEG, :tLABEL, :tCOLON2].include?(@previous_previous) + return true if @previous_value == :lambda && @previous_previous == :tLSHFT return false if [:tLSHFT, :tLAMBDA].include?(@previous_previous) true end diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 12b260f..56cc8d4 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -625,7 +625,8 @@ def literal(value) Arpaka.parse_source("foo.bar do\n 1\nend").statements.first) assert_nothing_raised { Arpaka.parse_source("foo before: :bar do |value| value end\n") } tokens = Arpaka.const_get(:Lexer, false).new("items << lambda do").each.to_a - assert_equal([:tIDENTIFIER, :tLSHFT, :tIDENTIFIER, :keyword_do_block], tokens[0, 4].map(&:first)) + assert_equal([:tIDENTIFIER, :tLSHFT, :tIDENTIFIER, :keyword_do], tokens[0, 4].map(&:first)) + assert_nothing_raised { Arpaka.parse_source("items << lambda do\n work\nend\n") } end test "source lexer recognizes shorthand percent literals in command arguments" do From b64fd697d1f3a57dd1570e4afd2e9460bb0c6220 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sun, 13 Sep 2026 12:42:02 +0900 Subject: [PATCH 139/157] Remove synthetic block argument tokens --- lib/lrama/ruby/languages/ruby/lexer.rb | 5 ----- test/lrama/ruby_language_test.rb | 14 +++++++++++++- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index af162fb..91416fb 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -1100,11 +1100,6 @@ def operator_or_punctuation(start) return [token, nil] end value = byte.chr - if value == "{" && no_argument_brace_block? - advance - @pending.unshift([:tLBRACE_ARG, nil]) - return [:tAMPER, nil] - end advance if value == "(" || value == "[" || value == "{" brace_block = value == "{" && @previous != "(" && diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 56cc8d4..d69d5d3 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -504,7 +504,7 @@ def literal(value) brace_lexer = Arpaka.const_get(:Lexer, false).new("f { 1 }") brace_lexer.each.to_a brace_states = brace_lexer.context.state_history - assert_equal(1, brace_states.find { |entry| entry.fetch(:token) == :tLBRACE_ARG }.fetch(:block_depth)) + assert_equal(1, brace_states.find { |entry| entry.fetch(:token) == "{" }.fetch(:block_depth)) assert_equal(0, brace_states.find { |entry| entry.fetch(:token) == "}" }.fetch(:block_depth)) other = Arpaka.const_get(:Lexer, false).new("value") @@ -597,6 +597,18 @@ def literal(value) assert_nothing_raised { Arpaka.parse_source("assert ?h.in?(\"hello\")") } end + test "source lexer uses regular brace blocks for calls and proc values" do + ["foo { 1 }", "[proc { 1 }]", "foo { 1 }.first", "foo { 1 } || bar"].each do |source| + tokens = Arpaka.const_get(:Lexer, false).new(source).each.to_a + assert_not_include(tokens.map(&:first), :tAMPER, source) + assert_nothing_raised { Arpaka.parse_source(source) } + end + + assert_nothing_raised { Arpaka.parse_source("f(&block)") } + assert_nothing_raised { Arpaka.parse_source("map(&:to_s)") } + assert_nothing_raised { Arpaka.parse_source("a & b") } + end + test "source lexer keeps definition and block contexts for operators" do assert_nothing_raised { Arpaka.parse_source("def ==(x); self.x == x; end") } assert_nothing_raised { Arpaka.parse_source("def []=(x); x; end") } From 40a0e754aa5530acae1f48372693d11d9caf9df9 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sun, 13 Sep 2026 13:05:35 +0900 Subject: [PATCH 140/157] Track block delimiter ownership --- lib/lrama/ruby/languages/ruby/lexer.rb | 11 +++++++++-- test/lrama/ruby_language_test.rb | 8 ++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 91416fb..7933db8 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -33,6 +33,7 @@ def initialize @token_history = [] @state_history = [] @block_stack = [] + @block_delimiter_depths = [] @scope_stack = [] @parser_events = [] @parser_delimiter_stack = [] @@ -78,11 +79,13 @@ def condition? @condition_stack.any? end - def push_block(value) + def push_block(value, delimiter_depth: self.delimiter_depth) @block_stack << value + @block_delimiter_depths << delimiter_depth end def pop_block + @block_delimiter_depths.pop @block_stack.pop end @@ -90,6 +93,10 @@ def block_depth @block_stack.length end + def block_body_at_current_delimiter? + @block_delimiter_depths.last == delimiter_depth + end + def push_scope(value) @scope_stack << value end @@ -362,7 +369,7 @@ def skip_space_and_comments end def newline_ignored? - ignored = (@context.delimiter_depth.positive? && @previous != "}" && + ignored = (@context.delimiter_depth.positive? && !@context.block_body_at_current_delimiter? && @previous != "}" && !(@previous == :keyword_end && @context.block_depth.positive?)) || @previous == "\n" || ["+", "-", "*", "/", "%", "=", "?", ":", ",", ".", "&", "|", "^", "<<", ">>", "&&", "||", "=>", :keyword_and, :keyword_or, :keyword_not, :tAMPER, :tPIPE, :tSTAR, :tDSTAR, :tDOT2, :tDOT3, :tPOW, :tCMP, :tEQ, :tEQQ, :tNEQ, :tGEQ, :tLEQ, :tANDOP, :tOROP, :tMATCH, :tNMATCH, :tLSHFT, :tRSHFT, :tASSOC, :tLAMBDA, :tCOLON2, :tANDDOT].include?(@previous) || @previous == :tLABEL || diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index d69d5d3..4df0155 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -685,6 +685,14 @@ def literal(value) assert_nothing_raised { Arpaka.parse_source("Module.new do\n define_method(:up) do\n 1\n end\n define_method(:down) do\n 2\n end\nend\n") } end + test "source lexer keeps block newlines at their owning delimiter" do + source = "f(-> {\n if condition\n first\n else\n second\n end\n})\n" + assert_nothing_raised { Arpaka.parse_source(source) } + + nested = "f {\n g(\n first,\n second\n )\n}\n" + assert_nothing_raised { Arpaka.parse_source(nested) } + end + test "source lexer ignores newlines after operator assignments" do assert_nothing_raised { Arpaka.parse_source("value ||=\n if condition\n fallback\n end\n") } end From 730b7b7e7812e4c28674e99693d37fc4a893d0f1 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sun, 13 Sep 2026 14:14:49 +0900 Subject: [PATCH 141/157] Scan regexp interpolation boundaries --- lib/lrama/ruby/languages/ruby/lexer.rb | 19 +++++++++++++++++-- test/lrama/ruby_language_test.rb | 9 +++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 7933db8..cd8da58 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -793,7 +793,8 @@ def regexp_or_operator(start) if @context.begin_expression advance value = read_regexp(start) - @pending = [[:tSTRING_CONTENT, value], [:tREGEXP_END, nil]] + content = value.is_a?(Array) ? value : [[:tSTRING_CONTENT, value]] + @pending = content + [[:tREGEXP_END, nil]] return [:tREGEXP_BEG, nil] end operator_or_punctuation(start) @@ -801,6 +802,8 @@ def regexp_or_operator(start) def read_regexp(start) result = +"" + tokens = [] + interpolated = false in_class = false until eof? value = byte @@ -818,10 +821,22 @@ def read_regexp(start) in_class = false result << value.chr advance + elsif value == 35 && byte(1) == 123 && !in_class + tokens << [:tSTRING_CONTENT, result] unless result.empty? + result = +"" + interpolated = true + advance(2) + expression = read_interpolation_source(start) + inner = self.class.new(expression, filename: @filename).each.to_a + inner.pop if inner.last == [0, nil] + tokens << [:tSTRING_DBEG, nil] + tokens.concat(inner) + tokens << [:tSTRING_DEND, nil] elsif value == 47 && !in_class advance advance while byte && byte.between?(97, 122) - return result + tokens << [:tSTRING_CONTENT, result] unless result.empty? + return interpolated ? tokens : result else result << value.chr advance diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 4df0155..9e6973d 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -866,6 +866,15 @@ def literal(value) assert_nothing_raised { Arpaka.parse_source("super(:/, left, right)") } end + test "source lexer scans slashes inside regexp interpolation" do + source = '/#{value.sub(/x/, "")}/' + tokens = Arpaka.const_get(:Lexer, false).new(source).each.to_a + assert_equal([:tREGEXP_BEG, :tSTRING_DBEG, :tIDENTIFIER, ".", :tIDENTIFIER, + "(", :tREGEXP_BEG, :tSTRING_CONTENT, :tREGEXP_END, ",", :tSTRING_BEG, + :tSTRING_END, ")", :tSTRING_DEND, :tREGEXP_END, 0], tokens.map(&:first)) + assert_kind_of(AST::RegexpLiteral, Arpaka.parse_source(source).statements.first) + end + test "source lexer accepts ampersand operator symbols" do assert_nothing_raised { Arpaka.parse_source("items.reduce(:&)\n") } assert_nothing_raised { Arpaka.parse_source("delegate :[], :[]=, to: :paths\n") } From 6b6b5dae435926ac70c6a6e9e9e74e955c3f0c83 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sun, 13 Sep 2026 18:17:38 +0900 Subject: [PATCH 142/157] Lex bracket operators in aliases --- lib/lrama/ruby/languages/ruby/lexer.rb | 4 ++-- test/lrama/ruby_language_test.rb | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index cd8da58..b70180f 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -1097,12 +1097,12 @@ def operator_or_punctuation(start) text = OPERATORS.sort_by { |operator| -operator.bytesize }.find do |operator| next false if ["[]", "[]="].include?(operator) && start.positive? && [9, 10, 11, 12, 13, 32].include?(@source.getbyte(start - 1)) && - @previous != :keyword_def + ![:keyword_def, :keyword_alias].include?(@previous) next false if ["[]", "[]="].include?(operator) && [:tIDENTIFIER, :tCONSTANT, :tFID, :tINTEGER, :tSTRING_END, :tREGEXP_END].include?(@previous) @source.byteslice(@index, operator.bytesize) == operator end - operator_method = [:keyword_def, ".", :tCOLON2, :tANDDOT, :tSYMBEG].include?(@previous) + operator_method = [:keyword_def, :keyword_alias, ".", :tCOLON2, :tANDDOT, :tSYMBEG].include?(@previous) if text && !(begin_expression? && !operator_method && ["[]", "[]="].include?(text)) advance(text.bytesize) token = if text == "**" && @context.begin_expression diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 9e6973d..90ad6ff 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -847,6 +847,8 @@ def literal(value) assert_nothing_raised { Arpaka.parse_source(source) } assert_nothing_raised { Arpaka.parse_source("undef_method :==, :!, :!=\ndef hash; 1; end") } + assert_nothing_raised { Arpaka.parse_source("alias [] get") } + assert_nothing_raised { Arpaka.parse_source("alias []= get") } end test "source lexer allows keyword-shaped receiver setters" do From be5b9be76eac349b8a203b51d3f3c88ac14e7ced Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sun, 13 Sep 2026 18:23:15 +0900 Subject: [PATCH 143/157] Ignore newline before block parameters --- lib/lrama/ruby/languages/ruby/lexer.rb | 1 + test/lrama/ruby_language_test.rb | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index b70180f..2ffe1c1 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -375,6 +375,7 @@ def newline_ignored? @previous == :tLABEL || [:modifier_if, :modifier_unless, :modifier_while, :modifier_until].include?(@previous) || [:tANDOP, :tOROP, :tMATCH, :tNMATCH, :tASSOC, :tOP_ASGN].include?(@previous) || + (@previous == "{" && next_non_space_byte == 124) || next_non_space_byte == 46 ignored = false if @previous_previous == :tSYMBEG && [:tLSHFT, :tRSHFT, :tSTAR, :tDSTAR, :tPOW, :tEQ, :tEQQ, :tNEQ, diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 90ad6ff..3998d0d 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -813,6 +813,10 @@ def literal(value) assert_nothing_raised { Arpaka.parse_source("value\n .first\n .to_s\n") } end + test "source lexer ignores newlines before block parameters" do + assert_nothing_raised { Arpaka.parse_source("call {\n |value| value\n}\n") } + end + test "source lexer continues keyword arguments after label newlines" do assert_nothing_raised do Arpaka.parse_source("parse(file, context:\n build_context)\n") From 6ed16bfbc8cef1ebe4b16866a2668658f78f7768 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sun, 13 Sep 2026 18:26:31 +0900 Subject: [PATCH 144/157] Preserve alias newline lexer state --- lib/lrama/ruby/languages/ruby/lexer.rb | 17 +++++++++-------- test/lrama/ruby_language_test.rb | 1 + 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 2ffe1c1..05173c0 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -306,19 +306,20 @@ def next_token value = case byte when 10 advance + return next_token if @previous == "\n" if @context.alias_context && @previous == :tEQ @context.alias_context = false - return ["\n", nil] - end - if @class_superclass && next_word_is_terminator? + ["\n", nil] + elsif @class_superclass && next_word_is_terminator? @class_superclass = false return [";", nil] + else + @class_superclass = false + return next_token if newline_ignored? + @context.pop_condition if @context.condition? && @context.condition_do + @context.condition_do = false + ["\n", nil] end - @class_superclass = false - return next_token if newline_ignored? - @context.pop_condition if @context.condition? && @context.condition_do - @context.condition_do = false - ["\n", nil] when 39, 34, 96 string_token(byte, start) when 47 diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 3998d0d..1db3a28 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -853,6 +853,7 @@ def literal(value) assert_nothing_raised { Arpaka.parse_source("undef_method :==, :!, :!=\ndef hash; 1; end") } assert_nothing_raised { Arpaka.parse_source("alias [] get") } assert_nothing_raised { Arpaka.parse_source("alias []= get") } + assert_nothing_raised { Arpaka.parse_source("alias :eql? :==\n\ndef hash; 1; end") } end test "source lexer allows keyword-shaped receiver setters" do From 46922d7d03ff520063bd8156ffe074329e5fac72 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sun, 13 Sep 2026 18:32:16 +0900 Subject: [PATCH 145/157] Lex setter definitions without arguments --- lib/lrama/ruby/languages/ruby/lexer.rb | 2 +- test/lrama/ruby_language_test.rb | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 05173c0..6abd726 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -496,7 +496,7 @@ def identifier_token(start) advance return [:tFID, (word + suffix).to_sym] end - if byte == 61 && byte(1) == 40 && [:keyword_def, ".", :tCOLON2].include?(@previous) + if byte == 61 && [:keyword_def, ".", :tCOLON2].include?(@previous) advance return [:tFID, (word + "=").to_sym] end diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 1db3a28..329213e 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -113,6 +113,7 @@ def literal(value) test "setter method definitions preserve the equals suffix" do expected = AST::Program.new([AST::Def.new(:"value=", :value, [AST::BareCall.new(:value)])]) assert_equal(expected, Arpaka.parse_source("def value=(value)\n value\nend")) + assert_nothing_raised { Arpaka.parse_source("def self.value=; end") } end test "keywords after a receiver are method names" do @@ -544,7 +545,7 @@ def literal(value) lexer = Arpaka.const_get(:Lexer, false).new("def value=; end") lexer.each.to_a def_name = lexer.context.state_history.find do |entry| - entry.fetch(:token) == :tIDENTIFIER + entry.fetch(:token) == :tFID end assert_equal(:expr_fname, def_name.fetch(:lex_state)) From 86ac8ff22e85c728a74caef25ab7fb0da11e708e Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sun, 13 Sep 2026 18:39:08 +0900 Subject: [PATCH 146/157] Classify splat method body hashes --- lib/lrama/ruby/languages/ruby/lexer.rb | 2 +- test/lrama/ruby_language_test.rb | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 6abd726..d25968b 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -1128,7 +1128,7 @@ def operator_or_punctuation(start) if value == "(" || value == "[" || value == "{" brace_block = value == "{" && @previous != "(" && ([:tIDENTIFIER, :tCONSTANT, :tFID].include?(@previous) || - @previous == ")" || + (@previous == ")" && !(@previous_previous == :tSTAR && @context.scope_stack.last == :method)) || [".", :tCOLON2].include?(@previous_previous) || [:keyword_super, :keyword_yield].include?(@previous) || [:proc, :lambda].include?(@previous_value)) diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 329213e..0606678 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -330,6 +330,12 @@ def literal(value) Arpaka.parse_source("each do |value| value end")) end + test "method bodies after splat arguments accept hash literals" do + assert_nothing_raised do + Arpaka.parse_source('def decode(*) { foo: "decoded" } end') + end + end + test "if, unless, elsif and else build conditional AST nodes" do expected = AST::Program.new([AST::If.new(literal(true), [literal(1)], [literal(2)])]) assert_equal(expected, parse([:keyword_if, nil], [:keyword_true, nil], [:keyword_then, nil], [:tINTEGER, 1], [:keyword_else, nil], [:tINTEGER, 2], [:keyword_end, nil])) From 6feb8adbe44fe7b513941d05f20eec85fc550ed6 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sun, 13 Sep 2026 20:51:07 +0900 Subject: [PATCH 147/157] Preserve bracket operators in aliases --- lib/lrama/ruby/languages/ruby/lexer.rb | 9 ++++++--- test/lrama/ruby_language_test.rb | 4 ++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index d25968b..ddb9c1a 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -1099,12 +1099,15 @@ def operator_or_punctuation(start) text = OPERATORS.sort_by { |operator| -operator.bytesize }.find do |operator| next false if ["[]", "[]="].include?(operator) && start.positive? && [9, 10, 11, 12, 13, 32].include?(@source.getbyte(start - 1)) && - ![:keyword_def, :keyword_alias].include?(@previous) + ![:keyword_def, :keyword_alias].include?(@previous) && + @previous_previous != :keyword_alias next false if ["[]", "[]="].include?(operator) && - [:tIDENTIFIER, :tCONSTANT, :tFID, :tINTEGER, :tSTRING_END, :tREGEXP_END].include?(@previous) + [:tIDENTIFIER, :tCONSTANT, :tFID, :tINTEGER, :tSTRING_END, :tREGEXP_END].include?(@previous) && + @previous_previous != :keyword_alias @source.byteslice(@index, operator.bytesize) == operator end - operator_method = [:keyword_def, :keyword_alias, ".", :tCOLON2, :tANDDOT, :tSYMBEG].include?(@previous) + operator_method = [:keyword_def, :keyword_alias, ".", :tCOLON2, :tANDDOT, :tSYMBEG].include?(@previous) || + @previous_previous == :keyword_alias if text && !(begin_expression? && !operator_method && ["[]", "[]="].include?(text)) advance(text.bytesize) token = if text == "**" && @context.begin_expression diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 0606678..593feff 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -116,6 +116,10 @@ def literal(value) assert_nothing_raised { Arpaka.parse_source("def self.value=; end") } end + test "alias preserves bracket operator names" do + assert_nothing_raised { Arpaka.parse_source("alias store []=") } + end + test "keywords after a receiver are method names" do receiver = AST::ReceiverCall.new(literal(:self), :".", :class, []) assert_equal(AST::Program.new([AST::ReceiverCall.new(receiver, :".", :to_s, [])]), From 09e536e61106c93bf3b29e9bfebfe6d7febd7f10 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sun, 13 Sep 2026 20:53:45 +0900 Subject: [PATCH 148/157] Restore lambda block context --- lib/lrama/ruby/languages/ruby/lexer.rb | 2 +- test/lrama/ruby_language_test.rb | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index ddb9c1a..37305b7 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -470,7 +470,7 @@ def update_block_context(token) when "{", :tLBRACE_ARG @context.push_block(:brace_block) when "}" - @context.pop_block if @context.block_stack.last == :brace_block + @context.pop_block if [:brace_block, :lambda].include?(@context.block_stack.last) @context.pop_scope if @context.scope_stack.last == :lambda when :keyword_end @context.pop_block if [:do_block, :lambda].include?(@context.block_stack.last) diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 593feff..cefa1c8 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -340,6 +340,12 @@ def literal(value) end end + test "lambda blocks restore an enclosing do block" do + assert_nothing_raised do + Arpaka.parse_source("included do\n value = ->(body) { body }\n Mime[:html]\nend") + end + end + test "if, unless, elsif and else build conditional AST nodes" do expected = AST::Program.new([AST::If.new(literal(true), [literal(1)], [literal(2)])]) assert_equal(expected, parse([:keyword_if, nil], [:keyword_true, nil], [:keyword_then, nil], [:tINTEGER, 1], [:keyword_else, nil], [:tINTEGER, 2], [:keyword_end, nil])) From 2002677225be0bf47a2d51230e41a0a8bd47bcf0 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sun, 13 Sep 2026 20:57:42 +0900 Subject: [PATCH 149/157] Preserve heredoc suffix context --- lib/lrama/ruby/languages/ruby/lexer.rb | 21 ++++++++++++++++++--- test/lrama/ruby_language_test.rb | 6 ++++++ 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 37305b7..05ece21 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -275,7 +275,7 @@ def next_token elsif @pending.empty? && @deferred_suffix suffix = @deferred_suffix @deferred_suffix = nil - @pending.concat(self.class.new(suffix, filename: @filename).each.to_a) + @pending.concat(heredoc_suffix_tokens(suffix)) @pending.pop if @pending.last == [0, nil] @pending << ["\n", nil] @heredoc_newline_pending = false @@ -1026,8 +1026,7 @@ def heredoc_token(start) @pending = content_tokens + [[:tSTRING_END, nil]] prefix, headers, suffix = extract_heredoc_headers(suffix) unless prefix.empty? - prefix_tokens = self.class.new(prefix, filename: @filename).each.to_a - prefix_tokens.pop if prefix_tokens.last == [0, nil] + prefix_tokens = heredoc_suffix_tokens(prefix) @pending.concat(prefix_tokens) end @heredoc_queue.concat(headers) @@ -1068,6 +1067,22 @@ def extract_heredoc_headers(suffix) [suffix[0...match.begin(0)], headers, trailing] end + def heredoc_suffix_tokens(source) + suffix_lexer = self.class.new(source, filename: @filename) + suffix_lexer.context.begin_expression = false + suffix_lexer.context.lex_state = :expr_end + suffix_lexer.instance_variable_set(:@previous, :tSTRING_END) + tokens = suffix_lexer.each.to_a + tokens.pop if tokens.last == [0, nil] + if [:keyword_if, :keyword_unless, :keyword_while, :keyword_until, :keyword_rescue].include?(tokens.first&.first) + token = {keyword_if: :modifier_if, keyword_unless: :modifier_unless, + keyword_while: :modifier_while, keyword_until: :modifier_until, + keyword_rescue: :modifier_rescue}.fetch(tokens.first.first) + tokens[0] = [token, tokens.first.last] + end + tokens + end + def load_queued_heredoc descriptor = @heredoc_queue.shift body = read_heredoc_body(descriptor.fetch(:delimiter), descriptor.fetch(:indent), descriptor.fetch(:squiggly), @index) diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index cefa1c8..3bf94b4 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -346,6 +346,12 @@ def literal(value) end end + test "heredoc suffixes retain literal expression context" do + assert_nothing_raised do + Arpaka.parse_source("value = <<~TEXT unless condition\n body\nTEXT\n") + end + end + test "if, unless, elsif and else build conditional AST nodes" do expected = AST::Program.new([AST::If.new(literal(true), [literal(1)], [literal(2)])]) assert_equal(expected, parse([:keyword_if, nil], [:keyword_true, nil], [:keyword_then, nil], [:tINTEGER, 1], [:keyword_else, nil], [:tINTEGER, 2], [:keyword_end, nil])) From 0cc8a57a819f20cf581b23dedeff4b7d22c17e7e Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sun, 13 Sep 2026 21:01:43 +0900 Subject: [PATCH 150/157] Preserve blocks around singleton classes --- lib/lrama/ruby/languages/ruby/lexer.rb | 14 +++++++++++--- test/lrama/ruby_language_test.rb | 14 ++++++++++++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 05ece21..3e68379 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -10,7 +10,7 @@ module Ruby class LexicalContext attr_accessor :begin_expression, :condition_do, :condition_line, :ternary_depth, :lambda_pending, :alias_context, :argument_label, - :lex_state, :command_start, :label_pending + :lex_state, :command_start, :label_pending, :singleton_class_depth attr_reader :delimiter_stack, :cmdarg_stack, :condition_stack, :token_history, :state_history, :block_stack, :scope_stack, :parser_events, :parser_delimiter_stack, :parser_cmdarg_stack, :parser_block_stack, @@ -25,6 +25,7 @@ def initialize @alias_context = false @argument_label = nil @label_pending = false + @singleton_class_depth = 0 @lex_state = :expr_beg @command_start = true @delimiter_stack = [] @@ -465,6 +466,8 @@ def update_block_context(token) @context.push_block(:lambda) when :tLAMBDA @context.push_scope(:lambda) + when :tLSHFT + @context.singleton_class_depth += 1 if @previous_previous == :keyword_class when :tLAMBEG @context.push_block(:lambda) when "{", :tLBRACE_ARG @@ -473,8 +476,13 @@ def update_block_context(token) @context.pop_block if [:brace_block, :lambda].include?(@context.block_stack.last) @context.pop_scope if @context.scope_stack.last == :lambda when :keyword_end - @context.pop_block if [:do_block, :lambda].include?(@context.block_stack.last) - @context.pop_scope if [:method, :lambda].include?(@context.scope_stack.last) + if @context.singleton_class_depth.positive? + @context.singleton_class_depth -= 1 + @context.pop_scope if @context.scope_stack.last == :keyword_class + else + @context.pop_block if [:do_block, :lambda].include?(@context.block_stack.last) + @context.pop_scope if [:method, :lambda].include?(@context.scope_stack.last) + end end end diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 3bf94b4..3d14b91 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -352,6 +352,20 @@ def literal(value) end end + test "singleton class endings preserve enclosing blocks" do + source = <<~RUBY + tests(Class.new(ActionController::Base) do + class << self + attr_accessor :last_parameters + end + + def parse + end + end) + RUBY + assert_nothing_raised { Arpaka.parse_source(source) } + end + test "if, unless, elsif and else build conditional AST nodes" do expected = AST::Program.new([AST::If.new(literal(true), [literal(1)], [literal(2)])]) assert_equal(expected, parse([:keyword_if, nil], [:keyword_true, nil], [:keyword_then, nil], [:tINTEGER, 1], [:keyword_else, nil], [:tINTEGER, 2], [:keyword_end, nil])) From 9765c2b6c5cdc225caaf6dddbc8f7068fc49b5b1 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sun, 13 Sep 2026 21:05:40 +0900 Subject: [PATCH 151/157] Continue method chains through comments --- lib/lrama/ruby/languages/ruby/lexer.rb | 14 +++++++++++++- test/lrama/ruby_language_test.rb | 7 +++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 3e68379..8f5caff 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -308,6 +308,7 @@ def next_token when 10 advance return next_token if @previous == "\n" + return next_token if next_chain_byte == 46 if @context.alias_context && @previous == :tEQ @context.alias_context = false ["\n", nil] @@ -378,7 +379,7 @@ def newline_ignored? [:modifier_if, :modifier_unless, :modifier_while, :modifier_until].include?(@previous) || [:tANDOP, :tOROP, :tMATCH, :tNMATCH, :tASSOC, :tOP_ASGN].include?(@previous) || (@previous == "{" && next_non_space_byte == 124) || - next_non_space_byte == 46 + next_chain_byte == 46 ignored = false if @previous_previous == :tSYMBEG && [:tLSHFT, :tRSHFT, :tSTAR, :tDSTAR, :tPOW, :tEQ, :tEQQ, :tNEQ, :tCMP, :tGEQ, :tLEQ, :tANDOP, :tOROP, :tMATCH, :tNMATCH, @@ -402,6 +403,17 @@ def next_non_space_byte @source.getbyte(position) end + def next_chain_byte + position = @index + loop do + position += 1 while [9, 11, 12, 13, 32, 10].include?(@source.getbyte(position)) + break unless @source.getbyte(position) == 35 + position += 1 + position += 1 while @source.getbyte(position) && @source.getbyte(position) != 10 + end + @source.getbyte(position) + end + def update_pending_delimiter(token) if ["(", "[", :tLPAREN_ARG].include?(token) @context.push_delimiter(token) diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 3d14b91..bf39110 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -366,6 +366,13 @@ def parse assert_nothing_raised { Arpaka.parse_source(source) } end + test "method chains may continue after a newline before a dot" do + assert_nothing_raised { Arpaka.parse_source("value = result\n .first\n .last") } + assert_nothing_raised do + Arpaka.parse_source("value = result\n .first\n # keep chaining\n .last") + end + end + test "if, unless, elsif and else build conditional AST nodes" do expected = AST::Program.new([AST::If.new(literal(true), [literal(1)], [literal(2)])]) assert_equal(expected, parse([:keyword_if, nil], [:keyword_true, nil], [:keyword_then, nil], [:tINTEGER, 1], [:keyword_else, nil], [:tINTEGER, 2], [:keyword_end, nil])) From 3cb4becceea4d88224126aea7c0349077fbb7ca8 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sun, 13 Sep 2026 21:08:56 +0900 Subject: [PATCH 152/157] Track conditional block endings --- lib/lrama/ruby/languages/ruby/lexer.rb | 10 ++++++++-- test/lrama/ruby_language_test.rb | 6 ++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 8f5caff..1a7659e 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -10,7 +10,7 @@ module Ruby class LexicalContext attr_accessor :begin_expression, :condition_do, :condition_line, :ternary_depth, :lambda_pending, :alias_context, :argument_label, - :lex_state, :command_start, :label_pending, :singleton_class_depth + :lex_state, :command_start, :label_pending, :singleton_class_depth, :control_depth attr_reader :delimiter_stack, :cmdarg_stack, :condition_stack, :token_history, :state_history, :block_stack, :scope_stack, :parser_events, :parser_delimiter_stack, :parser_cmdarg_stack, :parser_block_stack, @@ -26,6 +26,7 @@ def initialize @argument_label = nil @label_pending = false @singleton_class_depth = 0 + @control_depth = 0 @lex_state = :expr_beg @command_start = true @delimiter_stack = [] @@ -470,6 +471,9 @@ def command_start_after(token) def update_block_context(token) case token + when :keyword_if, :keyword_unless, :keyword_case, :keyword_begin, + :keyword_while, :keyword_until, :keyword_for + @context.control_depth += 1 when :keyword_def @context.push_scope(:method) when :keyword_do, :keyword_do_block @@ -488,7 +492,9 @@ def update_block_context(token) @context.pop_block if [:brace_block, :lambda].include?(@context.block_stack.last) @context.pop_scope if @context.scope_stack.last == :lambda when :keyword_end - if @context.singleton_class_depth.positive? + if @context.control_depth.positive? + @context.control_depth -= 1 + elsif @context.singleton_class_depth.positive? @context.singleton_class_depth -= 1 @context.pop_scope if @context.scope_stack.last == :keyword_class else diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index bf39110..7dce893 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -373,6 +373,12 @@ def parse end end + test "conditional endings preserve enclosing blocks" do + assert_nothing_raised do + Arpaka.parse_source("include(Module.new do\n define_method(:value) do\n if condition\n value\n else\n other\n end\n end\nend)") + end + end + test "if, unless, elsif and else build conditional AST nodes" do expected = AST::Program.new([AST::If.new(literal(true), [literal(1)], [literal(2)])]) assert_equal(expected, parse([:keyword_if, nil], [:keyword_true, nil], [:keyword_then, nil], [:tINTEGER, 1], [:keyword_else, nil], [:tINTEGER, 2], [:keyword_end, nil])) From 7d3ac259f27b37245147fb44758c1ee2674abb6e Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sun, 13 Sep 2026 21:11:16 +0900 Subject: [PATCH 153/157] Preserve method scopes inside blocks --- lib/lrama/ruby/languages/ruby/lexer.rb | 4 +++- test/lrama/ruby_language_test.rb | 10 ++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 1a7659e..3b71735 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -497,9 +497,11 @@ def update_block_context(token) elsif @context.singleton_class_depth.positive? @context.singleton_class_depth -= 1 @context.pop_scope if @context.scope_stack.last == :keyword_class + elsif [:method, :keyword_class, :keyword_module].include?(@context.scope_stack.last) + @context.pop_scope else @context.pop_block if [:do_block, :lambda].include?(@context.block_stack.last) - @context.pop_scope if [:method, :lambda].include?(@context.scope_stack.last) + @context.pop_scope if @context.scope_stack.last == :lambda end end end diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 7dce893..ae4d1c1 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -379,6 +379,16 @@ def parse end end + test "method endings preserve enclosing blocks" do + source = <<~RUBY + values = Array.new(3, Class.new do + def create; end + def update; end + end) + RUBY + assert_nothing_raised { Arpaka.parse_source(source) } + end + test "if, unless, elsif and else build conditional AST nodes" do expected = AST::Program.new([AST::If.new(literal(true), [literal(1)], [literal(2)])]) assert_equal(expected, parse([:keyword_if, nil], [:keyword_true, nil], [:keyword_then, nil], [:tINTEGER, 1], [:keyword_else, nil], [:tINTEGER, 2], [:keyword_end, nil])) From 9c3d8b734dc05463cce1161b13f970e67fefe919 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sun, 13 Sep 2026 21:15:07 +0900 Subject: [PATCH 154/157] Classify negative command blocks --- lib/lrama/ruby/languages/ruby/lexer.rb | 4 +++- test/lrama/ruby_language_test.rb | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 3b71735..6e0bd69 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -113,7 +113,7 @@ def scope_depth def remember_token(token) @token_history << token - @token_history.shift while @token_history.length > 4 + @token_history.shift while @token_history.length > 8 end def remember_state(token) @@ -581,6 +581,8 @@ def do_token elsif @context.lambda_pending @context.lambda_pending = false :keyword_do_LAMBDA + elsif @previous == ")" && @context.token_history.each_cons(2).any? { |left, right| left == :tUMINUS && right == :tLPAREN } + :keyword_do_block elsif [")", :keyword_super, :keyword_yield].include?(@previous) :keyword_do elsif @previous == :tLAMBDA diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index ae4d1c1..5eb475b 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -389,6 +389,10 @@ def update; end assert_nothing_raised { Arpaka.parse_source(source) } end + test "negative command arguments use a block do token" do + assert_nothing_raised { Arpaka.parse_source('assert_difference "x", -(value.count) do; value; end') } + end + test "if, unless, elsif and else build conditional AST nodes" do expected = AST::Program.new([AST::If.new(literal(true), [literal(1)], [literal(2)])]) assert_equal(expected, parse([:keyword_if, nil], [:keyword_true, nil], [:keyword_then, nil], [:tINTEGER, 1], [:keyword_else, nil], [:tINTEGER, 2], [:keyword_end, nil])) From 9bebe2a735bccb6616aca37feb8c138fae9cd9c2 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sun, 13 Sep 2026 21:19:04 +0900 Subject: [PATCH 155/157] Lex operator symbols without literals --- lib/lrama/ruby/languages/ruby/lexer.rb | 4 ++-- test/lrama/ruby_language_test.rb | 8 ++++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 6e0bd69..48b9fec 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -326,9 +326,9 @@ def next_token when 39, 34, 96 string_token(byte, start) when 47 - regexp_or_operator(start) + [:keyword_def, :keyword_alias, :tSYMBEG].include?(@previous) ? operator_or_punctuation(start) : regexp_or_operator(start) when 37 - percent_token(start) + [:keyword_def, :keyword_alias, :tSYMBEG].include?(@previous) ? operator_or_punctuation(start) : percent_token(start) when 60 heredoc_or_operator(start) when 48..57 diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index 5eb475b..b75a309 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -393,6 +393,14 @@ def update; end assert_nothing_raised { Arpaka.parse_source('assert_difference "x", -(value.count) do; value; end') } end + test "operator method definitions do not start literals" do + assert_nothing_raised { Arpaka.parse_source("def %(other); end\ndef /(other); end") } + end + + test "slash and percent symbols do not start literals" do + assert_nothing_raised { Arpaka.parse_source("calculate(:/, value)\ncalculate(:%, value)") } + end + test "if, unless, elsif and else build conditional AST nodes" do expected = AST::Program.new([AST::If.new(literal(true), [literal(1)], [literal(2)])]) assert_equal(expected, parse([:keyword_if, nil], [:keyword_true, nil], [:keyword_then, nil], [:tINTEGER, 1], [:keyword_else, nil], [:tINTEGER, 2], [:keyword_end, nil])) From acf769252be31c7ce198aaa5c9676e1b8cc9fd57 Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sun, 13 Sep 2026 21:37:42 +0900 Subject: [PATCH 156/157] Finish Rails Ruby lexer compatibility --- lib/lrama/ruby/languages/ruby/lexer.rb | 27 +++++++++++++++++++++++--- test/lrama/ruby_language_test.rb | 20 +++++++++++++++++++ 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb index 48b9fec..3207ff7 100644 --- a/lib/lrama/ruby/languages/ruby/lexer.rb +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -10,7 +10,8 @@ module Ruby class LexicalContext attr_accessor :begin_expression, :condition_do, :condition_line, :ternary_depth, :lambda_pending, :alias_context, :argument_label, - :lex_state, :command_start, :label_pending, :singleton_class_depth, :control_depth + :lex_state, :command_start, :label_pending, :singleton_class_depth, :control_depth, + :method_header, :method_body_start attr_reader :delimiter_stack, :cmdarg_stack, :condition_stack, :token_history, :state_history, :block_stack, :scope_stack, :parser_events, :parser_delimiter_stack, :parser_cmdarg_stack, :parser_block_stack, @@ -27,6 +28,8 @@ def initialize @label_pending = false @singleton_class_depth = 0 @control_depth = 0 + @method_header = false + @method_body_start = false @lex_state = :expr_beg @command_start = true @delimiter_stack = [] @@ -382,7 +385,7 @@ def newline_ignored? (@previous == "{" && next_non_space_byte == 124) || next_chain_byte == 46 ignored = false if @previous_previous == :tSYMBEG && - [:tLSHFT, :tRSHFT, :tSTAR, :tDSTAR, :tPOW, :tEQ, :tEQQ, :tNEQ, + [:tLSHFT, :tRSHFT, :tSTAR, :tDSTAR, :tPOW, :tEQ, :tEQQ, :tNEQ, "&", :tCMP, :tGEQ, :tLEQ, :tANDOP, :tOROP, :tMATCH, :tNMATCH, :tASSOC, :tOP_ASGN].include?(@previous) @context.condition_line = false if ignored && @context.condition_line @@ -424,6 +427,8 @@ def update_pending_delimiter(token) }.fetch(token) @context.pop_delimiter(opener) @context.pop_cmdarg + @context.method_body_start = true if token == ")" && @context.method_header + @context.method_header = false if token == ")" end end @@ -441,6 +446,7 @@ def expression_begin_after(token) return false if token == ")" || token == "]" || token == "}" return false if token == :keyword_end return false if token == :tINTEGER || token == :tFLOAT || token == :tRATIONAL || token == :tIMAGINARY + return false if @previous_previous == :tSYMBEG && [:tASET, "&"].include?(token) return false if token == :tIDENTIFIER || token == :tCONSTANT || token == :tFID || token == :tSTRING_END || token == :tREGEXP_END return false if token == :keyword_true || token == :keyword_false || token == :keyword_nil || token == :keyword_self return false if [:keyword__LINE__, :keyword__FILE__, :keyword__ENCODING__].include?(token) @@ -461,6 +467,7 @@ def lexical_state_after(token) :tCONSTANT, :tFID, :tIVAR, :tGVAR, :tCVAR, :tNTH_REF, :keyword_true, :keyword_false, :keyword_nil, :keyword_self, :keyword__LINE__, :keyword__FILE__, :keyword__ENCODING__].include?(token) + return :expr_end if @previous_previous == :tSYMBEG && [:tASET, "&"].include?(token) return :expr_label if token == :tLABEL :expr_beg end @@ -475,6 +482,7 @@ def update_block_context(token) :keyword_while, :keyword_until, :keyword_for @context.control_depth += 1 when :keyword_def + @context.method_header = true @context.push_scope(:method) when :keyword_do, :keyword_do_block @context.push_block(:do_block) @@ -504,6 +512,12 @@ def update_block_context(token) @context.pop_scope if @context.scope_stack.last == :lambda end end + if @context.method_header && token != :keyword_def && token != "(" && + !@context.delimiter_stack.include?("(") && + !(@previous_previous == :keyword_def && [:tIDENTIFIER, :tFID].include?(token)) + @context.method_header = false + end + @context.method_body_start = false unless [")", "\n"].include?(token) end def identifier_token(start) @@ -581,6 +595,8 @@ def do_token elsif @context.lambda_pending @context.lambda_pending = false :keyword_do_LAMBDA + elsif @previous == :tIDENTIFIER && @previous_previous == :tLABEL && @previous_value == :lambda + :keyword_do elsif @previous == ")" && @context.token_history.each_cons(2).any? { |left, right| left == :tUMINUS && right == :tLPAREN } :keyword_do_block elsif [")", :keyword_super, :keyword_yield].include?(@previous) @@ -1191,7 +1207,7 @@ def operator_or_punctuation(start) if command_arg return [:tLPAREN_ARG, nil] end - method_definition = @previous_previous == :keyword_def + method_definition = @previous == :keyword_def || @previous_previous == :keyword_def return [@context.begin_expression && !method_definition && ![".", :tCOLON2, :tANDDOT, :keyword_super, :keyword_yield, :tLAMBDA, :tAREF].include?(@previous) ? :tLPAREN : "(", nil] end @@ -1204,11 +1220,16 @@ def operator_or_punctuation(start) [:tIDENTIFIER, :tFID, :tCONSTANT].include?(@previous) array_argument ||= value == "[" && @previous == ")" && @previous_previous == "(" && @context.scope_stack.last == :method + array_argument ||= value == "[" && @context.method_body_start return [value == "[" && (@context.begin_expression || array_argument) ? :tLBRACK : (value == "[" ? "[" : (brace_block ? "{" : :tLBRACE)), nil] elsif value == ")" || value == "]" || value == "}" opener = { ")" => "(", "]" => "[", "}" => "{" }.fetch(value) @context.pop_delimiter(opener) @context.pop_cmdarg + if value == ")" && @context.method_header + @context.method_body_start = true + @context.method_header = false + end elsif value == "-" && @context.begin_expression return [:tUMINUS, nil] elsif value == "+" && @context.begin_expression diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index b75a309..abd0b9c 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -401,6 +401,26 @@ def update; end assert_nothing_raised { Arpaka.parse_source("calculate(:/, value)\ncalculate(:%, value)") } end + test "operator symbols end an alias expression" do + assert_nothing_raised { Arpaka.parse_source("alias_method :regular_writer, :[]=\nunless condition\nend") } + end + + test "ampersand symbols end an expression" do + assert_nothing_raised { Arpaka.parse_source("name == :&\nname\n") } + end + + test "source lexer recognizes lambda as a lambda expression" do + assert_nothing_raised { Arpaka.parse_source("value = lambda do\n work\nend\n") } + end + + test "method bodies may start with an array literal after arguments" do + assert_nothing_raised { Arpaka.parse_source("def multi(a, b, c) [a, b, c] end") } + end + + test "singleton method definitions may parenthesize their receiver" do + assert_nothing_raised { Arpaka.parse_source("def (object.foo).bar(value)\n value\nend") } + end + test "if, unless, elsif and else build conditional AST nodes" do expected = AST::Program.new([AST::If.new(literal(true), [literal(1)], [literal(2)])]) assert_equal(expected, parse([:keyword_if, nil], [:keyword_true, nil], [:keyword_then, nil], [:tINTEGER, 1], [:keyword_else, nil], [:tINTEGER, 2], [:keyword_end, nil])) From 5dfe43d15cb790f1b005c417fdf349ee0792dace Mon Sep 17 00:00:00 2001 From: "MATSUMOTO, Katsuyoshi" Date: Sun, 13 Sep 2026 21:38:54 +0900 Subject: [PATCH 157/157] Cover remaining Rails trick syntax --- test/lrama/ruby_language_test.rb | 44 ++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/test/lrama/ruby_language_test.rb b/test/lrama/ruby_language_test.rb index abd0b9c..319aa89 100644 --- a/test/lrama/ruby_language_test.rb +++ b/test/lrama/ruby_language_test.rb @@ -409,6 +409,50 @@ def update; end assert_nothing_raised { Arpaka.parse_source("name == :&\nname\n") } end + test "Rails trick syntax remains parseable" do + assert_nothing_raised do + Arpaka.parse_source(<<~RUBY) + def source_location + if line_number + "on line" + else + "in " + end + file_name + end + RUBY + Arpaka.parse_source(<<~RUBY) + helper(Module.new do + def render_from_helper + from_test_case(suffix: "!") + end + end) + RUBY + Arpaka.parse_source(<<~RUBY) + def exec_queries + super do |record| + set_inverse(record) + yield record if block_given? + end + end + RUBY + Arpaka.parse_source(<<~RUBY) + result + .first + # keep chaining + .last + RUBY + Arpaka.parse_source(<<~RUBY) + def create_table(table_name, **options) + if block_given? + super { |table| yield compatible_table_definition(table) } + else + super + end + end + RUBY + end + end + test "source lexer recognizes lambda as a lambda expression" do assert_nothing_raised { Arpaka.parse_source("value = lambda do\n work\nend\n") } end