Replace Prism action lexing with a Ruby scanner

README.md

@@ -132,6 +132,19 @@ values are Floats, and `tIDENTIFIER` values are Symbols or Strings. Operators an
 keywords may carry `nil`. Parentheses use Ruby's context-dependent token names
 (for example `tLPAREN` followed by `")"`), not a source string tokenizer.
 
+For source input, use `Arpaka.parse_source`:
+
+```ruby
+tree = Arpaka.parse_source("a = 1\na + 2", filename: "example.rb")
+```
+
+The source lexer is byte-oriented and context-sensitive. It supports the same
+frontend slice as the token API, including numbers, variables, calls, arrays,
+hashes, control flow, definitions, strings, regular expressions, percent
+literals and static heredocs. Lexer failures include filename, line and byte
+column. Ruby interpolation and syntax whose value cannot be represented by the
+current AST raises an explicit lexer error.
+
 The supported slice includes numeric and nil/boolean literals, binary `+ - * /`,
 unary `+ -`, single-expression parentheses, simple local assignments, and
 statements separated by semicolons or newlines. Every successful parse returns
@@ -141,10 +154,19 @@ an `AST::Program`. Its frozen `statements` array contains immutable Data nodes:
 An unassigned bare identifier is a `BareCall`; assignment registers a local
 before reading its right-hand side, including in `a = a`.
 
-This is not yet a complete Ruby frontend. Strings, explicit calls, control flow,
-methods, and multi-statement parentheses are unsupported. Nodes have no source
-locations. Input must already reflect the lexical decisions Ruby's parser and
-lexer normally make together; this API does not parse Ruby source text.
+Explicit receiver calls without blocks produce
+`ReceiverCall(receiver, operator, name, arguments)`. They preserve chained calls,
+ordered positional arguments, the `.` / `::` / `&.` operator, and the implicit
+`call` method in `handler.(message)`. Receiverless calls use `Call(name, arguments)`.
+
+Double quoted strings and backtick strings may contain simple interpolation and
+produce `InterpolatedString(parts)`, where each part is a string or an AST node.
+
+This is not yet a complete Ruby frontend. Percent literal interpolation, blocks,
+method argument scopes, qualified constant paths, and other constructs are still incomplete; a successful
+parse does not guarantee that all source semantics appear in the AST. Nodes have
+no source locations. Token input must already reflect the lexical decisions
+Ruby's parser and lexer normally make together; use `parse_source` for source text.
 
 `Arpaka::ParseError` exposes `token` and `state`. `Arpaka::UnsupportedSyntax`
 exposes the original `rule` and upstream `line`. Both inherit from

arpaka.gemspec

@@ -28,5 +28,4 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_dependency "lrama", ">= 0.8.0"
-  spec.add_dependency "prism", "~> 1.0"
 end

lib/lrama/ruby.rb

@@ -22,7 +22,11 @@ def self.compile(source, filename: "(grammar)", class_name: "Parser", mode: :par
       code = generate(source, filename: filename, class_name: class_name,
         mode: mode, allow_error_rules: allow_error_rules)
       box = ::Ruby::Box.new
-      box.eval(code)
+      begin
+        box.eval(code)
+      rescue SyntaxError => error
+        raise Error, "Invalid generated Ruby for #{filename}: #{error.message}"
+      end
       box.const_get(class_name, false)
     end
   end

lib/lrama/ruby/action_code.rb

@@ -1,31 +1,34 @@
 # frozen_string_literal: true
 
-require "prism"
+require_relative "action_scanner"
 
 module Lrama
   module Ruby
     # Loaded only in the generator's box. Ruby strings, comments, regular
     # expressions and instance variables must not become grammar references.
     module ActionReferences
+      def ruby_short_reference?(offset)
+        references
+        @ruby_short_references.include?(offset)
+      end
+
       private
 
       def _references
-        Prism.lex(s_value).value.filter_map do |token, _state|
-          next unless [:GLOBAL_VARIABLE, :NUMBERED_REFERENCE].include?(token.type)
-
-          value = token.value
-          unless /\A\$(?:\$|[1-9][0-9]*|[a-zA-Z_][a-zA-Z0-9_]*)\z/.match?(value)
-            raise Backend::Error, "Unsupported semantic reference: #{value}"
-          end
-
-          attributes = if /\A\$[0-9]+\z/.match?(value)
-            { number: value.delete_prefix("$").to_i, index: value.delete_prefix("$").to_i }
+        result = ActionScanner.new(s_value, filename: location.grammar_file.path,
+          line: location.first_line, column: location.first_column).scan
+        @ruby_short_references = result.references.select(&:short_interpolation).map(&:first_column)
+        result.references.map do |reference|
+          attributes = if reference.number
+            { number: reference.number, index: reference.number }
           else
-            { name: value.delete_prefix("$") }
+            { name: reference.name }
           end
           Grammar::Reference.new(type: :dollar, **attributes,
-            first_column: token.location.start_offset, last_column: token.location.end_offset)
-        end.sort_by(&:first_column)
+            first_column: reference.first_column, last_column: reference.last_column)
+        end
+      rescue ActionScanner::Error => error
+        raise Backend::Error, error.message
       end
     end
 
@@ -34,25 +37,15 @@ def lex_c_code
         return super unless @end_symbol == "}"
 
         reset_first_position
-        depth = 0
-        # Prism recovers from the yacc references (which are not yet valid
-        # Ruby assignments) and still provides byte-accurate token locations.
-        Prism.lex(@scanner.rest).value.each do |token, _state|
-          case token.type
-          when :BRACE_LEFT, :LAMBDA_BEGIN, :EMBEXPR_BEGIN
-            depth += 1
-          when :BRACE_RIGHT, :EMBEXPR_END
-            if depth.zero?
-              code = @scanner.rest.byteslice(0, token.location.start_offset)
-              @scanner.pos += code.bytesize
-              @line += code.count("\n")
-              @head = @scanner.pos - code.bytesize + code.b.rindex("\n") + 1 if code.include?("\n")
-              return [:C_DECLARATION, Lexer::Token::UserCode.new(s_value: code, location: location)]
-            end
-            depth -= 1
-          end
-        end
-        raise Backend::Error, "Unterminated Ruby action at line #{line}"
+        result = ActionScanner.new(@scanner.rest, filename: @grammar_file.path,
+          line: line, column: column).scan(action: true)
+        code = @scanner.string.byteslice(@scanner.pos, result.end_offset)
+        @scanner.pos += code.bytesize
+        @line += code.count("\n")
+        @head = @scanner.pos - code.bytesize + code.b.rindex("\n") + 1 if code.include?("\n")
+        [:C_DECLARATION, Lexer::Token::UserCode.new(s_value: code, location: location)]
+      rescue ActionScanner::Error => error
+        raise Backend::Error, error.message
       end
     end
   end