diff --git a/README.md b/README.md index b57032e..6aa640b 100644 --- a/README.md +++ b/README.md @@ -132,6 +132,19 @@ values are Floats, and `tIDENTIFIER` values are Symbols or Strings. Operators an keywords may carry `nil`. Parentheses use Ruby's context-dependent token names (for example `tLPAREN` followed by `")"`), not a source string tokenizer. +For source input, use `Arpaka.parse_source`: + +```ruby +tree = Arpaka.parse_source("a = 1\na + 2", filename: "example.rb") +``` + +The source lexer is byte-oriented and context-sensitive. It supports the same +frontend slice as the token API, including numbers, variables, calls, arrays, +hashes, control flow, definitions, strings, regular expressions, percent +literals and static heredocs. Lexer failures include filename, line and byte +column. Ruby interpolation and syntax whose value cannot be represented by the +current AST raises an explicit lexer error. + The supported slice includes numeric and nil/boolean literals, binary `+ - * /`, unary `+ -`, single-expression parentheses, simple local assignments, and statements separated by semicolons or newlines. Every successful parse returns @@ -141,10 +154,19 @@ an `AST::Program`. Its frozen `statements` array contains immutable Data nodes: An unassigned bare identifier is a `BareCall`; assignment registers a local before reading its right-hand side, including in `a = a`. -This is not yet a complete Ruby frontend. Strings, explicit calls, control flow, -methods, and multi-statement parentheses are unsupported. Nodes have no source -locations. Input must already reflect the lexical decisions Ruby's parser and -lexer normally make together; this API does not parse Ruby source text. +Explicit receiver calls without blocks produce +`ReceiverCall(receiver, operator, name, arguments)`. They preserve chained calls, +ordered positional arguments, the `.` / `::` / `&.` operator, and the implicit +`call` method in `handler.(message)`. Receiverless calls use `Call(name, arguments)`. + +Double quoted strings and backtick strings may contain simple interpolation and +produce `InterpolatedString(parts)`, where each part is a string or an AST node. + +This is not yet a complete Ruby frontend. Percent literal interpolation, blocks, +method argument scopes, qualified constant paths, and other constructs are still incomplete; a successful +parse does not guarantee that all source semantics appear in the AST. Nodes have +no source locations. Token input must already reflect the lexical decisions +Ruby's parser and lexer normally make together; use `parse_source` for source text. `Arpaka::ParseError` exposes `token` and `state`. `Arpaka::UnsupportedSyntax` exposes the original `rule` and upstream `line`. Both inherit from diff --git a/arpaka.gemspec b/arpaka.gemspec index ff928b4..a012033 100644 --- a/arpaka.gemspec +++ b/arpaka.gemspec @@ -28,5 +28,4 @@ Gem::Specification.new do |spec| spec.require_paths = ["lib"] spec.add_dependency "lrama", ">= 0.8.0" - spec.add_dependency "prism", "~> 1.0" end diff --git a/lib/lrama/ruby.rb b/lib/lrama/ruby.rb index a2e4b0f..5e76003 100644 --- a/lib/lrama/ruby.rb +++ b/lib/lrama/ruby.rb @@ -22,7 +22,11 @@ def self.compile(source, filename: "(grammar)", class_name: "Parser", mode: :par code = generate(source, filename: filename, class_name: class_name, mode: mode, allow_error_rules: allow_error_rules) box = ::Ruby::Box.new - box.eval(code) + begin + box.eval(code) + rescue SyntaxError => error + raise Error, "Invalid generated Ruby for #{filename}: #{error.message}" + end box.const_get(class_name, false) end end diff --git a/lib/lrama/ruby/action_code.rb b/lib/lrama/ruby/action_code.rb index 3295b60..a670c86 100644 --- a/lib/lrama/ruby/action_code.rb +++ b/lib/lrama/ruby/action_code.rb @@ -1,31 +1,34 @@ # frozen_string_literal: true -require "prism" +require_relative "action_scanner" module Lrama module Ruby # Loaded only in the generator's box. Ruby strings, comments, regular # expressions and instance variables must not become grammar references. module ActionReferences + def ruby_short_reference?(offset) + references + @ruby_short_references.include?(offset) + end + private def _references - Prism.lex(s_value).value.filter_map do |token, _state| - next unless [:GLOBAL_VARIABLE, :NUMBERED_REFERENCE].include?(token.type) - - value = token.value - unless /\A\$(?:\$|[1-9][0-9]*|[a-zA-Z_][a-zA-Z0-9_]*)\z/.match?(value) - raise Backend::Error, "Unsupported semantic reference: #{value}" - end - - attributes = if /\A\$[0-9]+\z/.match?(value) - { number: value.delete_prefix("$").to_i, index: value.delete_prefix("$").to_i } + result = ActionScanner.new(s_value, filename: location.grammar_file.path, + line: location.first_line, column: location.first_column).scan + @ruby_short_references = result.references.select(&:short_interpolation).map(&:first_column) + result.references.map do |reference| + attributes = if reference.number + { number: reference.number, index: reference.number } else - { name: value.delete_prefix("$") } + { name: reference.name } end Grammar::Reference.new(type: :dollar, **attributes, - first_column: token.location.start_offset, last_column: token.location.end_offset) - end.sort_by(&:first_column) + first_column: reference.first_column, last_column: reference.last_column) + end + rescue ActionScanner::Error => error + raise Backend::Error, error.message end end @@ -34,25 +37,15 @@ def lex_c_code return super unless @end_symbol == "}" reset_first_position - depth = 0 - # Prism recovers from the yacc references (which are not yet valid - # Ruby assignments) and still provides byte-accurate token locations. - Prism.lex(@scanner.rest).value.each do |token, _state| - case token.type - when :BRACE_LEFT, :LAMBDA_BEGIN, :EMBEXPR_BEGIN - depth += 1 - when :BRACE_RIGHT, :EMBEXPR_END - if depth.zero? - code = @scanner.rest.byteslice(0, token.location.start_offset) - @scanner.pos += code.bytesize - @line += code.count("\n") - @head = @scanner.pos - code.bytesize + code.b.rindex("\n") + 1 if code.include?("\n") - return [:C_DECLARATION, Lexer::Token::UserCode.new(s_value: code, location: location)] - end - depth -= 1 - end - end - raise Backend::Error, "Unterminated Ruby action at line #{line}" + result = ActionScanner.new(@scanner.rest, filename: @grammar_file.path, + line: line, column: column).scan(action: true) + code = @scanner.string.byteslice(@scanner.pos, result.end_offset) + @scanner.pos += code.bytesize + @line += code.count("\n") + @head = @scanner.pos - code.bytesize + code.b.rindex("\n") + 1 if code.include?("\n") + [:C_DECLARATION, Lexer::Token::UserCode.new(s_value: code, location: location)] + rescue ActionScanner::Error => error + raise Backend::Error, error.message end end end diff --git a/lib/lrama/ruby/action_scanner.rb b/lib/lrama/ruby/action_scanner.rb new file mode 100644 index 0000000..f64c347 --- /dev/null +++ b/lib/lrama/ruby/action_scanner.rb @@ -0,0 +1,353 @@ +# frozen_string_literal: true + +require "strscan" + +module Lrama + module Ruby + # A Ruby source lexer used for action boundary detection and references. + # All positions are bytes in the original source; literals are never decoded. + class ActionScanner + Reference = Data.define(:name, :number, :first_column, :last_column, :short_interpolation) + Result = Data.define(:end_offset, :references) + Heredoc = Data.define(:delimiter, :indent, :interpolate, :offset) + class Error < StandardError; end + + IDENTIFIER = /[a-zA-Z_\x80-\xff][a-zA-Z_0-9\x80-\xff]*[!?]?/n + NUMBER = /(?:0[xX][0-9a-fA-F_]+|0[bB][01_]+|0[oO][0-7_]+|[0-9][0-9_]*(?:\.[0-9_]+)?(?:[eE][+-]?[0-9_]+)?)[ri]*/ + BEGIN_WORDS = %w[if unless while until when return yield break next rescue else elsif then do begin and or not in case for].freeze + END_WORDS = %w[nil true false self end __FILE__ __LINE__ __ENCODING__].freeze + PAIRS = { "(" => ")", "[" => "]", "{" => "}" }.freeze + LITERAL_PAIRS = PAIRS.merge("<" => ">").freeze + # Bound recursive interpolation rather than allowing SystemStackError. + MAX_INTERPOLATION_DEPTH = 128 + + def initialize(source, filename: "(grammar)", line: 1, column: 0) + @source = source.b + @scanner = StringScanner.new(@source) + @filename, @line, @column = filename, line, column + @references = [] + @interpolation_depth = 0 + @strict_references = false + end + + def scan(action: false) + @strict_references = action + code(action ? "}" : nil) + Result.new(@scanner.pos, @references.sort_by(&:first_column).freeze) + end + + private + + def fail_at(message, offset = @scanner.pos) + prefix = @source.byteslice(0, offset) + lines = prefix.count("\n") + column = lines.zero? ? @column + offset : offset - prefix.rindex("\n") - 1 + raise Error, "#{@filename}:#{@line + lines}:#{column}: #{message}" + end + + def code(terminator) + brackets = [] + heredocs = [] + state = :begin + spaced = false + until @scanner.eos? + offset = @scanner.pos + char = @scanner.peek(1) + if @scanner.scan(/[ \t\r\f]+/) + spaced = true + next + elsif @scanner.scan(/\\\r?\n/) + spaced = true + next + elsif @scanner.scan(/\n/) + heredocs.each { |item| heredoc_body(item) } + heredocs.clear + state = :begin unless [:begin, :method].include?(state) + spaced = true + next + elsif char == "#" + @scanner.scan(/[^\n]*/) + next + elsif (offset.zero? || @source.getbyte(offset - 1) == 10) && @scanner.check(/=begin(?=\s|\z)/) + @scanner.scan(/[^\n]*(?:\n|\z)/) + found = false + until @scanner.eos? + line = @scanner.scan(/[^\n]*(?:\n|\z)/) + if /\A=end(?:\s|\z)/.match?(line) + found = true + break + end + end + fail_at("Unterminated block comment", offset) unless found + next + elsif terminator.nil? && (offset.zero? || @source.getbyte(offset - 1) == 10) && + @scanner.check(/__END__(?:\s|\z)/) + @scanner.terminate + return + elsif char == terminator && brackets.empty? + fail_at("Close the action/interpolation after the heredoc body", offset) unless heredocs.empty? + return + elsif PAIRS.key?(char) + brackets << [PAIRS.fetch(char), offset] + @scanner.getch + state = :begin + elsif [")", "]", "}"].include?(char) + fail_at("Mismatched closing #{char}", offset) unless brackets.last&.first == char + brackets.pop + @scanner.getch + state = :end + elsif ["'", '"', "`"].include?(char) + @scanner.getch + literal(char, interpolate: char != "'", offset: offset) + state = :end + elsif char == "$" + reference + state = :end + elsif @scanner.scan(/@@?[a-zA-Z_\x80-\xff][a-zA-Z_0-9\x80-\xff]*/n) + state = :end + elsif @scanner.check(NUMBER) + @scanner.scan(NUMBER) + state = :end + elsif @scanner.check(IDENTIFIER) + word = @scanner.scan(IDENTIFIER) + fail_at("__END__ is not supported in Ruby actions", offset) if word == "__END__" + # Method names after dot/:: are not keywords. + state = if state == :method + :bare + elsif BEGIN_WORDS.include?(word) + :begin + elsif END_WORDS.include?(word) + :end + else + :bare + end + elsif char == "/" || char == "%" || @scanner.check(/<|===|==|=~|!~|!=|<=|>=|<<|>>|\*\*|[+\-~]@?|[!*\/%&|^<>`])/) + fail_at("Unsupported symbol literal", offset) + end + state = :end + elsif char == "@" && @scanner.check(/@@?[a-zA-Z_\x80-\xff][a-zA-Z_0-9\x80-\xff]*/n) + @scanner.scan(/@@?[a-zA-Z_\x80-\xff][a-zA-Z_0-9\x80-\xff]*/n) + state = :end + elsif char == "@" && state == :begin + @scanner.getch + state = :end + elsif @scanner.scan(/(?:\.\.\.?|->|\*\*=|&&=?|\|\|=?|<=>|===|==|=>|!=|!~|=~|<=|>=|>>=?|\*\*|[+\-*|&^]=|[=+\-*!,;:<>?~|&^])/) + state = :begin + else + fail_at("Unsupported Ruby action token #{char.inspect}", offset) + end + spaced = false + end + fail_at("Unterminated heredoc", heredocs.first.offset) unless heredocs.empty? + fail_at("Unclosed #{brackets.last.first}", brackets.last.last) unless brackets.empty? + fail_at("Unterminated Ruby action/interpolation") if terminator + end + + def reference(short: false) + offset = @scanner.pos + @global_variable_start = offset + value = @scanner.scan(/\$(?:\$|[0-9]+|[a-zA-Z_][a-zA-Z0-9_]*)/) + unless value + return scan_global_variable unless @strict_references + fail_at("Unsupported semantic reference", offset) + end + if value.match?(/\A\$0/) || @scanner.check(/[a-zA-Z_0-9\x80-\xff]/n) + return scan_global_variable unless @strict_references + fail_at("Unsupported semantic reference", offset) + end + number = value.match?(/\A\$[0-9]+\z/) ? value.delete_prefix("$").to_i : nil + @references << Reference.new(number ? nil : value.delete_prefix("$"), number, + offset, @scanner.pos, short) + end + + def scan_global_variable + @scanner.pos = @global_variable_start || @scanner.pos + @scanner.scan(/\$(?:`|')/) || + @scanner.scan(/\$(?:[<>]?|[!@&+~?=\/\\;,.:$-][a-zA-Z]?|[<>][^>\n]*>|[0-9]+|[a-zA-Z_][a-zA-Z0-9_]*)/) + fail_at("Unsupported global variable", @scanner.pos) if @scanner.pos == (@global_variable_start || @scanner.pos) + end + + def interpolation + return false unless @scanner.peek(1) == "#" + if @scanner.scan(/#\{/) + @interpolation_depth += 1 + fail_at("Interpolation nesting exceeds #{MAX_INTERPOLATION_DEPTH}") if @interpolation_depth > MAX_INTERPOLATION_DEPTH + code("}") + @scanner.getch + @interpolation_depth -= 1 + elsif @scanner.scan(/#(?=\$)/) + reference(short: true) + elsif @scanner.scan(/#@@?[a-zA-Z_\x80-\xff][a-zA-Z_0-9\x80-\xff]*/n) + # Instance/class variable interpolation is preserved verbatim. + else + return false + end + true + end + + def literal(close, interpolate:, offset:, open: nil, regexp: false) + depth = 0 + until @scanner.eos? + char = @scanner.peek(1) + if char == "\\" + @scanner.getch + @scanner.getch || fail_at("Unterminated escape", offset) + elsif interpolate && interpolation + next + elsif regexp && char == "[" + regexp_class(offset) + elsif char == close + @scanner.getch + return if depth.zero? + depth -= 1 + elsif open && char == open + depth += 1 + @scanner.getch + else + @scanner.getch + end + end + fail_at("Unterminated literal (expected #{close})", offset) + end + + def regexp_class(offset) + @scanner.getch + @scanner.scan(/\^/) + @scanner.scan(/\]/) # A leading ] is literal. + until @scanner.eos? + if @scanner.scan(/\\[\s\S]/) + next + elsif interpolation + next + elsif @scanner.scan(/\[:[^\]\n]*:\]/) + next # POSIX character class inside a bracket expression. + elsif @scanner.scan(/\]/) + return + else + @scanner.getch + end + end + fail_at("Unterminated regexp character class", offset) + end + + def percent_literal(offset) + @scanner.getch + kind = @scanner.scan(/[qQwWiIrsx]/) || "Q" + delimiter = @scanner.getch + unless delimiter && delimiter.bytesize == 1 && + !delimiter.match?(/[a-zA-Z0-9\\\r\n]/n) + fail_at("Unsupported percent literal delimiter", offset) + end + literal(LITERAL_PAIRS.fetch(delimiter, delimiter), open: LITERAL_PAIRS.key?(delimiter) ? delimiter : nil, + interpolate: !%w[q w i s].include?(kind), regexp: kind == "r", offset: offset) + @scanner.scan(/[a-z]*/) if kind == "r" + end + + def heredoc_start(offset) + @scanner.scan(/< error @@ -38,6 +42,11 @@ def self.parse(tokens) end end + def self.parse_source(source, filename: "(ruby)") + lexer = Lexer.new(source, filename: filename) + parse(lexer.each, lexical_context: lexer.context) + end + def self.generated_parser PARSER_MUTEX.synchronize do @generated_parser ||= begin @@ -55,3 +64,4 @@ def self.generated_parser require_relative "ruby/ast" require_relative "ruby/builder" +require_relative "ruby/lexer" diff --git a/lib/lrama/ruby/languages/ruby/ast.rb b/lib/lrama/ruby/languages/ruby/ast.rb index 0bb8e50..1f17c1e 100644 --- a/lib/lrama/ruby/languages/ruby/ast.rb +++ b/lib/lrama/ruby/languages/ruby/ast.rb @@ -19,7 +19,10 @@ module AST RangeLiteral = Data.define(:operator, :begin_value, :end_value) Ternary = Data.define(:condition, :then_value, :else_value) Call = Data.define(:name, :arguments) + ReceiverCall = Data.define(:receiver, :operator, :name, :arguments) + BlockCall = Data.define(:call, :body) StringLiteral = Data.define(:value) + InterpolatedString = Data.define(:parts) Index = Data.define(:receiver, :arguments) Loop = Data.define(:kind, :condition, :body) Control = Data.define(:kind) diff --git a/lib/lrama/ruby/languages/ruby/builder.rb b/lib/lrama/ruby/languages/ruby/builder.rb index e142bc2..aaa2580 100644 --- a/lib/lrama/ruby/languages/ruby/builder.rb +++ b/lib/lrama/ruby/languages/ruby/builder.rb @@ -57,16 +57,33 @@ def call(name, arguments) AST::Call.new(name, arguments.freeze) end + def receiver_call(receiver, operator, name, arguments) + AST::ReceiverCall.new(receiver, operator, name, (arguments || []).freeze) + end + + def block_call(call, body) + AST::BlockCall.new(call, body_nodes(body)) + end + + def append_arguments(arguments, argument) + list = arguments.is_a?(Array) ? arguments : [arguments] + (list + [argument]).freeze + end + def string(value) + if value.is_a?(Array) + return AST::InterpolatedString.new(value.freeze) if value.any? { |part| !part.is_a?(String) } + value = value.join + end AST::StringLiteral.new(value.to_s) end def regexp(value) - AST::RegexpLiteral.new(value.to_s) + AST::RegexpLiteral.new(value.is_a?(Array) ? value.join : value.to_s) end def xstring(value) - AST::XStringLiteral.new(value.to_s) + AST::XStringLiteral.new(value.is_a?(Array) ? value.join : value.to_s) end def for_node(variable, enumerable, body) @@ -94,15 +111,24 @@ def body_nodes(body) end def symbol(value) + value = value.map { |part| part.is_a?(String) ? part : part.to_s }.join if value.is_a?(Array) + value = value.name if value.is_a?(AST::Variable) AST::Literal.new(value.to_sym) end def join_strings(parts) - parts.join + parts + end + + def interpolation(value) + value.is_a?(Array) && value.length == 1 ? value.first : value end def concat_strings(left, right) - AST::StringLiteral.new(left.value + right.value) + left_parts = left.is_a?(AST::InterpolatedString) ? left.parts : [left.value] + right_parts = right.is_a?(AST::InterpolatedString) ? right.parts : [right.value] + parts = left_parts + right_parts + parts.all? { |part| part.is_a?(String) } ? AST::StringLiteral.new(parts.join) : AST::InterpolatedString.new(parts.freeze) end def index(receiver, arguments) @@ -201,7 +227,11 @@ def array(elements) end def word_array(elements, symbols: false) - values = elements.map { |value| symbols ? value.to_sym : AST::StringLiteral.new(value.to_s) } + elements = elements.elements if elements.is_a?(AST::ArrayLiteral) + values = elements.map do |value| + value = value.value if value.is_a?(AST::StringLiteral) + symbols ? value.to_sym : AST::StringLiteral.new(value.to_s) + end AST::ArrayLiteral.new(values.freeze) end diff --git a/lib/lrama/ruby/languages/ruby/lexer.rb b/lib/lrama/ruby/languages/ruby/lexer.rb new file mode 100644 index 0000000..3207ff7 --- /dev/null +++ b/lib/lrama/ruby/languages/ruby/lexer.rb @@ -0,0 +1,1263 @@ +# frozen_string_literal: true + +module Lrama + module Ruby + module Languages + module Ruby + # Lexer for Ruby source input. This deliberately does not use Prism: + # the parser needs the same small amount of lexical context that CRuby + # keeps in its parser (EXPR_BEG/EXPR_END and delimiter nesting). + class LexicalContext + attr_accessor :begin_expression, :condition_do, :condition_line, + :ternary_depth, :lambda_pending, :alias_context, :argument_label, + :lex_state, :command_start, :label_pending, :singleton_class_depth, :control_depth, + :method_header, :method_body_start + attr_reader :delimiter_stack, :cmdarg_stack, :condition_stack, + :token_history, :state_history, :block_stack, :scope_stack, :parser_events, + :parser_delimiter_stack, :parser_cmdarg_stack, :parser_block_stack, + :parser_condition_stack, :parser_scope_stack + + def initialize + @begin_expression = true + @condition_do = false + @condition_line = false + @ternary_depth = 0 + @lambda_pending = false + @alias_context = false + @argument_label = nil + @label_pending = false + @singleton_class_depth = 0 + @control_depth = 0 + @method_header = false + @method_body_start = false + @lex_state = :expr_beg + @command_start = true + @delimiter_stack = [] + @cmdarg_stack = [] + @condition_stack = [] + @token_history = [] + @state_history = [] + @block_stack = [] + @block_delimiter_depths = [] + @scope_stack = [] + @parser_events = [] + @parser_delimiter_stack = [] + @parser_cmdarg_stack = [] + @parser_block_stack = [] + @parser_condition_stack = [] + @parser_scope_stack = [] + end + + def delimiter_depth + @delimiter_stack.length + end + + def push_delimiter(value) + @delimiter_stack << value + end + + def pop_delimiter(value) + @delimiter_stack.pop if @delimiter_stack.last == value + end + + def push_cmdarg(value) + @cmdarg_stack << value + end + + def pop_cmdarg + @cmdarg_stack.pop + end + + def cmdarg? + @cmdarg_stack.last == true + end + + def push_condition(value = true) + @condition_stack << value + end + + def pop_condition + @condition_stack.pop + end + + def condition? + @condition_stack.any? + end + + def push_block(value, delimiter_depth: self.delimiter_depth) + @block_stack << value + @block_delimiter_depths << delimiter_depth + end + + def pop_block + @block_delimiter_depths.pop + @block_stack.pop + end + + def block_depth + @block_stack.length + end + + def block_body_at_current_delimiter? + @block_delimiter_depths.last == delimiter_depth + end + + def push_scope(value) + @scope_stack << value + end + + def pop_scope + @scope_stack.pop + end + + def scope_depth + @scope_stack.length + end + + def remember_token(token) + @token_history << token + @token_history.shift while @token_history.length > 8 + end + + def remember_state(token) + @state_history << { + token: token, + lex_state: @lex_state, + begin_expression: @begin_expression, + command_start: @command_start, + delimiter_depth: delimiter_depth, + cmdarg_depth: @cmdarg_stack.length, + condition_depth: @condition_stack.length, + block_depth: @block_stack.length, + scope_depth: @scope_stack.length + }.freeze + end + + def parser_shift(token, value, state, action) + @parser_events << [:shift, token, state, action].freeze + if ["(", "[", :tLPAREN_ARG].include?(token) + @parser_delimiter_stack << token + @parser_cmdarg_stack << (token == :tLPAREN_ARG) + elsif [")", "]", "}"].include?(token) + opener = {")" => "(", "]" => "[", "}" => "{" + }.fetch(token) + @parser_delimiter_stack.pop if @parser_delimiter_stack.last == opener + @parser_cmdarg_stack.pop + end + if [:keyword_do, :keyword_do_block, :keyword_do_LAMBDA, :tLAMBEG].include?(token) || + token == "{" || token == :tLBRACE_ARG + @parser_block_stack << token + elsif token == "}" && @parser_block_stack.last + @parser_block_stack.pop + elsif token == :keyword_end && @parser_block_stack.last + @parser_block_stack.pop + end + if [:keyword_while, :keyword_until, :keyword_for].include?(token) + @parser_condition_stack << token + elsif token == :keyword_do_cond && @parser_condition_stack.last + @parser_condition_stack.pop + elsif token == :keyword_end && @parser_condition_stack.last + @parser_condition_stack.pop + end + if token == :keyword_def + @parser_scope_stack << :method + elsif [:keyword_class, :keyword_module].include?(token) + @parser_scope_stack << token + elsif token == :tLAMBDA + @parser_scope_stack << :lambda + elsif token == "}" && @parser_scope_stack.last == :lambda + @parser_scope_stack.pop + elsif token == :keyword_end && @parser_scope_stack.last + @parser_scope_stack.pop + end + end + + def parser_reduce(rule, state) + @parser_events << [:reduce, rule, state].freeze + end + end + + class Lexer + Error = LexerError + + attr_reader :context + + KEYWORDS = { + "class" => :keyword_class, "module" => :keyword_module, + "def" => :keyword_def, "begin" => :keyword_begin, + "rescue" => :keyword_rescue, "ensure" => :keyword_ensure, + "end" => :keyword_end, "if" => :keyword_if, + "unless" => :keyword_unless, "then" => :keyword_then, + "elsif" => :keyword_elsif, "else" => :keyword_else, + "case" => :keyword_case, "when" => :keyword_when, + "while" => :keyword_while, "until" => :keyword_until, + "for" => :keyword_for, "break" => :keyword_break, + "next" => :keyword_next, "redo" => :keyword_redo, + "retry" => :keyword_retry, "return" => :keyword_return, + "yield" => :keyword_yield, "super" => :keyword_super, + "self" => :keyword_self, "nil" => :keyword_nil, + "true" => :keyword_true, "false" => :keyword_false, + "and" => :keyword_and, "or" => :keyword_or, + "not" => :keyword_not, "alias" => :keyword_alias, + "defined?" => :keyword_defined, "in" => :keyword_in, + "BEGIN" => :keyword_BEGIN, "END" => :keyword_END, + "__LINE__" => :keyword__LINE__, "__FILE__" => :keyword__FILE__, + "__ENCODING__" => :keyword__ENCODING__ + }.freeze + + OPERATORS = %w[[]= [] ... .. <=> === == != =~ !~ >= <= && || << >> ** => :: &. -> += -= *= /= %= **= <<= >>= &&= ||= |= &= ^=].freeze + OP_TOKENS = { + "**" => :tPOW, "<=>" => :tCMP, "==" => :tEQ, "===" => :tEQQ, + "!=" => :tNEQ, ">=" => :tGEQ, "<=" => :tLEQ, "&&" => :tANDOP, + "||" => :tOROP, "=~" => :tMATCH, "!~" => :tNMATCH, + ".." => :tDOT2, "..." => :tDOT3, "<<" => :tLSHFT, + ">>" => :tRSHFT, "&." => :tANDDOT, "::" => :tCOLON2, + "=>" => :tASSOC, "->" => :tLAMBDA, "[]" => :tAREF, "[]=" => :tASET + }.freeze + + def initialize(source, filename: "(ruby)") + unless source.is_a?(String) + raise ArgumentError, "source must be a String" + end + unless [Encoding::UTF_8, Encoding::US_ASCII].include?(source.encoding) && source.valid_encoding? + raise Error, "#{filename}:1:0: invalid UTF-8 source" + end + @source = source.b + @filename = filename + @index = 0 + @line = 1 + @column = 0 + @previous_previous = nil + @previous = nil + @previous_value = nil + @context = LexicalContext.new + @pending = [] + @heredoc_queue = [] + @deferred_suffix = nil + @heredoc_newline_pending = false + @class_superclass = false + end + + def each + return enum_for(__method__) unless block_given? + until eof? && @pending.empty? && @heredoc_queue.empty? && @deferred_suffix.nil? + token = next_token + yield token if token + end + yield [0, nil] + end + + private + + def eof? + @index >= @source.bytesize + end + + def byte(offset = 0) + @source.getbyte(@index + offset) + end + + def advance(count = 1) + count.times do + value = byte + @index += 1 + if value == 10 + @line += 1 + @column = 0 + else + @column += 1 + end + end + end + + def fail!(message, index = @index, line = @line, column = @column) + raise Error, "#{@filename}:#{line}:#{column}: #{message}" + end + + def next_token + if @pending.empty? && @heredoc_queue.any? + load_queued_heredoc + elsif @pending.empty? && @deferred_suffix + suffix = @deferred_suffix + @deferred_suffix = nil + @pending.concat(heredoc_suffix_tokens(suffix)) + @pending.pop if @pending.last == [0, nil] + @pending << ["\n", nil] + @heredoc_newline_pending = false + elsif @pending.empty? && @heredoc_newline_pending + @pending << ["\n", nil] + @heredoc_newline_pending = false + end + unless @pending.empty? + value = @pending.shift + return next_token if value[0] == "\n" && newline_ignored? + @previous_previous = @previous + @previous = value[0] + @previous_value = value[1] + update_pending_delimiter(value[0]) + update_argument_label(value[0], value[1]) + @context.remember_token(value[0]) + @context.begin_expression = expression_begin_after(value[0]) + @context.lex_state = lexical_state_after(value[0]) + @context.command_start = command_start_after(value[0]) + update_block_context(value[0]) + @context.remember_state(value[0]) + return value + end + skip_space_and_comments + return nil if eof? + + start = @index + value = case byte + when 10 + advance + return next_token if @previous == "\n" + return next_token if next_chain_byte == 46 + if @context.alias_context && @previous == :tEQ + @context.alias_context = false + ["\n", nil] + elsif @class_superclass && next_word_is_terminator? + @class_superclass = false + return [";", nil] + else + @class_superclass = false + return next_token if newline_ignored? + @context.pop_condition if @context.condition? && @context.condition_do + @context.condition_do = false + ["\n", nil] + end + when 39, 34, 96 + string_token(byte, start) + when 47 + [:keyword_def, :keyword_alias, :tSYMBEG].include?(@previous) ? operator_or_punctuation(start) : regexp_or_operator(start) + when 37 + [:keyword_def, :keyword_alias, :tSYMBEG].include?(@previous) ? operator_or_punctuation(start) : percent_token(start) + when 60 + heredoc_or_operator(start) + when 48..57 + number_token(start) + when 36, 64 + variable_token(start) + when 63 + character_or_question(start) + when 58 + symbol_or_colon(start) + when 65..90, 95, 97..122, 128..255 + identifier_token(start) + else + operator_or_punctuation(start) + end + @previous_previous = @previous + @previous = value && value[0] + @previous_value = value && value[1] + update_argument_label(value && value[0], value && value[1]) + @context.remember_token(value && value[0]) + @context.begin_expression = expression_begin_after(value && value[0]) + @context.lex_state = lexical_state_after(value && value[0]) + @context.command_start = command_start_after(value && value[0]) + update_block_context(value && value[0]) + @context.remember_state(value && value[0]) + value + end + + def skip_space_and_comments + loop do + while [9, 11, 12, 13, 32].include?(byte) + advance + end + if byte == 92 && byte(1) == 10 + advance(2) + next + end + break unless byte == 35 + advance + advance while !eof? && byte != 10 + end + end + + def newline_ignored? + ignored = (@context.delimiter_depth.positive? && !@context.block_body_at_current_delimiter? && @previous != "}" && + !(@previous == :keyword_end && @context.block_depth.positive?)) || @previous == "\n" || + ["+", "-", "*", "/", "%", "=", "?", ":", ",", ".", "&", "|", "^", "<<", ">>", "&&", "||", "=>", :keyword_and, :keyword_or, :keyword_not, :tAMPER, :tPIPE, :tSTAR, :tDSTAR, :tDOT2, :tDOT3, :tPOW, :tCMP, :tEQ, :tEQQ, :tNEQ, :tGEQ, :tLEQ, :tANDOP, :tOROP, :tMATCH, :tNMATCH, :tLSHFT, :tRSHFT, :tASSOC, :tLAMBDA, :tCOLON2, :tANDDOT].include?(@previous) || + @previous == :tLABEL || + [:modifier_if, :modifier_unless, :modifier_while, :modifier_until].include?(@previous) || + [:tANDOP, :tOROP, :tMATCH, :tNMATCH, :tASSOC, :tOP_ASGN].include?(@previous) || + (@previous == "{" && next_non_space_byte == 124) || + next_chain_byte == 46 + ignored = false if @previous_previous == :tSYMBEG && + [:tLSHFT, :tRSHFT, :tSTAR, :tDSTAR, :tPOW, :tEQ, :tEQQ, :tNEQ, "&", + :tCMP, :tGEQ, :tLEQ, :tANDOP, :tOROP, :tMATCH, :tNMATCH, + :tASSOC, :tOP_ASGN].include?(@previous) + @context.condition_line = false if ignored && @context.condition_line + ignored + end + + def next_word_is_terminator? + position = @index + position += 1 while [9, 11, 12, 13, 32].include?(@source.getbyte(position)) + %w[else elsif end when].any? do |word| + @source.byteslice(position, word.bytesize) == word && + !identifier_byte?(@source.getbyte(position + word.bytesize)) + end + end + + def next_non_space_byte + position = @index + position += 1 while [9, 11, 12, 13, 32].include?(@source.getbyte(position)) + @source.getbyte(position) + end + + def next_chain_byte + position = @index + loop do + position += 1 while [9, 11, 12, 13, 32, 10].include?(@source.getbyte(position)) + break unless @source.getbyte(position) == 35 + position += 1 + position += 1 while @source.getbyte(position) && @source.getbyte(position) != 10 + end + @source.getbyte(position) + end + + def update_pending_delimiter(token) + if ["(", "[", :tLPAREN_ARG].include?(token) + @context.push_delimiter(token) + @context.push_cmdarg(token == :tLPAREN_ARG) + elsif [")", "]", "}"].include?(token) + opener = {")" => "(", "]" => "[", "}" => "{" + }.fetch(token) + @context.pop_delimiter(opener) + @context.pop_cmdarg + @context.method_body_start = true if token == ")" && @context.method_header + @context.method_header = false if token == ")" + end + end + + def update_argument_label(token, value) + if token == :tLABEL + @context.argument_label = value + @context.label_pending = true + elsif ["\n", ";"].include?(token) + @context.argument_label = nil + end + end + + def expression_begin_after(token) + return true if token.nil? || token == "\n" + return false if token == ")" || token == "]" || token == "}" + return false if token == :keyword_end + return false if token == :tINTEGER || token == :tFLOAT || token == :tRATIONAL || token == :tIMAGINARY + return false if @previous_previous == :tSYMBEG && [:tASET, "&"].include?(token) + return false if token == :tIDENTIFIER || token == :tCONSTANT || token == :tFID || token == :tSTRING_END || token == :tREGEXP_END + return false if token == :keyword_true || token == :keyword_false || token == :keyword_nil || token == :keyword_self + return false if [:keyword__LINE__, :keyword__FILE__, :keyword__ENCODING__].include?(token) + return false if token == :tIVAR || token == :tGVAR || token == :tCVAR || token == :tNTH_REF + true + end + + def lexical_state_after(token) + return :expr_beg if token.nil? || token == "\n" + return :expr_fname if @previous_previous == :keyword_def || @context.alias_context + return :expr_end if token == 0 + if @context.label_pending && token != :tLABEL + @context.label_pending = false + return :expr_labeled + end + return :expr_end if [")", "]", "}", :keyword_end, :tSTRING_END, :tREGEXP_END, + :tINTEGER, :tFLOAT, :tRATIONAL, :tIMAGINARY, :tIDENTIFIER, + :tCONSTANT, :tFID, :tIVAR, :tGVAR, :tCVAR, :tNTH_REF, + :keyword_true, :keyword_false, :keyword_nil, :keyword_self, + :keyword__LINE__, :keyword__FILE__, :keyword__ENCODING__].include?(token) + return :expr_end if @previous_previous == :tSYMBEG && [:tASET, "&"].include?(token) + return :expr_label if token == :tLABEL + :expr_beg + end + + def command_start_after(token) + [nil, 0, "\n", ";"].include?(token) + end + + def update_block_context(token) + case token + when :keyword_if, :keyword_unless, :keyword_case, :keyword_begin, + :keyword_while, :keyword_until, :keyword_for + @context.control_depth += 1 + when :keyword_def + @context.method_header = true + @context.push_scope(:method) + when :keyword_do, :keyword_do_block + @context.push_block(:do_block) + when :keyword_do_LAMBDA + @context.push_block(:lambda) + when :tLAMBDA + @context.push_scope(:lambda) + when :tLSHFT + @context.singleton_class_depth += 1 if @previous_previous == :keyword_class + when :tLAMBEG + @context.push_block(:lambda) + when "{", :tLBRACE_ARG + @context.push_block(:brace_block) + when "}" + @context.pop_block if [:brace_block, :lambda].include?(@context.block_stack.last) + @context.pop_scope if @context.scope_stack.last == :lambda + when :keyword_end + if @context.control_depth.positive? + @context.control_depth -= 1 + elsif @context.singleton_class_depth.positive? + @context.singleton_class_depth -= 1 + @context.pop_scope if @context.scope_stack.last == :keyword_class + elsif [:method, :keyword_class, :keyword_module].include?(@context.scope_stack.last) + @context.pop_scope + else + @context.pop_block if [:do_block, :lambda].include?(@context.block_stack.last) + @context.pop_scope if @context.scope_stack.last == :lambda + end + end + if @context.method_header && token != :keyword_def && token != "(" && + !@context.delimiter_stack.include?("(") && + !(@previous_previous == :keyword_def && [:tIDENTIFIER, :tFID].include?(token)) + @context.method_header = false + end + @context.method_body_start = false unless [")", "\n"].include?(token) + end + + def identifier_token(start) + while identifier_byte?(byte) + advance + end + word = @source.byteslice(start, @index - start).force_encoding(Encoding::UTF_8) + if byte == 63 && word == "defined" + advance + word = "defined?" + end + if byte == 63 && byte(1) == 58 + advance(2) + return [:tLABEL, (word + "?").to_sym] + end + if [33, 63].include?(byte) && !identifier_byte?(byte(1)) && byte(1) != 61 + suffix = byte.chr + advance + return [:tFID, (word + suffix).to_sym] + end + if byte == 61 && [:keyword_def, ".", :tCOLON2].include?(@previous) + advance + return [:tFID, (word + "=").to_sym] + end + if byte == 61 && @context.alias_context + advance + return [:tFID, (word + "=").to_sym] + end + if byte == 61 && @previous == :tSYMBEG + advance + return [:tIDENTIFIER, (word + "=").to_sym] + end + if byte == 58 && byte(1) != 58 + advance + return [:tLABEL, word.to_sym] + end + if word == "do" && @context.lambda_pending + return [do_token, word.to_sym] + end + if word == "do" && !@context.condition_do && no_argument_block? + return [:keyword_do, word.to_sym] + end + token = KEYWORDS[word] + if token && [".", :tCOLON2, :tANDDOT].include?(@previous) + token = next_non_space_byte == 61 ? :tIDENTIFIER : :tFID + elsif token && @previous == :keyword_def && word != "self" + token = :tFID + elsif (!@context.begin_expression || [:keyword_return, :keyword_break, :keyword_next, :keyword_end, :keyword_yield, :keyword_super].include?(@previous)) && { "if" => :modifier_if, "unless" => :modifier_unless, + "while" => :modifier_while, "until" => :modifier_until, + "rescue" => :modifier_rescue }.key?(word) + token = { "if" => :modifier_if, "unless" => :modifier_unless, + "while" => :modifier_while, "until" => :modifier_until, + "rescue" => :modifier_rescue }.fetch(word, token) + end + token ||= word == "do" ? do_token : (word.getbyte(0).between?(65, 90) ? :tCONSTANT : :tIDENTIFIER) + if [:keyword_while, :keyword_until, :keyword_for].include?(token) + @context.condition_do = true + @context.push_condition + end + @context.alias_context = true if token == :keyword_alias + [token, word.to_sym] + rescue EncodingError + fail!("invalid UTF-8 identifier", start) + end + + def identifier_byte?(value) + value && (value == 95 || value.between?(65, 90) || value.between?(97, 122) || value >= 128 || value.between?(48, 57)) + end + + def do_token + if @context.condition_do && @context.condition? + @context.pop_condition + @context.condition_do = false + :keyword_do_cond + elsif @context.lambda_pending + @context.lambda_pending = false + :keyword_do_LAMBDA + elsif @previous == :tIDENTIFIER && @previous_previous == :tLABEL && @previous_value == :lambda + :keyword_do + elsif @previous == ")" && @context.token_history.each_cons(2).any? { |left, right| left == :tUMINUS && right == :tLPAREN } + :keyword_do_block + elsif [")", :keyword_super, :keyword_yield].include?(@previous) + :keyword_do + elsif @previous == :tLAMBDA + :keyword_do_LAMBDA + else + :keyword_do_block + end + end + + def no_argument_block? + return false unless [:tIDENTIFIER, :tCONSTANT, :tFID].include?(@previous) + return false if @context.argument_label == :at && [".", :tCOLON2, :tANDDOT].include?(@previous_previous) + return false if @context.token_history == ["+", :tINTEGER, ".", :tIDENTIFIER] + return false if [:tIDENTIFIER, :tCONSTANT, :tFID].include?(@previous_previous) + return false if [:tSYMBEG, :tLABEL, :tCOLON2].include?(@previous_previous) + return true if @previous_value == :lambda && @previous_previous == :tLSHFT + return false if [:tLSHFT, :tLAMBDA].include?(@previous_previous) + true + end + + def no_argument_brace_block? + return false if [".", :tCOLON2].include?(@previous_previous) + return false if [:tLABEL, :tLSHFT, :tOROP, :tANDOP, :tASSOC, :tAMPER, "=", "(", ",", ":", "?"].include?(@previous_previous) + no_argument_block? + end + + def number_token(start) + base = 10 + if byte == 48 && [120, 88, 98, 66, 111, 79].include?(byte(1)) + base = { 120 => 16, 88 => 16, 98 => 2, 66 => 2, 111 => 8, 79 => 8 }.fetch(byte(1)) + advance(2) + digit_start = @index + advance while digit_byte?(byte, base) || byte == 95 + fail!("invalid numeric literal", start) if @index == digit_start + text = @source.byteslice(digit_start, @index - digit_start).delete("_") + return [:tINTEGER, text.to_i(base)] + end + advance while digit_byte?(byte, 10) || byte == 95 + float = false + if byte == 46 && digit_byte?(byte(1), 10) + float = true + advance + advance while digit_byte?(byte, 10) || byte == 95 + end + if byte == 101 || byte == 69 + float = true + advance + advance if byte == 43 || byte == 45 + fail!("invalid numeric literal", start) unless digit_byte?(byte, 10) + advance while digit_byte?(byte, 10) || byte == 95 + end + suffix = byte + advance if [105, 114].include?(suffix) + text = @source.byteslice(start, @index - start).delete("_") + return [:tRATIONAL, Rational(text.delete_suffix("r"))] if suffix == 114 + return [:tIMAGINARY, Complex(0, text.delete_suffix("i").to_f)] if suffix == 105 + [float ? :tFLOAT : :tINTEGER, float ? text.to_f : text.to_i] + rescue ArgumentError, ZeroDivisionError + fail!("invalid numeric literal", start) + end + + def digit_byte?(value, base) + return false unless value + value.between?(48, 57) && value - 48 < base || base == 16 && value.between?(65, 70) || base == 16 && value.between?(97, 102) + end + + def variable_token(start) + marker = byte + advance + advance if marker == 64 && byte == 64 + if marker == 36 && [33, 38, 39, 43, 58, 60, 62, 61, 63, 96, 126].include?(byte) + advance + text = @source.byteslice(start, @index - start) + return [:tGVAR, text.to_sym] + end + if marker == 36 && byte == 36 + advance + return [:tGVAR, :"$$"] + end + if marker == 36 && byte && byte.between?(48, 57) + advance while byte && byte.between?(48, 57) + return [:tNTH_REF, @source.byteslice(start + 1, @index - start - 1).to_i] + end + advance while identifier_byte?(byte) + text = @source.byteslice(start, @index - start) + token = marker == 36 ? :tGVAR : (text.start_with?("@@") ? :tCVAR : :tIVAR) + [token, text.to_sym] + end + + def character_or_question(start) + command_character = !@context.begin_expression && + [:tIDENTIFIER, :tCONSTANT, :tFID].include?(@previous) && + start.positive? && [9, 32].include?(@source.getbyte(start - 1)) + return operator_or_punctuation(start) unless (@context.begin_expression || command_character) && + byte(1) && byte(1) != 32 && byte(1) != 10 + advance + value = if byte == 92 + escape_sequence(start) + else + length = utf8_character_length(byte) + character = @source.byteslice(@index, length).force_encoding(Encoding::UTF_8) + advance(length) + character + end + [:tCHAR, value] + end + + def utf8_character_length(first_byte) + return 1 if first_byte < 0x80 + return 2 if first_byte.between?(0xC2, 0xDF) + return 3 if first_byte.between?(0xE0, 0xEF) + return 4 if first_byte.between?(0xF0, 0xF4) + fail!("invalid UTF-8 character") + end + + def symbol_or_colon(start) + symbol_position = @context.begin_expression || ([:tIDENTIFIER, :tFID].include?(@previous) && @context.ternary_depth.zero?) + return operator_or_punctuation(start) unless symbol_position && byte(1) && + ![9, 10, 11, 12, 13, 32].include?(byte(1)) && byte(1) != 58 + advance + if byte == 39 || byte == 34 + quote = byte + advance + @pending = read_interpolated_quoted(quote, start) + @pending << [:tSTRING_END, nil] + end + [:tSYMBEG, nil] + end + + def string_token(quote, start) + advance + content_tokens = read_interpolated_quoted(quote, start) + label = byte == 58 + advance if label + terminator = label ? :tLABEL_END : :tSTRING_END + @pending = content_tokens + @pending << [terminator, nil] + [quote == 96 ? :tXSTRING_BEG : :tSTRING_BEG, nil] + end + + def read_interpolated_quoted(quote, start) + return [[:tSTRING_CONTENT, read_quoted(quote, interpolate: false, start: start)]] if quote == 39 + content = +"" + tokens = [] + until eof? + value = byte + if value == quote + advance + tokens << [:tSTRING_CONTENT, content] unless content.empty? + return tokens + elsif value == 92 + content << escape_sequence(start) + elsif value == 35 && byte(1) == 123 + tokens << [:tSTRING_CONTENT, content] unless content.empty? + content = +"" + advance(2) + expression = read_interpolation_source(start) + inner = self.class.new(expression, filename: @filename).each.to_a + inner.pop if inner.last == [0, nil] + tokens << [:tSTRING_DBEG, nil] + tokens.concat(inner) + tokens << [:tSTRING_DEND, nil] + else + content << value.chr + advance + end + end + fail!("unterminated literal", start) + end + + def read_interpolation_source(start) + begin_index = @index + depth = 1 + quote = nil + escaped = false + while !eof? + value = byte + if quote + if escaped + escaped = false + elsif value == 92 + escaped = true + elsif value == quote + quote = nil + end + elsif [39, 34, 96].include?(value) && @source.getbyte(@index - 1) != 36 + quote = value + elsif value == 123 + depth += 1 + elsif value == 125 + depth -= 1 + if depth.zero? + expression = @source.byteslice(begin_index, @index - begin_index) + advance + return expression.force_encoding(Encoding::UTF_8) + end + end + advance + end + fail!("unterminated string interpolation", start) + end + + def read_quoted(quote, interpolate:, start:) + result = +"" + until eof? + value = byte + if value == quote + advance + return result + elsif value == 92 + result << escape_sequence(start) + elsif value == 35 && interpolate && byte(1) == 123 + fail!("string interpolation cannot be represented by the AST", start) + else + result << value + advance + end + end + fail!("unterminated literal", start) + end + + def escape_sequence(start) + advance + fail!("unterminated escape", start) if eof? + value = byte + advance + return "\n" if value == 110 + return "\t" if value == 116 + return "\r" if value == 114 + return "\f" if value == 102 + return "\a" if value == 97 + return "\e" if value == 101 + return value.chr if value == 92 || value == 34 || value == 39 + value.chr + end + + def regexp_or_operator(start) + return operator_or_punctuation(start) if @previous == :keyword_def + return operator_or_punctuation(start) if @previous == :tSYMBEG + if @context.begin_expression + advance + value = read_regexp(start) + content = value.is_a?(Array) ? value : [[:tSTRING_CONTENT, value]] + @pending = content + [[:tREGEXP_END, nil]] + return [:tREGEXP_BEG, nil] + end + operator_or_punctuation(start) + end + + def read_regexp(start) + result = +"" + tokens = [] + interpolated = false + in_class = false + until eof? + value = byte + if value == 92 + result << value.chr + advance + fail!("unterminated regexp", start) if eof? + result << byte.chr + advance + elsif value == 91 + in_class = true + result << value.chr + advance + elsif value == 93 + in_class = false + result << value.chr + advance + elsif value == 35 && byte(1) == 123 && !in_class + tokens << [:tSTRING_CONTENT, result] unless result.empty? + result = +"" + interpolated = true + advance(2) + expression = read_interpolation_source(start) + inner = self.class.new(expression, filename: @filename).each.to_a + inner.pop if inner.last == [0, nil] + tokens << [:tSTRING_DBEG, nil] + tokens.concat(inner) + tokens << [:tSTRING_DEND, nil] + elsif value == 47 && !in_class + advance + advance while byte && byte.between?(97, 122) + tokens << [:tSTRING_CONTENT, result] unless result.empty? + return interpolated ? tokens : result + else + result << value.chr + advance + end + end + fail!("unterminated regexp", start) + end + + def percent_token(start) + literal_kind = [113, 81, 119, 87, 105, 73, 114, 115, 120, 88].include?(byte(1)) + command_argument = start.positive? && [9, 32].include?(@source.getbyte(start - 1)) && + [:tIDENTIFIER, :tCONSTANT, :tFID].include?(@previous) && + ![:tIDENTIFIER, :tCONSTANT, :tFID].include?(@previous_previous) + shorthand_delimiter = byte(1) && !identifier_byte?(byte(1)) && + ![9, 10, 11, 12, 13, 32, 61].include?(byte(1)) + return operator_or_punctuation(start) unless @context.begin_expression || + ((literal_kind || shorthand_delimiter) && command_argument) + advance + kind = byte + if [113, 81, 119, 87, 105, 73, 114, 115, 120, 88].include?(kind) + advance + else + kind = 81 + end + delimiter = byte + fail!("invalid percent literal", start) unless delimiter + advance + closing = { 40 => 41, 91 => 93, 123 => 125, 60 => 62 }.fetch(delimiter, delimiter) + interpolate = [81, 87, 73, 114, 120, 88].include?(kind) + content = interpolate ? + read_interpolated_delimited(closing, start) : + read_delimited(closing, start, interpolate: interpolate) + advance while kind == 114 && byte && byte.between?(97, 122) + token = { 113 => :tSTRING_BEG, 81 => :tSTRING_BEG, 119 => :tWORDS_BEG, 87 => :tQWORDS_BEG, + 105 => :tSYMBOLS_BEG, 73 => :tQSYMBOLS_BEG, 114 => :tREGEXP_BEG, + 115 => :tSYMBEG, 120 => :tXSTRING_BEG, 88 => :tXSTRING_BEG }.fetch(kind) + terminator = kind == 114 ? :tREGEXP_END : :tSTRING_END + @pending = if [119, 87, 105, 73].include?(kind) + content = content.map { |token, value| value.to_s }.join if content.is_a?(Array) + word_tokens(content, symbols: [105, 73].include?(kind)) + [[terminator, nil]] + elsif content.is_a?(Array) + content + [[terminator, nil]] + else + [[:tSTRING_CONTENT, content], [terminator, nil]] + end + [token, nil] + end + + def word_tokens(content, symbols:) + words = content.split(/[\t\n\f\r ]+/).reject(&:empty?) + tokens = [[" ", nil]] + words.each do |word| + tokens << [:tSTRING_CONTENT, symbols ? word.to_sym : word] + tokens << [" ", nil] + end + tokens + end + + def read_delimited(closing, start, interpolate: false) + result = +"" + depth = 0 + until eof? + value = byte + if value == 92 + result << escape_sequence(start) + elsif value == 35 && interpolate && byte(1) == 123 + fail!("string interpolation cannot be represented by the AST", start) + elsif value == closing && depth.zero? + advance + return result + else + depth += 1 if value == ({ 41 => 40, 93 => 91, 125 => 123, 62 => 60 }.fetch(closing, -1)) + depth -= 1 if value == closing && depth.positive? + result << value.chr + advance + end + end + fail!("unterminated percent literal", start) + end + + def read_interpolated_delimited(closing, start) + result = +"" + tokens = [] + depth = 0 + opening = { 41 => 40, 93 => 91, 125 => 123, 62 => 60 }.fetch(closing, -1) + until eof? + value = byte + if value == 92 + result << escape_sequence(start) + elsif value == 35 && byte(1) == 123 + tokens << [:tSTRING_CONTENT, result] unless result.empty? + result = +"" + advance(2) + expression = read_interpolation_source(start) + inner = self.class.new(expression, filename: @filename).each.to_a + inner.pop if inner.last == [0, nil] + tokens << [:tSTRING_DBEG, nil] + tokens.concat(inner) + tokens << [:tSTRING_DEND, nil] + elsif value == closing && depth.zero? + advance + tokens << [:tSTRING_CONTENT, result] unless result.empty? + return tokens + else + depth += 1 if value == opening + depth -= 1 if value == closing && depth.positive? + result << value.chr + advance + end + end + fail!("unterminated percent literal", start) + end + + def interpolated_content_tokens(source, start) + return [[:tSTRING_CONTENT, source]] unless source.include?("#" + "{") + tokens = [] + literal = +"" + index = 0 + while index < source.bytesize + if source.getbyte(index) == 35 && source.getbyte(index + 1) == 123 + tokens << [:tSTRING_CONTENT, literal] unless literal.empty? + depth = 1 + expr_start = index + 2 + index = expr_start + while index < source.bytesize && depth.positive? + depth += 1 if source.getbyte(index) == 123 + depth -= 1 if source.getbyte(index) == 125 + index += 1 + end + fail!("unterminated string interpolation", start) unless depth.zero? + expression = source.byteslice(expr_start, index - expr_start - 1) + inner = self.class.new(expression, filename: @filename).each.to_a + inner.pop if inner.last == [0, nil] + tokens << [:tSTRING_DBEG, nil] + tokens.concat(inner) + tokens << [:tSTRING_DEND, nil] + literal = +"" + else + literal << source.getbyte(index).chr + index += 1 + end + end + tokens << [:tSTRING_CONTENT, literal] unless literal.empty? + tokens + end + + def heredoc_or_operator(start) + if byte(1) == 60 && heredoc_prefix? + return heredoc_token(start) if @context.begin_expression || @previous == :tIDENTIFIER + end + operator_or_punctuation(start) + end + + def heredoc_prefix? + value = byte(2) + value && ([39, 34, 96, 45, 126].include?(value) || identifier_byte?(value)) + end + + def heredoc_token(start) + advance(2) + indent = byte == 45 || byte == 126 + squiggly = byte == 126 + advance if indent + quote = byte + if quote == 39 || quote == 34 || quote == 96 + advance + delimiter_start = @index + advance while byte && byte != quote + fail!("unterminated heredoc identifier", start) if eof? + delimiter = @source.byteslice(delimiter_start, @index - delimiter_start) + advance + else + delimiter_start = @index + advance while identifier_byte?(byte) + delimiter = @source.byteslice(delimiter_start, @index - delimiter_start) + end + fail!("invalid heredoc identifier", start) if delimiter.empty? + suffix_start = @index + advance while byte && byte != 10 + suffix = @source.byteslice(suffix_start, @index - suffix_start).force_encoding(Encoding::UTF_8) + advance if byte == 10 + interpolate = quote != 39 + body = read_heredoc_body(delimiter, indent, squiggly, start) + content_tokens = interpolate ? interpolated_content_tokens(body, start) : [[:tSTRING_CONTENT, body]] + @pending = content_tokens + [[:tSTRING_END, nil]] + prefix, headers, suffix = extract_heredoc_headers(suffix) + unless prefix.empty? + prefix_tokens = heredoc_suffix_tokens(prefix) + @pending.concat(prefix_tokens) + end + @heredoc_queue.concat(headers) + @deferred_suffix = suffix unless suffix.empty? + @heredoc_newline_pending = true + [quote == 96 ? :tXSTRING_BEG : :tSTRING_BEG, nil] + end + + def read_heredoc_body(delimiter, indent, squiggly, start) + body = +"" + loop do + line_start = @index + advance while !eof? && byte != 10 + line = @source.byteslice(line_start, @index - line_start) + line_without_cr = line.delete_suffix("\r") + terminator = indent ? line_without_cr.sub(/\A[ \t]*/, "") : line_without_cr + if terminator == delimiter + advance if byte == 10 + break + end + body << line << "\n" + advance if byte == 10 + fail!("unterminated heredoc", start) if eof? + end + squiggly ? dedent_heredoc(body) : body + end + + def extract_heredoc_headers(suffix) + pattern = /<<(\-|~)?([A-Za-z_][A-Za-z0-9_]*)/ + match = pattern.match(suffix) + return [suffix, [], ""] unless match + + headers = [{delimiter: match[2], indent: !match[1].nil?, squiggly: match[1] == "~", quote: nil}] + trailing = suffix[match.end(0)..].to_s.gsub(pattern) do + headers << {delimiter: Regexp.last_match(2), indent: !Regexp.last_match(1).nil?, squiggly: Regexp.last_match(1) == "~", quote: nil} + "" + end + [suffix[0...match.begin(0)], headers, trailing] + end + + def heredoc_suffix_tokens(source) + suffix_lexer = self.class.new(source, filename: @filename) + suffix_lexer.context.begin_expression = false + suffix_lexer.context.lex_state = :expr_end + suffix_lexer.instance_variable_set(:@previous, :tSTRING_END) + tokens = suffix_lexer.each.to_a + tokens.pop if tokens.last == [0, nil] + if [:keyword_if, :keyword_unless, :keyword_while, :keyword_until, :keyword_rescue].include?(tokens.first&.first) + token = {keyword_if: :modifier_if, keyword_unless: :modifier_unless, + keyword_while: :modifier_while, keyword_until: :modifier_until, + keyword_rescue: :modifier_rescue}.fetch(tokens.first.first) + tokens[0] = [token, tokens.first.last] + end + tokens + end + + def load_queued_heredoc + descriptor = @heredoc_queue.shift + body = read_heredoc_body(descriptor.fetch(:delimiter), descriptor.fetch(:indent), descriptor.fetch(:squiggly), @index) + content = descriptor.fetch(:quote) == 39 ? [[:tSTRING_CONTENT, body]] : interpolated_content_tokens(body, @index) + @pending.concat([descriptor.fetch(:quote) == 96 ? [:tXSTRING_BEG, nil] : [:tSTRING_BEG, nil], *content, [:tSTRING_END, nil]]) + end + + def dedent_heredoc(body) + indents = body.lines.filter_map do |line| + next if line.strip.empty? + line[/\A[ \t]*/].bytesize + end + return body if indents.empty? + + width = indents.min + body.lines.map { |line| line.sub(/\A[ \t]{0,#{width}}/, "") }.join + end + + def operator_or_punctuation(start) + if @previous == :keyword_def && @source.getbyte(@index) == 38 + advance + return ["&", nil] + end + if @previous == :tSYMBEG && ["-@", "+@"].include?(@source.byteslice(@index, 2)) + value = @source.byteslice(@index, 2) + advance(2) + return [value == "-@" ? :tUMINUS : :tUPLUS, nil] + end + text = OPERATORS.sort_by { |operator| -operator.bytesize }.find do |operator| + next false if ["[]", "[]="].include?(operator) && start.positive? && + [9, 10, 11, 12, 13, 32].include?(@source.getbyte(start - 1)) && + ![:keyword_def, :keyword_alias].include?(@previous) && + @previous_previous != :keyword_alias + next false if ["[]", "[]="].include?(operator) && + [:tIDENTIFIER, :tCONSTANT, :tFID, :tINTEGER, :tSTRING_END, :tREGEXP_END].include?(@previous) && + @previous_previous != :keyword_alias + @source.byteslice(@index, operator.bytesize) == operator + end + operator_method = [:keyword_def, :keyword_alias, ".", :tCOLON2, :tANDDOT, :tSYMBEG].include?(@previous) || + @previous_previous == :keyword_alias + if text && !(begin_expression? && !operator_method && ["[]", "[]="].include?(text)) + advance(text.bytesize) + token = if text == "**" && @context.begin_expression + :tDSTAR + elsif text == ".." && @context.begin_expression + :tBDOT2 + elsif text == "..." && (@context.begin_expression || [:tLPAREN, "(", ","].include?(@previous)) + :tBDOT3 + elsif text == "::" && (@context.begin_expression || (start.positive? && [9, 32].include?(@source.getbyte(start - 1)))) + :tCOLON3 + elsif text == "&" && @previous == :keyword_def + "&" + else + OP_TOKENS.fetch(text, :tOP_ASGN) + end + @context.lambda_pending = true if token == :tLAMBDA + return [token, nil] + end + value = byte.chr + advance + if value == "(" || value == "[" || value == "{" + brace_block = value == "{" && @previous != "(" && + ([:tIDENTIFIER, :tCONSTANT, :tFID].include?(@previous) || + (@previous == ")" && !(@previous_previous == :tSTAR && @context.scope_stack.last == :method)) || + [".", :tCOLON2].include?(@previous_previous) || + [:keyword_super, :keyword_yield].include?(@previous) || + [:proc, :lambda].include?(@previous_value)) + lambda_block = value == "{" && @context.lambda_pending + command_arg = value == "(" && !@context.begin_expression && start.positive? && [9, 32].include?(@source.getbyte(start - 1)) && + [:tIDENTIFIER, :tCONSTANT, :tFID].include?(@previous) + unless brace_block || lambda_block + @context.push_delimiter(value) + @context.push_cmdarg(command_arg) + end + if value == "(" + if command_arg + return [:tLPAREN_ARG, nil] + end + method_definition = @previous == :keyword_def || @previous_previous == :keyword_def + return [@context.begin_expression && !method_definition && + ![".", :tCOLON2, :tANDDOT, :keyword_super, :keyword_yield, :tLAMBDA, :tAREF].include?(@previous) ? :tLPAREN : "(", nil] + end + if value == "{" && @context.lambda_pending + @context.lambda_pending = false + return [:tLAMBEG, nil] + end + array_argument = value == "[" && start.positive? && + [9, 10, 11, 12, 13, 32].include?(@source.getbyte(start - 1)) && + [:tIDENTIFIER, :tFID, :tCONSTANT].include?(@previous) + array_argument ||= value == "[" && @previous == ")" && + @previous_previous == "(" && @context.scope_stack.last == :method + array_argument ||= value == "[" && @context.method_body_start + return [value == "[" && (@context.begin_expression || array_argument) ? :tLBRACK : (value == "[" ? "[" : (brace_block ? "{" : :tLBRACE)), nil] + elsif value == ")" || value == "]" || value == "}" + opener = { ")" => "(", "]" => "[", "}" => "{" }.fetch(value) + @context.pop_delimiter(opener) + @context.pop_cmdarg + if value == ")" && @context.method_header + @context.method_body_start = true + @context.method_header = false + end + elsif value == "-" && @context.begin_expression + return [:tUMINUS, nil] + elsif value == "+" && @context.begin_expression + return [:tUPLUS, nil] + elsif value == "*" && @context.begin_expression + return [:tSTAR, nil] + elsif value == "&" && @previous == :tSYMBEG + return [value, nil] + elsif value == "&" && @context.begin_expression + return [:tAMPER, nil] + elsif value == "?" && !@context.begin_expression + @context.ternary_depth += 1 + elsif value == ":" && @context.ternary_depth.positive? + @context.ternary_depth -= 1 + elsif value == "<" && @previous == :tCONSTANT && @previous_previous == :keyword_class + @class_superclass = true + end + [value, nil] + rescue EncodingError + fail!("invalid byte", start) + end + + def begin_expression? + @context.begin_expression + end + end + private_constant :Lexer + end + end + end +end diff --git a/lib/lrama/ruby/languages/ruby/parse.y b/lib/lrama/ruby/languages/ruby/parse.y index 84415c8..00a26f2 100644 --- a/lib/lrama/ruby/languages/ruby/parse.y +++ b/lib/lrama/ruby/languages/ruby/parse.y @@ -183,6 +183,7 @@ %right tUMINUS tUMINUS_NUM %right tPOW %right tUPLUS '!' '~' +%left keyword_do_block %start program %% /* upstream parse.y:3176: $@1: %empty */ @@ -374,7 +375,7 @@ block_command: block_call { $$ = $1 }; /* upstream parse.y:3574: block_command: block_call call_op2 operation2 command_args */ block_command: block_call call_op2 operation2 command_args { $$ = $1 }; /* upstream parse.y:3581: cmd_brace_block: "{ arg" brace_body '}' */ -cmd_brace_block: tLBRACE_ARG brace_body '}' %prec '}' { $$ = $1 }; +cmd_brace_block: tLBRACE_ARG brace_body '}' %prec '}' { $$ = $2 }; /* upstream parse.y:3589: fcall: "local variable or method" */ fcall: tIDENTIFIER %prec tIDENTIFIER { $$ = $1 }; /* upstream parse.y:3589: fcall: "constant" */ @@ -384,17 +385,17 @@ fcall: tFID %prec tFID { $$ = $1 }; /* upstream parse.y:3596: command: fcall command_args */ command: fcall command_args %prec tLOWEST { $$ = @builder.call($1, $2 || []) }; /* upstream parse.y:3603: command: fcall command_args cmd_brace_block */ -command: fcall command_args cmd_brace_block { $$ = $1 }; +command: fcall command_args cmd_brace_block { $$ = @builder.block_call(@builder.call($1, $2 || []), $3) }; /* upstream parse.y:3612: command: primary_value call_op operation2 command_args */ -command: primary_value call_op operation2 command_args %prec tLOWEST { $$ = $1 }; +command: primary_value call_op operation2 command_args %prec tLOWEST { $$ = @builder.receiver_call($1, $2, $3, $4) }; /* upstream parse.y:3617: command: primary_value call_op operation2 command_args cmd_brace_block */ -command: primary_value call_op operation2 command_args cmd_brace_block { $$ = $1 }; +command: primary_value call_op operation2 command_args cmd_brace_block { $$ = @builder.block_call(@builder.receiver_call($1, $2, $3, $4), $5) }; /* upstream parse.y:3622: command: primary_value "::" operation2 command_args */ -command: primary_value tCOLON2 operation2 command_args %prec tLOWEST { $$ = $1 }; +command: primary_value tCOLON2 operation2 command_args %prec tLOWEST { $$ = @builder.receiver_call($1, :"::", $3, $4) }; /* upstream parse.y:3627: command: primary_value "::" operation2 command_args cmd_brace_block */ -command: primary_value tCOLON2 operation2 command_args cmd_brace_block %prec tCOLON2 { $$ = $1 }; +command: primary_value tCOLON2 operation2 command_args cmd_brace_block %prec tCOLON2 { $$ = @builder.block_call(@builder.receiver_call($1, :"::", $3, $4), $5) }; /* upstream parse.y:3632: command: primary_value "::" "constant" '{' brace_body '}' */ -command: primary_value tCOLON2 tCONSTANT '{' brace_body '}' %prec '}' { $$ = $1 }; +command: primary_value tCOLON2 tCONSTANT '{' brace_body '}' %prec '}' { $$ = @builder.block_call(@builder.receiver_call($1, :"::", $3, []), $5) }; /* upstream parse.y:3638: command: "'super'" command_args */ command: keyword_super command_args %prec keyword_super { $$ = @builder.call(:super, $2 || []) }; /* upstream parse.y:3644: command: k_yield command_args */ @@ -416,15 +417,15 @@ mlhs_inner: tLPAREN mlhs_inner rparen %prec tLPAREN { $$ = $2 }; /* upstream parse.y:3687: mlhs_basic: mlhs_head */ mlhs_basic: mlhs_head { $$ = $1 }; /* upstream parse.y:3692: mlhs_basic: mlhs_head mlhs_item */ -mlhs_basic: mlhs_head mlhs_item { $$ = ($1 + [$2]).freeze }; +mlhs_basic: mlhs_head mlhs_item { $$ = (Array($1) + [$2]).freeze }; /* upstream parse.y:3697: mlhs_basic: mlhs_head "*" mlhs_node */ -mlhs_basic: mlhs_head tSTAR mlhs_node %prec tSTAR { $$ = ($1 + [$3]).freeze }; +mlhs_basic: mlhs_head tSTAR mlhs_node %prec tSTAR { $$ = (Array($1) + [$3]).freeze }; /* upstream parse.y:3048: mlhs_items_mlhs_item: mlhs_item */ mlhs_items_mlhs_item: mlhs_item { $$ = $1 }; /* upstream parse.y:3053: mlhs_items_mlhs_item: mlhs_items_mlhs_item ',' mlhs_item */ -mlhs_items_mlhs_item: mlhs_items_mlhs_item ',' mlhs_item %prec ',' { $$ = ($1 + [$3]).freeze }; +mlhs_items_mlhs_item: mlhs_items_mlhs_item ',' mlhs_item %prec ',' { $$ = (Array($1) + [$3]).freeze }; /* upstream parse.y:3702: mlhs_basic: mlhs_head "*" mlhs_node ',' mlhs_items_mlhs_item */ -mlhs_basic: mlhs_head tSTAR mlhs_node ',' mlhs_items_mlhs_item %prec ',' { $$ = ($1 + [$3] + $5).freeze }; +mlhs_basic: mlhs_head tSTAR mlhs_node ',' mlhs_items_mlhs_item %prec ',' { $$ = (Array($1) + [$3] + Array($5)).freeze }; /* upstream parse.y:3707: mlhs_basic: mlhs_head "*" */ mlhs_basic: mlhs_head tSTAR %prec tSTAR { $$ = $1 }; /* upstream parse.y:3712: mlhs_basic: mlhs_head "*" ',' mlhs_items_mlhs_item */ @@ -490,7 +491,7 @@ cpath: tCOLON3 cname %prec tCOLON3 { $$ = $2 }; /* upstream parse.y:3853: cpath: cname */ cpath: cname { $$ = $1 }; /* upstream parse.y:3858: cpath: primary_value "::" cname */ -cpath: primary_value tCOLON2 cname %prec tCOLON2 { $$ = $2 }; +cpath: primary_value tCOLON2 cname %prec tCOLON2 { $$ = $3 }; /* upstream parse.y:3864: fname: "local variable or method" */ fname: tIDENTIFIER %prec tIDENTIFIER { $$ = $1 }; /* upstream parse.y:3864: fname: "constant" */ @@ -814,7 +815,7 @@ opt_call_args: args ',' assocs ',' %prec ',' { $$ = $1 }; /* upstream parse.y:4212: opt_call_args: assocs ',' */ opt_call_args: assocs ',' %prec ',' { $$ = $1 }; /* upstream parse.y:3161: value_expr_command: command */ -value_expr_command: command { $$ = $1 }; +value_expr_command: command %prec tLBRACE_ARG { $$ = $1 }; /* upstream parse.y:4219: call_args: value_expr_command */ call_args: value_expr_command { $$ = $1 }; /* upstream parse.y:4224: call_args: def_endless_method_endless_command */ @@ -844,11 +845,11 @@ args: arg_value { $$ = [$1].freeze }; /* upstream parse.y:4319: args: arg_splat */ args: arg_splat { $$ = $1 }; /* upstream parse.y:4324: args: args ',' arg_value */ -args: args ',' arg_value %prec ',' { $$ = ($1 + [$3]).freeze }; +args: args ',' arg_value %prec ',' { $$ = @builder.append_arguments($1, $3) }; /* upstream parse.y:4329: args: args ',' arg_splat */ -args: args ',' arg_splat %prec ',' { $$ = ($1 + [$3]).freeze }; +args: args ',' arg_splat %prec ',' { $$ = @builder.append_arguments($1, $3) }; /* upstream parse.y:4337: arg_splat: "*" arg_value */ -arg_splat: tSTAR arg_value %prec tSTAR { $$ = $2 }; +arg_splat: tSTAR arg_value %prec tSTAR { $$ = [$2].freeze }; /* upstream parse.y:4342: arg_splat: "*" */ arg_splat: tSTAR %prec tSTAR { $$ = nil }; /* upstream parse.y:4350: mrhs_arg: mrhs */ @@ -916,11 +917,11 @@ primary: keyword_not '(' expr rparen %prec '(' { $$ = @builder.unary(:"!", $3) } /* upstream parse.y:4470: primary: "'not'" '(' rparen */ primary: keyword_not '(' rparen %prec '(' { $$ = @builder.reserved_word("not") }; /* upstream parse.y:4475: primary: fcall brace_block */ -primary: fcall brace_block { $$ = $1 }; +primary: fcall brace_block { $$ = @builder.block_call(@builder.call($1, []), $2) }; /* upstream parse.y:4479: primary: method_call */ primary: method_call { $$ = $1 }; /* upstream parse.y:4481: primary: method_call brace_block */ -primary: method_call brace_block { $$ = $1 }; +primary: method_call brace_block { $$ = @builder.block_call($1, $2) }; /* upstream parse.y:4486: primary: lambda */ primary: lambda { $$ = $1 }; /* upstream parse.y:4491: primary: k_if expr_value then compstmt_stmts if_tail k_end */ @@ -1188,7 +1189,7 @@ lambda_body: keyword_do_LAMBDA midrule_28 bodystmt k_end %prec keyword_do_LAMBDA /* upstream parse.y:5153: do_block: k_do_block do_body k_end */ do_block: k_do_block do_body k_end { $$ = $2 }; /* upstream parse.y:5161: block_call: command do_block */ -block_call: command do_block { $$ = $1 }; +block_call: command do_block { $$ = @builder.block_call($1, $2) }; /* upstream parse.y:5167: block_call: block_call call_op2 operation2 opt_paren_args */ block_call: block_call call_op2 operation2 opt_paren_args { $$ = $1 }; /* upstream parse.y:5177: block_call: block_call call_op2 operation2 opt_paren_args brace_block */ @@ -1200,13 +1201,13 @@ block_call: block_call call_op2 paren_args { $$ = $1 }; /* upstream parse.y:5196: method_call: fcall paren_args */ method_call: fcall paren_args { $$ = @builder.call($1, $2 || []) }; /* upstream parse.y:5203: method_call: primary_value call_op operation2 opt_paren_args */ -method_call: primary_value call_op operation2 opt_paren_args { $$ = $1 }; +method_call: primary_value call_op operation2 opt_paren_args { $$ = @builder.receiver_call($1, $2, $3, $4) }; /* upstream parse.y:5214: method_call: primary_value "::" operation2 paren_args */ -method_call: primary_value tCOLON2 operation2 paren_args %prec tCOLON2 { $$ = $1 }; +method_call: primary_value tCOLON2 operation2 paren_args %prec tCOLON2 { $$ = @builder.receiver_call($1, :"::", $3, $4) }; /* upstream parse.y:5220: method_call: primary_value "::" operation3 */ -method_call: primary_value tCOLON2 operation3 %prec tCOLON2 { $$ = $1 }; +method_call: primary_value tCOLON2 operation3 %prec tCOLON2 { $$ = @builder.receiver_call($1, :"::", $3, []) }; /* upstream parse.y:5225: method_call: primary_value call_op2 paren_args */ -method_call: primary_value call_op2 paren_args { $$ = $1 }; +method_call: primary_value call_op2 paren_args { $$ = @builder.receiver_call($1, $2, :call, $3) }; /* upstream parse.y:5231: method_call: "'super'" paren_args */ method_call: keyword_super paren_args %prec keyword_super { $$ = @builder.call(:super, $2 || []) }; /* upstream parse.y:5241: method_call: "'super'" */ @@ -1542,7 +1543,7 @@ midrule_37: %empty { $$ = nil }; /* upstream parse.y:6074: @38: %empty */ midrule_38: %empty { $$ = nil }; /* upstream parse.y:6079: string_content: "'#{'" @35 @36 @37 @38 compstmt_stmts string_dend */ -string_content: tSTRING_DBEG midrule_35 midrule_36 midrule_37 midrule_38 compstmt_stmts string_dend %prec tSTRING_DBEG { $$ = $7 }; +string_content: tSTRING_DBEG midrule_35 midrule_36 midrule_37 midrule_38 compstmt_stmts string_dend %prec tSTRING_DBEG { $$ = @builder.interpolation($6) }; /* upstream parse.y:6094: string_dend: "'}'" */ string_dend: tSTRING_DEND %prec tSTRING_DEND { $$ = nil }; /* upstream parse.y:6095: string_dend: "end-of-input" */ @@ -1562,7 +1563,7 @@ sym: fname { $$ = $1 }; /* upstream parse.y:6127: sym: nonlocal_var */ sym: nonlocal_var { $$ = $1 }; /* upstream parse.y:6131: dsym: "symbol literal" string_contents "terminator" */ -dsym: tSYMBEG string_contents tSTRING_END %prec tSTRING_END { $$ = $1 }; +dsym: tSYMBEG string_contents tSTRING_END %prec tSTRING_END { $$ = @builder.symbol(@builder.join_strings($2)) }; /* upstream parse.y:6138: numeric: simple_numeric */ numeric: simple_numeric { $$ = $1 }; /* upstream parse.y:6140: numeric: tUMINUS_NUM simple_numeric */ @@ -1796,9 +1797,9 @@ restarg_mark: '*' %prec '*' { $$ = nil }; /* upstream parse.y:6508: restarg_mark: "*" */ restarg_mark: tSTAR %prec tSTAR { $$ = nil }; /* upstream parse.y:6512: f_rest_arg: restarg_mark "local variable or method" */ -f_rest_arg: restarg_mark tIDENTIFIER %prec tIDENTIFIER { $$ = $2 }; +f_rest_arg: restarg_mark tIDENTIFIER %prec tIDENTIFIER { $$ = [$2].freeze }; /* upstream parse.y:6518: f_rest_arg: restarg_mark */ -f_rest_arg: restarg_mark { $$ = $1 }; +f_rest_arg: restarg_mark { $$ = [] }; /* upstream parse.y:6525: blkarg_mark: '&' */ blkarg_mark: '&' %prec '&' { $$ = nil }; /* upstream parse.y:6526: blkarg_mark: "&" */ @@ -1891,3 +1892,6 @@ terms: term { $$ = nil }; terms: terms ';' %prec ';' { $$ = nil }; /* upstream parse.y:6728: none: %empty */ none: %empty { $$ = nil }; +/* arpaka extension: block call as a parenthesized argument */ +call_args: block_call { $$ = [$1].freeze }; +call_args: args ',' block_call %prec tLOWEST { $$ = ($1 + [$3]).freeze }; diff --git a/lib/lrama/ruby/languages/ruby/rules.json b/lib/lrama/ruby/languages/ruby/rules.json index a86029c..09fa3e2 100644 --- a/lib/lrama/ruby/languages/ruby/rules.json +++ b/lib/lrama/ruby/languages/ruby/rules.json @@ -567,7 +567,7 @@ "id": 95, "rule": "cmd_brace_block: \"{ arg\" brace_body '}'", "line": 3581, - "status": "pass_through" + "status": "ported" }, { "id": 96, @@ -597,37 +597,37 @@ "id": 100, "rule": "command: fcall command_args cmd_brace_block", "line": 3603, - "status": "pass_through" + "status": "ported" }, { "id": 101, "rule": "command: primary_value call_op operation2 command_args", "line": 3612, - "status": "pass_through" + "status": "ported" }, { "id": 102, "rule": "command: primary_value call_op operation2 command_args cmd_brace_block", "line": 3617, - "status": "pass_through" + "status": "ported" }, { "id": 103, "rule": "command: primary_value \"::\" operation2 command_args", "line": 3622, - "status": "pass_through" + "status": "ported" }, { "id": 104, "rule": "command: primary_value \"::\" operation2 command_args cmd_brace_block", "line": 3627, - "status": "pass_through" + "status": "ported" }, { "id": 105, "rule": "command: primary_value \"::\" \"constant\" '{' brace_body '}'", "line": 3632, - "status": "pass_through" + "status": "ported" }, { "id": 106, @@ -1989,7 +1989,7 @@ "id": 332, "rule": "arg_splat: \"*\" arg_value", "line": 4337, - "status": "pass_through" + "status": "ported" }, { "id": 333, @@ -2193,7 +2193,7 @@ "id": 366, "rule": "primary: fcall brace_block", "line": 4475, - "status": "pass_through" + "status": "ported" }, { "id": 367, @@ -2205,7 +2205,7 @@ "id": 368, "rule": "primary: method_call brace_block", "line": 4481, - "status": "pass_through" + "status": "ported" }, { "id": 369, @@ -3009,7 +3009,7 @@ "id": 502, "rule": "block_call: command do_block", "line": 5161, - "status": "pass_through" + "status": "ported" }, { "id": 503, @@ -3045,25 +3045,25 @@ "id": 508, "rule": "method_call: primary_value call_op operation2 opt_paren_args", "line": 5203, - "status": "pass_through" + "status": "ported" }, { "id": 509, "rule": "method_call: primary_value \"::\" operation2 paren_args", "line": 5214, - "status": "pass_through" + "status": "ported" }, { "id": 510, "rule": "method_call: primary_value \"::\" operation3", "line": 5220, - "status": "pass_through" + "status": "ported" }, { "id": 511, "rule": "method_call: primary_value call_op2 paren_args", "line": 5225, - "status": "pass_through" + "status": "ported" }, { "id": 512, @@ -4071,7 +4071,7 @@ "id": 679, "rule": "string_content: \"'#{'\" @35 @36 @37 @38 compstmt_stmts string_dend", "line": 6079, - "status": "pass_through" + "status": "ported" }, { "id": 680, @@ -4131,7 +4131,7 @@ "id": 689, "rule": "dsym: \"symbol literal\" string_contents \"terminator\"", "line": 6131, - "status": "pass_through" + "status": "ported" }, { "id": 690, @@ -4833,13 +4833,13 @@ "id": 806, "rule": "f_rest_arg: restarg_mark \"local variable or method\"", "line": 6512, - "status": "pass_through" + "status": "ported" }, { "id": 807, "rule": "f_rest_arg: restarg_mark", "line": 6518, - "status": "pass_through" + "status": "empty" }, { "id": 808, diff --git a/lib/lrama/ruby/parser.rb.erb b/lib/lrama/ruby/parser.rb.erb index c8c10a2..a4bd26d 100644 --- a/lib/lrama/ruby/parser.rb.erb +++ b/lib/lrama/ruby/parser.rb.erb @@ -25,15 +25,22 @@ class <%= class_name %> YYPACT_NINF = <%= context.yypact_ninf %> YYTABLE_NINF = <%= context.yytable_ninf %> + attr_accessor :lexical_context + attr_reader :parse_trace, :lookahead_token, :lookahead_value, :lookahead_state + # tokens yields [token name or numeric ID, semantic value] pairs. # Exhaustion is EOF; [0, nil] can also terminate the stream explicitly. def parse(tokens) + @parse_trace = [] input = tokens.to_enum states = [0] values = [] lookahead = nil token = nil value = nil + @lookahead_token = nil + @lookahead_value = nil + @lookahead_state = nil while true state = states.last @@ -52,6 +59,10 @@ class <%= class_name %> raise ArgumentError, "Each token must be a [name or ID, value] pair" end token, value = pair + @parse_trace << [:read, token, state].freeze + @lookahead_token = token + @lookahead_value = value + @lookahead_state = state id = token.is_a?(Integer) ? token : TOKENS[token.to_s] lookahead = id && id >= 0 ? (YYTRANSLATE[id] || 2) : 2 # The reserved error symbol is internal to recovery, never lexer input. @@ -67,9 +78,14 @@ class <%= class_name %> rule = YYDEFACT[state] raise ParseError.new(token, state) if rule.zero? elsif action.positive? + @parse_trace << [:shift, token, state, action].freeze + @lexical_context&.parser_shift(token, value, state, action) states << action values << value lookahead = nil + @lookahead_token = nil + @lookahead_value = nil + @lookahead_state = nil next elsif action.zero? || action == YYTABLE_NINF raise ParseError.new(token, state) @@ -78,6 +94,8 @@ class <%= class_name %> end length = YYR2[rule] + @parse_trace << [:reduce, rule, state].freeze + @lexical_context&.parser_reduce(rule, state) result = reduce(rule, values, length) states.pop(length) values.pop(length) diff --git a/sig/arpaka.rbs b/sig/arpaka.rbs index a434f95..28bc2b9 100644 --- a/sig/arpaka.rbs +++ b/sig/arpaka.rbs @@ -1,3 +1,7 @@ module Arpaka def self.parse: (Enumerable[[String | Symbol | Integer, untyped]] tokens) -> ::Lrama::Ruby::Languages::Ruby::AST::Program + def self.parse_source: (String source, ?filename: String) -> ::Lrama::Ruby::Languages::Ruby::AST::Program + + class LexerError < ::Lrama::Ruby::Error + end end diff --git a/sig/lrama/ruby.rbs b/sig/lrama/ruby.rbs index 3576673..ca6c5f3 100644 --- a/sig/lrama/ruby.rbs +++ b/sig/lrama/ruby.rbs @@ -18,6 +18,7 @@ module Lrama module Languages module Ruby def self.parse: (Enumerable[[String | Symbol | Integer, untyped]] tokens) -> AST::Program + def self.parse_source: (String source, ?filename: String) -> AST::Program class ParseError < ::Lrama::Ruby::Error attr_reader token: untyped @@ -31,8 +32,11 @@ module Lrama def initialize: (String rule, Integer line) -> void end + class LexerError < ::Lrama::Ruby::Error + end + module AST - type node = Literal | Binary | Unary | If | LocalRead | LocalWrite | BareCall | ArrayLiteral | HashLiteral | RangeLiteral | Ternary | Call | StringLiteral | RegexpLiteral | XStringLiteral | Index | Loop | Control | Variable | Rescue | Case | When | For | Def | ClassDef | ModuleDef | Lambda + type node = Literal | Binary | Unary | If | LocalRead | LocalWrite | BareCall | ArrayLiteral | HashLiteral | RangeLiteral | Ternary | Call | ReceiverCall | BlockCall | StringLiteral | InterpolatedString | RegexpLiteral | XStringLiteral | Index | Loop | Control | Variable | Rescue | Case | When | For | Def | ClassDef | ModuleDef | Lambda type hash_pair = Pair class Program < ::Data @@ -131,12 +135,34 @@ module Lrama | (name: Symbol, arguments: Array[node]) -> instance end + class ReceiverCall < ::Data + attr_reader receiver: node + attr_reader operator: :"." | :"::" | :"&." + attr_reader name: Symbol + attr_reader arguments: Array[node] + def self.new: (node receiver, (:"." | :"::" | :"&.") operator, Symbol name, Array[node] arguments) -> instance + | (receiver: node, operator: (:"." | :"::" | :"&."), name: Symbol, arguments: Array[node]) -> instance + end + + class BlockCall < ::Data + attr_reader call: node + attr_reader body: Array[node] + def self.new: (node call, Array[node] body) -> instance + | (call: node, body: Array[node]) -> instance + end + class StringLiteral < ::Data attr_reader value: String def self.new: (String value) -> instance | (value: String) -> instance end + class InterpolatedString < ::Data + attr_reader parts: Array[node | String] + def self.new: (Array[node | String] parts) -> instance + | (parts: Array[node | String]) -> instance + end + class RegexpLiteral < ::Data attr_reader value: String def self.new: (String value) -> instance diff --git a/test/lrama/action_scanner_test.rb b/test/lrama/action_scanner_test.rb new file mode 100644 index 0000000..69520fd --- /dev/null +++ b/test/lrama/action_scanner_test.rb @@ -0,0 +1,167 @@ +# frozen_string_literal: true + +require "test_helper" +require "lrama/ruby/action_scanner" + +class ActionScannerTest < Test::Unit::TestCase + Scanner = Lrama::Ruby::ActionScanner + + def scan(source, **options) + Scanner.new(source).scan(**options) + end + + def reference_texts(source) + scan(source).references.map { |ref| source.byteslice(ref.first_column...ref.last_column) } + end + + test "action stops before its closing brace and preserves byte offsets" do + source = "text = \"日本語\"; $$ = $12 + $left } following grammar {" + result = scan(source, action: true) + assert_equal(source.b.index("} following"), result.end_offset) + assert_equal(["$$", "$12", "$left"], result.references.map { |r| source.byteslice(r.first_column...r.last_column) }) + assert_equal([nil, 12, nil], result.references.map(&:number)) + assert_equal(["$", nil, "left"], result.references.map(&:name)) + end + + test "strings comments and variable names do not become references" do + source = <<~'RUBY' + @foo = @@foo + # } $comment + =begin + } $comment + =end + a = '$single }' + b = "$double } \"" + c = `echo '$backtick }'` + $$ = $real + RUBY + assert_equal(["$$", "$real"], reference_texts(source)) + end + + test "nested interpolation and shorthand references are distinguished" do + source = <<~'RUBY' + $$ = "#{ { value: "#{$1}" } } #$left #@foo #@@foo \#$ignored" + RUBY + refs = scan(source).references + assert_equal(["$$", "$1", "$left"], reference_texts(source)) + assert_equal([false, false, true], refs.map(&:short_interpolation)) + end + + test "percent literal nesting and interpolation follow literal type" do + %w[q w i s].each do |kind| + source = "%#{kind}{nested { } $ignored \#{ $ignored } }" + assert_empty(scan(source).references, kind) + end + %w[Q W I x].each do |kind| + source = "%#{kind}{nested { } $ignored \#{ $1 } }" + assert_equal(["$1"], reference_texts(source), kind) + end + ["%{\#{ $1 }}", "%Q(\#{ $1 })", "%Q[\#{ $1 }]", "%Q<\#{ $1 }>", "%Q!\#{ $1 }!"].each do |source| + assert_equal(["$1"], reference_texts(source)) + end + end + + test "regexp classes escapes and interpolation preserve action boundaries" do + sources = [ + '/[}$]/', '/[[:alpha:]}]/', '/[]}$]/', '/[\]}/]/', + '%r{[}]\#{ignored}}', '/foo\/bar}/', '/\#{ { a: $1 } }/' + ] + sources.each do |source| + result = scan(source + " } next", action: true) + assert_equal(source.bytesize + 1, result.end_offset, source) + end + assert_equal(["$1"], reference_texts('/#{ { a: $1 } }/')) + assert_equal(["$1"], reference_texts('%r!#{ $1 }!im')) + end + + test "division modulo shifts and explicit calls are supported" do + ["$1 / $2", "$1 % $2", "$1 << $2", "(value) / 2", "(value) % 2", "(value) << 2", + "call(%r!}! )", "obj./(2)", "obj.%(2)", "obj.<<(2)"].each do |source| + assert_nothing_raised(source) { scan(source) } + end + end + + test "ordinary Ruby declarations and operators are supported" do + ["value / 2", "value % 2", "value << 2", "def value; end", "alias other value", "undef value"].each do |source| + assert_nothing_raised(source) { scan(source) } + end + end + + test "the lexer can scan its own Ruby source" do + [__FILE__, File.expand_path("../../lib/lrama/ruby/action_scanner.rb", __dir__)].each do |path| + assert_nothing_raised(path) { scan(File.binread(path)) } + end + end + + test "heredoc references are ordered by physical source position" do + source = <<~'RUBY' + $$ = [<(x) { { x: $2 } }] + RUBY + assert_equal(["$1", "$2"], reference_texts(source)) + end + + test "unsupported references fail in code and interpolation only" do + %w[$0 $01 $& $~ $? $! $-w $1 $[name] $1name $:].each do |ref| + assert_raise(Scanner::Error, ref) { scan(ref + " }", action: true) } + assert_raise(Scanner::Error, ref) { scan('"#{' + ref + '}" }', action: true) } + assert_empty(scan("'#{ref}'").references) + end + end + + test "unclosed constructs and mismatched brackets report source positions" do + ["'text", '"text', '%q{text', '/[abc/', '"#{ $1', "([)]", "(1", "$$ = <(body) { body }\n Mime[:html]\nend") + end + end + + test "heredoc suffixes retain literal expression context" do + assert_nothing_raised do + Arpaka.parse_source("value = <<~TEXT unless condition\n body\nTEXT\n") + end + end + + test "singleton class endings preserve enclosing blocks" do + source = <<~RUBY + tests(Class.new(ActionController::Base) do + class << self + attr_accessor :last_parameters + end + + def parse + end + end) + RUBY + assert_nothing_raised { Arpaka.parse_source(source) } + end + + test "method chains may continue after a newline before a dot" do + assert_nothing_raised { Arpaka.parse_source("value = result\n .first\n .last") } + assert_nothing_raised do + Arpaka.parse_source("value = result\n .first\n # keep chaining\n .last") + end + end + + test "conditional endings preserve enclosing blocks" do + assert_nothing_raised do + Arpaka.parse_source("include(Module.new do\n define_method(:value) do\n if condition\n value\n else\n other\n end\n end\nend)") + end + end + + test "method endings preserve enclosing blocks" do + source = <<~RUBY + values = Array.new(3, Class.new do + def create; end + def update; end + end) + RUBY + assert_nothing_raised { Arpaka.parse_source(source) } + end + + test "negative command arguments use a block do token" do + assert_nothing_raised { Arpaka.parse_source('assert_difference "x", -(value.count) do; value; end') } + end + + test "operator method definitions do not start literals" do + assert_nothing_raised { Arpaka.parse_source("def %(other); end\ndef /(other); end") } + end + + test "slash and percent symbols do not start literals" do + assert_nothing_raised { Arpaka.parse_source("calculate(:/, value)\ncalculate(:%, value)") } + end + + test "operator symbols end an alias expression" do + assert_nothing_raised { Arpaka.parse_source("alias_method :regular_writer, :[]=\nunless condition\nend") } + end + + test "ampersand symbols end an expression" do + assert_nothing_raised { Arpaka.parse_source("name == :&\nname\n") } + end + + test "Rails trick syntax remains parseable" do + assert_nothing_raised do + Arpaka.parse_source(<<~RUBY) + def source_location + if line_number + "on line" + else + "in " + end + file_name + end + RUBY + Arpaka.parse_source(<<~RUBY) + helper(Module.new do + def render_from_helper + from_test_case(suffix: "!") + end + end) + RUBY + Arpaka.parse_source(<<~RUBY) + def exec_queries + super do |record| + set_inverse(record) + yield record if block_given? + end + end + RUBY + Arpaka.parse_source(<<~RUBY) + result + .first + # keep chaining + .last + RUBY + Arpaka.parse_source(<<~RUBY) + def create_table(table_name, **options) + if block_given? + super { |table| yield compatible_table_definition(table) } + else + super + end + end + RUBY + end + end + + test "source lexer recognizes lambda as a lambda expression" do + assert_nothing_raised { Arpaka.parse_source("value = lambda do\n work\nend\n") } + end + + test "method bodies may start with an array literal after arguments" do + assert_nothing_raised { Arpaka.parse_source("def multi(a, b, c) [a, b, c] end") } + end + + test "singleton method definitions may parenthesize their receiver" do + assert_nothing_raised { Arpaka.parse_source("def (object.foo).bar(value)\n value\nend") } + end + test "if, unless, elsif and else build conditional AST nodes" do expected = AST::Program.new([AST::If.new(literal(true), [literal(1)], [literal(2)])]) assert_equal(expected, parse([:keyword_if, nil], [:keyword_true, nil], [:keyword_then, nil], [:tINTEGER, 1], [:keyword_else, nil], [:tINTEGER, 2], [:keyword_end, nil])) @@ -347,4 +513,595 @@ def literal(value) assert_raise(Arpaka::ParseError) { Arpaka.parse(tokens) } end end + + test "source input uses the Ruby lexer and preserves token parse results" do + expected = AST::Program.new([ + AST::LocalWrite.new(:a, literal(1)), + AST::Binary.new(:+, AST::LocalRead.new(:a), literal(2)) + ]) + assert_equal(expected, Arpaka.parse_source("a = 1\na + 2", filename: "example.rb")) + assert_equal(AST::Program.new([AST::Call.new(:f, [literal(1), literal(2)])]), + Arpaka.parse_source("f(1, 2)")) + assert_equal(AST::Program.new([AST::StringLiteral.new("hello")]), + Arpaka.parse_source('"hello"')) + end + + test "source lexer errors include source position" do + error = assert_raise(Arpaka::LexerError) { Arpaka.parse_source('"unterminated', filename: "broken.rb") } + assert_include(error.message, "broken.rb:1:") + end + + test "source lexer preserves literal kinds" do + assert_equal(AST::XStringLiteral.new("echo"), Arpaka.parse_source("`echo`").statements.first) + assert_equal(AST::RegexpLiteral.new("a[b]"), Arpaka.parse_source("/a[b]/").statements.first) + assert_equal(AST::StringLiteral.new("quiet"), Arpaka.parse_source("%q(quiet)").statements.first) + assert_equal(AST::ArrayLiteral.new([AST::StringLiteral.new("one"), AST::StringLiteral.new("two")]), + Arpaka.parse_source("%w(one two)").statements.first) + assert_equal(AST::ArrayLiteral.new([:one, :two]), + Arpaka.parse_source("%i(one two)").statements.first) + assert_equal(AST::RegexpLiteral.new("a"), Arpaka.parse_source("%r(a)").statements.first) + end + + test "source lexer preserves simple string interpolation" do + assert_equal(AST::InterpolatedString.new([ + "hello ", AST::Binary.new(:+, literal(1), literal(2)), " world" + ]), Arpaka.parse_source('"hello #{1 + 2} world"').statements.first) + assert_equal(AST::InterpolatedString.new([AST::BareCall.new(:name)]), + Arpaka.parse_source('"#{name}"').statements.first) + end + + test "source lexer preserves interpolation in percent strings" do + expected = AST::InterpolatedString.new(["hello ", AST::BareCall.new(:name)]) + assert_equal(expected, Arpaka.parse_source('%Q(hello #{name})').statements.first) + assert_equal(expected, Arpaka.parse_source('%(hello #{name})').statements.first) + assert_kind_of(AST::RegexpLiteral, Arpaka.parse_source('%r(#{x})').statements.first) + end + + test "source lexer handles heredoc indentation and line endings" do + assert_equal(AST::StringLiteral.new("one\n two\n"), + Arpaka.parse_source("<<~TEXT\n one\n two\n TEXT\n").statements.first) + assert_equal(AST::StringLiteral.new(" one\n"), + Arpaka.parse_source("<<-TEXT\n one\n TEXT\n").statements.first) + assert_equal(AST::StringLiteral.new("one\r\ntwo\r\n"), + Arpaka.parse_source("< { 1 }; end") + Lrama::Ruby::Languages::Ruby.parse(scope_lexer.each, lexical_context: scope_lexer.context) + assert_empty(scope_lexer.context.parser_scope_stack) + pretokenized = Arpaka.const_get(:Lexer, false).new("f(1)") + tokens = pretokenized.each.to_a + Lrama::Ruby::Languages::Ruby.parse(tokens) + assert_empty(pretokenized.context.parser_events) + end + + test "source lexer enters fname state for definitions and aliases" do + lexer = Arpaka.const_get(:Lexer, false).new("def value=; end") + lexer.each.to_a + def_name = lexer.context.state_history.find do |entry| + entry.fetch(:token) == :tFID + end + assert_equal(:expr_fname, def_name.fetch(:lex_state)) + + lexer = Arpaka.const_get(:Lexer, false).new("alias eql? ==") + lexer.each.to_a + alias_name = lexer.context.state_history.find do |entry| + entry.fetch(:token) == :tFID + end + assert_equal(:expr_fname, alias_name.fetch(:lex_state)) + end + + test "source lexer distinguishes label and labeled states" do + lexer = Arpaka.const_get(:Lexer, false).new("f at: 1") + lexer.each.to_a + states = lexer.context.state_history + assert_equal(:expr_label, states.find { |entry| entry.fetch(:token) == :tLABEL }.fetch(:lex_state)) + assert_equal(:expr_labeled, states.find { |entry| entry.fetch(:token) == :tINTEGER }.fetch(:lex_state)) + end + + test "source lexer treats spaced empty brackets as an array literal" do + tokens = Arpaka.const_get(:Lexer, false).new("assert_equal [], value").each.to_a + assert_equal([:tIDENTIFIER, :tLBRACK, "]", ",", :tIDENTIFIER, 0], tokens.map(&:first)) + end + + test "source lexer treats spaced bracket arguments as array literals" do + tree = Arpaka.parse_source("foo [1], name: 2").statements.first + assert_equal(AST::ArrayLiteral.new([literal(1)]), tree.arguments.first) + assert_equal(AST::Pair.new(AST::Literal.new(:name), literal(2)), tree.arguments.last) + end + + test "source lexer treats predicate and bang methods as identifiers" do + assert_equal(AST::Def.new(:stopping?, [], [literal(1)]), + Arpaka.parse_source("def stopping?; 1; end").statements.first) + assert_equal(AST::Def.new(:halt!, [], [literal(1)]), + Arpaka.parse_source("def halt!; 1; end").statements.first) + assert_nothing_raised { Arpaka.parse_source("value.match?(pattern)") } + end + + test "source lexer handles contextual punctuation in Ruby expressions" do + tokens = Arpaka.const_get(:Lexer, false).new("f(foo?: true)").each.to_a + assert_equal([:tIDENTIFIER, "(", :tLABEL, :keyword_true, ")", 0], tokens.map(&:first)) + assert_nothing_raised { Arpaka.parse_source("f(foo?: true)") } + + tokens = Arpaka.const_get(:Lexer, false).new("list_tables[]").each.to_a + assert_equal([:tIDENTIFIER, "[", "]", 0], tokens.map(&:first)) + assert_kind_of(AST::Index, Arpaka.parse_source("list_tables[]").statements.first) + + assert_nothing_raised { Arpaka.parse_source("x = if y; 'a'; else; 'b'; end + z") } + assert_nothing_raised { Arpaka.parse_source("helper.({a: 1})") } + assert_nothing_raised { Arpaka.parse_source("assert ?h.in?(\"hello\")") } + end + + test "source lexer uses regular brace blocks for calls and proc values" do + ["foo { 1 }", "[proc { 1 }]", "foo { 1 }.first", "foo { 1 } || bar"].each do |source| + tokens = Arpaka.const_get(:Lexer, false).new(source).each.to_a + assert_not_include(tokens.map(&:first), :tAMPER, source) + assert_nothing_raised { Arpaka.parse_source(source) } + end + + assert_nothing_raised { Arpaka.parse_source("f(&block)") } + assert_nothing_raised { Arpaka.parse_source("map(&:to_s)") } + assert_nothing_raised { Arpaka.parse_source("a & b") } + end + + test "source lexer keeps definition and block contexts for operators" do + assert_nothing_raised { Arpaka.parse_source("def ==(x); self.x == x; end") } + assert_nothing_raised { Arpaka.parse_source("def []=(x); x; end") } + assert_nothing_raised { Arpaka.parse_source("x = -> *args do; 1; end") } + assert_nothing_raised { Arpaka.parse_source("def f; super do |x| x end; end") } + assert_nothing_raised { Arpaka.parse_source("def f; super { |x| x }; end") } + end + + test "source lexer distinguishes splat and block argument operators" do + assert_equal(AST::Call.new(:f, [AST::BareCall.new(:args)]), + Arpaka.parse_source("f(*args)").statements.first) + assert_equal(AST::Call.new(:map, AST::Literal.new(:to_s)), + Arpaka.parse_source("map(&:to_s)").statements.first) + assert_nothing_raised { Arpaka.parse_source("f(1, *args)") } + end + + test "symbols can be built from variable-shaped values" do + builder = Lrama::Ruby::Languages::Ruby.const_get(:Builder, false).new + assert_equal(AST::Literal.new(:value), builder.symbol(AST::Variable.new(:local, :value))) + end + + test "source lexer accepts argumentless command blocks" do + assert_equal(AST::BlockCall.new(AST::Call.new(:included, []), [literal(1)]), + Arpaka.parse_source("included do\n 1\nend").statements.first) + assert_equal(AST::BlockCall.new(AST::ReceiverCall.new(AST::BareCall.new(:foo), :".", :bar, []), [literal(1)]), + Arpaka.parse_source("foo.bar do\n 1\nend").statements.first) + assert_nothing_raised { Arpaka.parse_source("foo before: :bar do |value| value end\n") } + tokens = Arpaka.const_get(:Lexer, false).new("items << lambda do").each.to_a + assert_equal([:tIDENTIFIER, :tLSHFT, :tIDENTIFIER, :keyword_do], tokens[0, 4].map(&:first)) + assert_nothing_raised { Arpaka.parse_source("items << lambda do\n work\nend\n") } + end + + test "source lexer recognizes shorthand percent literals in command arguments" do + tokens = Arpaka.const_get(:Lexer, false).new("assert %(value)\n").each.to_a + assert_equal([:tIDENTIFIER, :tSTRING_BEG, :tSTRING_CONTENT, :tSTRING_END], tokens[0, 4].map(&:first)) + assert_equal(AST::StringLiteral.new("value"), Arpaka.parse_source("assert %(value)\n").statements.first.arguments.first) + end + + test "source lexer recognizes beginless ranges" do + tokens = Arpaka.const_get(:Lexer, false).new("f in: ..range_end\n").each.to_a + assert_equal(:tBDOT2, tokens.map(&:first)[2]) + assert_nothing_raised { Arpaka.parse_source("f in: ..range_end\n") } + end + + test "source lexer keeps special global variables intact" do + tokens = Arpaka.const_get(:Lexer, false).new(%q{"pid #{$$}"\n}).each.to_a + assert_equal([:tSTRING_BEG, :tSTRING_CONTENT, :tSTRING_DBEG, :tGVAR, :tSTRING_DEND, :tSTRING_END], + tokens[0, 6].map(&:first)) + assert_equal(:"$$", tokens[3][1]) + end + + test "source lexer keeps the load path global intact" do + tokens = Arpaka.const_get(:Lexer, false).new("$:.unshift(\"lib\")\n").each.to_a + assert_equal(:tGVAR, tokens.first.first) + assert_equal(:"$:", tokens.first.last) + assert_nothing_raised { Arpaka.parse_source("$:.unshift(\"lib\")\n") } + end + + test "source lexer treats keyword-shaped method names as fname tokens" do + tokens = Arpaka.const_get(:Lexer, false).new("def then(&block)\nend\n").each.to_a + assert_equal(:tFID, tokens[1].first) + assert_nothing_raised { Arpaka.parse_source("def then(&block)\nend\n") } + end + + test "source lexer preserves newlines before terminators" do + assert_nothing_raised { Arpaka.parse_source("def empty\nend\n") } + assert_nothing_raised { Arpaka.parse_source("if condition\nend\n") } + assert_nothing_raised { Arpaka.parse_source("foo do\nend\n") } + end + + test "source lexer preserves block newlines inside calls" do + source = "Module.new {\n define_method(:up) { yield(:up); super() }\n define_method(:down) { yield(:down); super() }\n}\n" + assert_nothing_raised { Arpaka.parse_source(source) } + assert_nothing_raised { Arpaka.parse_source("Module.new do\n define_method(:up) do\n 1\n end\n define_method(:down) do\n 2\n end\nend\n") } + end + + test "source lexer keeps block newlines at their owning delimiter" do + source = "f(-> {\n if condition\n first\n else\n second\n end\n})\n" + assert_nothing_raised { Arpaka.parse_source(source) } + + nested = "f {\n g(\n first,\n second\n )\n}\n" + assert_nothing_raised { Arpaka.parse_source(nested) } + end + + test "source lexer ignores newlines after operator assignments" do + assert_nothing_raised { Arpaka.parse_source("value ||=\n if condition\n fallback\n end\n") } + end + + test "source lexer uses ordinary do after parenthesized calls" do + tokens = Arpaka.const_get(:Lexer, false).new("each(1) do\nend").each.to_a + assert_equal(:keyword_do, tokens.map(&:first)[-4]) + assert_equal(AST::BlockCall.new(AST::Call.new(:each, [literal(1)]), []), + Arpaka.parse_source("each(1) do\nend").statements.first) + end + + test "source lexer does not carry conditional do across statements" do + tokens = Arpaka.const_get(:Lexer, false).new("while condition\n body\nend\nfoo do\nend\n").each.to_a + assert_equal(:keyword_do, tokens.map(&:first)[-5]) + end + + test "source lexer balances conditional context" do + lexer = Arpaka.const_get(:Lexer, false).new("while condition do; body; end") + tokens = lexer.each.to_a + assert_includes(tokens.map(&:first), :keyword_do_cond) + assert_empty(lexer.context.condition_stack) + + lexer = Arpaka.const_get(:Lexer, false).new("until condition\nbody\nend") + lexer.each.to_a + assert_empty(lexer.context.condition_stack) + end + + test "grammar precedence accepts block calls as parenthesized arguments" do + tree = Arpaka.parse_source("call(lambda do\nend)\n") + assert_equal(AST::Call.new(:call, [AST::BlockCall.new(AST::Call.new(:lambda, []), [])]), tree.statements.first) + + assert_nothing_raised do + Arpaka.parse_source("subscribe(x, handler, lambda do\n work\nend)\n") + end + end + + test "source lexer uses brace blocks after parenthesized calls" do + tokens = Arpaka.const_get(:Lexer, false).new("each(1) { |value| value }\n").each.to_a + assert_equal("{", tokens.map(&:first)[4]) + assert_nothing_raised { Arpaka.parse_source("each(1) { |value| value }\n") } + end + + test "source lexer uses brace blocks after receiver method calls" do + tokens = Arpaka.const_get(:Lexer, false).new("mutex.synchronize { 1 }\n").each.to_a + assert_equal("{", tokens.map(&:first)[3]) + assert_nothing_raised { Arpaka.parse_source("mutex.synchronize { 1 }\n") } + assert_nothing_raised { Arpaka.parse_source("mutex.synchronize {\n first\n second\n}\n") } + end + + test "source lexer uses brace blocks after calls inside arguments" do + tokens = Arpaka.const_get(:Lexer, false).new("outer(1, inner { 2 })\n").each.to_a + assert_equal("{", tokens.map(&:first)[5]) + assert_nothing_raised { Arpaka.parse_source("outer(1, inner { 2 })\n") } + end + + test "source lexer chains methods after assignment brace blocks" do + tokens = Arpaka.const_get(:Lexer, false).new("value = call { 1 }.dup\n").each.to_a + assert_equal("{", tokens.map(&:first)[3]) + assert_nothing_raised { Arpaka.parse_source("value = call { 1 }.dup\n") } + end + + test "source lexer uses regular brace blocks inside call arguments" do + tokens = Arpaka.const_get(:Lexer, false).new("outer(call { 1 })\n").each.to_a + assert_equal("{", tokens.map(&:first)[3]) + assert_nothing_raised { Arpaka.parse_source("outer(call { 1 })\n") } + end + + test "source lexer starts one-line method bodies with array literals" do + tokens = Arpaka.const_get(:Lexer, false).new("def values() [1] end\n").each.to_a + assert_equal(:tLBRACK, tokens.map(&:first)[4]) + assert_nothing_raised { Arpaka.parse_source("def values() [1] end\n") } + end + + test "source lexer uses regular brace blocks after ternary branches" do + tokens = Arpaka.const_get(:Lexer, false).new("touch ? callback { 1 } : fallback\n").each.to_a + assert_equal("{", tokens.map(&:first)[3]) + assert_nothing_raised { Arpaka.parse_source("touch ? callback { 1 } : fallback\n") } + end + + test "source lexer preserves newlines inside command blocks" do + assert_nothing_raised do + Arpaka.parse_source("app = lambda { |env|\n req = Request.new(env)\n res = response(req)\n}\n") + end + end + + test "source lexer recognizes lambda bodies after lambda arguments" do + tokens = Arpaka.const_get(:Lexer, false).new("->(value) { value }\n").each.to_a + assert_includes(tokens.map(&:first), :tLAMBEG) + assert_nothing_raised { Arpaka.parse_source("->(value) { value }\n") } + assert_nothing_raised { Arpaka.parse_source("-> {\n first\n second\n}\n") } + assert_nothing_raised { Arpaka.parse_source("f(&lambda { |value| value })\n") } + end + + test "source lexer ignores newlines after logical operators" do + assert_nothing_raised { Arpaka.parse_source("left &&\nright\n") } + assert_nothing_raised { Arpaka.parse_source("left ||\nright\n") } + end + + test "source lexer recognizes modifiers after argumentless control keywords" do + tokens = Arpaka.const_get(:Lexer, false).new("return unless value\n").each.to_a + assert_equal(:modifier_unless, tokens[1].first) + assert_nothing_raised { Arpaka.parse_source("def stop\n return unless value\nend\n") } + end + + test "source lexer recognizes modifiers after block endings" do + tokens = Arpaka.const_get(:Lexer, false).new("foo do\nend unless value\n").each.to_a + assert_equal(:modifier_unless, tokens[4].first) + assert_nothing_raised { Arpaka.parse_source("foo do\nend unless value\n") } + end + + test "source lexer accepts escaped newlines between expressions" do + tokens = Arpaka.const_get(:Lexer, false).new("\"left\" \\\n\"right\"\n").each.to_a + assert_equal([:tSTRING_BEG, :tSTRING_CONTENT, :tSTRING_END, :tSTRING_BEG], tokens.map(&:first)[0, 4]) + assert_nothing_raised { Arpaka.parse_source("\"left\" \\\n\"right\"\n") } + end + + test "source lexer continues method chains after newlines" do + assert_nothing_raised { Arpaka.parse_source("value\n .first\n .to_s\n") } + end + + test "source lexer ignores newlines before block parameters" do + assert_nothing_raised { Arpaka.parse_source("call {\n |value| value\n}\n") } + end + + test "source lexer continues keyword arguments after label newlines" do + assert_nothing_raised do + Arpaka.parse_source("parse(file, context:\n build_context)\n") + end + end + + test "source lexer recognizes string hash labels" do + assert_nothing_raised { Arpaka.parse_source("{ \"key\": value }\n") } + end + + test "source lexer recognizes unary operator symbols" do + assert_nothing_raised { Arpaka.parse_source("alias :-@ :deduplicate\n") } + assert_nothing_raised { Arpaka.parse_source("alias_method :push, :<<\nalias_method :append, :<<\n") } + end + + test "source lexer recognizes operator method names and top-level constants" do + assert_equal(AST::Def.new(:[], :x, [AST::BareCall.new(:x)]), + Arpaka.parse_source("def [](x); x; end").statements.first) + assert_nothing_raised { Arpaka.parse_source("def /(other); other; end") } + assert_nothing_raised { Arpaka.parse_source("def &(other); other; end") } + assert_equal(:Foo, Arpaka.parse_source("::Foo").statements.first) + end + + test "source lexer preserves alias boundaries and symbol statement endings" do + tokens = Arpaka.const_get(:Lexer, false).new("alias foo= bar").each.to_a + assert_equal([:keyword_alias, :tFID, :tIDENTIFIER, 0], tokens.map(&:first)) + assert_nothing_raised { Arpaka.parse_source("alias foo= bar") } + + source = "alias_method :eql?, :==\ndef hash; 1; end" + tokens = Arpaka.const_get(:Lexer, false).new(source).each.to_a + assert_include(tokens.map(&:first), "\n") + assert_nothing_raised { Arpaka.parse_source(source) } + + assert_nothing_raised { Arpaka.parse_source("undef_method :==, :!, :!=\ndef hash; 1; end") } + assert_nothing_raised { Arpaka.parse_source("alias [] get") } + assert_nothing_raised { Arpaka.parse_source("alias []= get") } + assert_nothing_raised { Arpaka.parse_source("alias :eql? :==\n\ndef hash; 1; end") } + end + + test "source lexer allows keyword-shaped receiver setters" do + tokens = Arpaka.const_get(:Lexer, false).new("node.case = value").each.to_a + assert_equal([:tIDENTIFIER, ".", :tIDENTIFIER, "=", :tIDENTIFIER, 0], tokens.map(&:first)) + assert_nothing_raised { Arpaka.parse_source("node.case = value") } + end + + test "source lexer accepts symbols after predicate-like identifiers" do + assert_nothing_raised { Arpaka.parse_source("alias :merge! :update\n") } + assert_nothing_raised { Arpaka.parse_source("alias :default_options= :default\n") } + assert_nothing_raised { Arpaka.parse_source("block_given? ? yield : @default_render.call\n") } + end + + test "source lexer accepts interpolated symbols" do + assert_nothing_raised { Arpaka.parse_source(':"#{name}_settings"') } + assert_nothing_raised { Arpaka.parse_source("super(:/, left, right)") } + end + + test "source lexer scans slashes inside regexp interpolation" do + source = '/#{value.sub(/x/, "")}/' + tokens = Arpaka.const_get(:Lexer, false).new(source).each.to_a + assert_equal([:tREGEXP_BEG, :tSTRING_DBEG, :tIDENTIFIER, ".", :tIDENTIFIER, + "(", :tREGEXP_BEG, :tSTRING_CONTENT, :tREGEXP_END, ",", :tSTRING_BEG, + :tSTRING_END, ")", :tSTRING_DEND, :tREGEXP_END, 0], tokens.map(&:first)) + assert_kind_of(AST::RegexpLiteral, Arpaka.parse_source(source).statements.first) + end + + test "source lexer accepts ampersand operator symbols" do + assert_nothing_raised { Arpaka.parse_source("items.reduce(:&)\n") } + assert_nothing_raised { Arpaka.parse_source("delegate :[], :[]=, to: :paths\n") } + assert_nothing_raised { Arpaka.parse_source("left &\n right\n") } + assert_nothing_raised { Arpaka.parse_source("left |\n right\n") } + assert_nothing_raised { Arpaka.parse_source("left ^\n right\n") } + assert_nothing_raised { Arpaka.parse_source("foo *args\n") } + assert_nothing_raised { Arpaka.parse_source("foo **options\n") } + assert_nothing_raised { Arpaka.parse_source("first..\n last\n") } + assert_nothing_raised { Arpaka.parse_source("left and\n right\n") } + assert_nothing_raised { Arpaka.parse_source("left or\n right\n") } + assert_nothing_raised { Arpaka.parse_source("left ==\n right\n") } + assert_nothing_raised { Arpaka.parse_source("not\n value\n") } + assert_nothing_raised { Arpaka.parse_source("fn = ->\n { 1 }\n") } + end + + test "source lexer recognizes bitwise assignment operators" do + %w[|= &= ^=].each do |operator| + assert_nothing_raised { Arpaka.parse_source("value #{operator} other\n") } + end + end + + test "source lexer recognizes class variables" do + assert_nothing_raised { Arpaka.parse_source("@@templates = {}\n@@templates\n") } + end + + test "source lexer recognizes punctuation global variables" do + assert_nothing_raised { Arpaka.parse_source("-$`\n") } + assert_nothing_raised { Arpaka.parse_source("$&\n") } + end + + test "source lexer parses parenthesized yield arguments" do + assert_nothing_raised { Arpaka.parse_source("def each\n yield(value, other)\nend\n") } + end + + test "source lexer distinguishes modulo from percent literals" do + assert_nothing_raised { Arpaka.parse_source("value = (left + right) % 3\n") } + assert_nothing_raised { Arpaka.parse_source("value = %q(text)\n") } + assert_nothing_raised { Arpaka.parse_source("value = %r{pattern}i\n") } + end + + test "source lexer accepts percent literals as command arguments" do + assert_nothing_raised { Arpaka.parse_source("def f; assert_match %r{a}, value; end") } + assert_nothing_raised { Arpaka.parse_source("def f; assert_equal %w[a b], value; end") } + assert_nothing_raised { Arpaka.parse_source("test do; assert_equal %w[a b], value; end") } + end + + test "source lexer keeps expression-ending keywords and globals together" do + assert_nothing_raised { Arpaka.parse_source("test do; f __LINE__ + 1; end") } + assert_nothing_raised { Arpaka.parse_source("module M; def f; yield if true; end; end") } + assert_nothing_raised { Arpaka.parse_source("class C; def f; super if true; end; end") } + assert_nothing_raised { Arpaka.parse_source("$?.exitstatus") } + end + + test "source lexer distinguishes command argument delimiters and blocks" do + assert_nothing_raised { Arpaka.parse_source("f ::Time, value") } + assert_nothing_raised { Arpaka.parse_source("def f; assert_equal (count * 2) - 1, total; end") } + assert_nothing_raised { Arpaka.parse_source("value = { xml: lambda { 1 } }") } + assert_nothing_raised { Arpaka.parse_source("f(:x, proc { 1 })") } + assert_nothing_raised { Arpaka.parse_source("f :x, lambda { 1 }") } + assert_nothing_raised { Arpaka.parse_source("-> arg do; arg; end") } + assert_nothing_raised { Arpaka.parse_source("test do; f only: A::B do; 1; end; end") } + assert_nothing_raised { Arpaka.parse_source("f at: 30.days.from_now do; 1; end") } + assert_nothing_raised { Arpaka.parse_source("travel_to Time.now + 3.seconds do; 1; end") } + assert_nothing_raised { Arpaka.parse_source("value = left || proc { 1 }") } + end + + test "source lexer accepts lambda argument defaults" do + assert_nothing_raised { Arpaka.parse_source("def [](x, y = nil); end") } + assert_nothing_raised { Arpaka.parse_source("->(x = nil) { x }") } + end + + test "source lexer ignores pending heredoc newlines in delimiters" do + assert_nothing_raised { Arpaka.parse_source("f({a: <<~A,\nx\nA\nb: 1})") } + assert_nothing_raised { Arpaka.parse_source("f unless\n predicate") } + assert_nothing_raised { Arpaka.parse_source("f(\nproc { 1 }\n)\n") } + end + + test "source lexer terminates aliases whose target is an operator" do + assert_nothing_raised { Arpaka.parse_source("class C\n alias eql? ==\n def hash\n 1\n end\nend\n") } + end + + test "single quoted heredoc keeps interpolation literal" do + assert_equal(AST::StringLiteral.new("\#{x}\n"), + Arpaka.parse_source("<<'TEXT'\n\#{x}\nTEXT\n").statements.first) + end end diff --git a/test/lrama/ruby_receiver_call_test.rb b/test/lrama/ruby_receiver_call_test.rb new file mode 100644 index 0000000..5a6cb4f --- /dev/null +++ b/test/lrama/ruby_receiver_call_test.rb @@ -0,0 +1,107 @@ +# frozen_string_literal: true + +require "test_helper" + +class RubyReceiverCallTest < Test::Unit::TestCase + AST = Arpaka::AST + + test "receiver calls retain their operator, method name and ordered arguments" do + [".", "::", "&."].each do |operator| + ["object#{operator}fetch(1, 2)", "object#{operator}fetch 1, 2"].each do |source| + tree = Arpaka.parse_source(source).statements.fetch(0) + assert_equal(AST::ReceiverCall.new(AST::BareCall.new(:object), operator.to_sym, + :fetch, [AST::Literal.new(1), AST::Literal.new(2)]), tree, source) + assert_predicate(tree.arguments, :frozen?) + end + end + end + + test "parenthesized invocation without a method name means call" do + [".", "::", "&."].each do |operator| + source = "handler#{operator}(message)" + expected = AST::ReceiverCall.new(AST::BareCall.new(:handler), operator.to_sym, + :call, [AST::BareCall.new(:message)]) + assert_equal(AST::Program.new([expected]), Arpaka.parse_source(source), source) + end + end + + test "safe navigation allows keyword and operator method names" do + ["class", "if", "[]"].each do |name| + source = "object&.#{name}(1)" + expected = AST::ReceiverCall.new(AST::BareCall.new(:object), :"&.", + name.to_sym, [AST::Literal.new(1)]) + assert_equal(AST::Program.new([expected]), Arpaka.parse_source(source), source) + end + end + + test "receiver call trees agree with RubyVM for representative Rails expressions" do + sources = [ + "self.class.to_s", "object&.class", "object&.if(1)", "object&.[](1)", + "logger.warn \"deprecated\"", "File.join(\"a\", \"b\")", + "File::join(\"a\", \"b\")", "File::join \"a\", \"b\"", "Object::name", + "handler.(message)", "handler::(message)", "handler&.(message)", + "object.fetch(1, transform(2))", "object&.fetch 1, 2", + "object.fetch(1).to_s", "object&.fetch(1)&.to_s", "object.ready?" + ] + sources.each do |source| + expected = normalize_rubyvm(RubyVM::AbstractSyntaxTree.parse(source).children.fetch(2)) + actual = normalize_arpaka(Arpaka.parse_source(source).statements.fetch(0)) + assert_equal(expected, actual, source) + end + end + + private + + # This oracle deliberately covers only this test's expression slice. Unknown + # nodes fail instead of making an incomplete comparison look successful. + def normalize_rubyvm(node) + children = node.children + case node.type + when :CALL, :QCALL + receiver, name, arguments = children + [:call, normalize_rubyvm(receiver), node.type == :QCALL, name, rubyvm_arguments(arguments)] + when :FCALL + [:call, nil, false, children[0], rubyvm_arguments(children[1])] + when :VCALL + [:bare_call, children.fetch(0)] + when :CONST + [:constant, children.fetch(0)] + when :SELF + [:literal, :self] + when :STR + [:string, children.fetch(0)] + when :LIT, :INTEGER + [:literal, children.fetch(0)] + else + flunk("Unmapped RubyVM node: #{node.type}") + end + end + + def rubyvm_arguments(node) + return [] unless node + assert_equal(:LIST, node.type) + assert_nil(node.children.last) + node.children[0...-1].map { |child| normalize_rubyvm(child) } + end + + def normalize_arpaka(node) + case node + when AST::ReceiverCall + [:call, normalize_arpaka(node.receiver), node.operator == :"&.", node.name, + node.arguments.map { |argument| normalize_arpaka(argument) }] + when AST::Call + [:call, nil, false, node.name, node.arguments.map { |argument| normalize_arpaka(argument) }] + when AST::BareCall + [:bare_call, node.name] + when AST::Variable + assert_equal(:constant, node.kind) + [:constant, node.name] + when AST::StringLiteral + [:string, node.value] + when AST::Literal + [:literal, node.value] + else + flunk("Unmapped Arpaka node: #{node.inspect}") + end + end +end diff --git a/test/lrama/ruby_test.rb b/test/lrama/ruby_test.rb index 472ac64..c453b52 100644 --- a/test/lrama/ruby_test.rb +++ b/test/lrama/ruby_test.rb @@ -19,6 +19,19 @@ class Lrama::RubyTest < Test::Unit::TestCase assert_equal(20, parser.parse([["(", nil], [:NUMBER, 2], ["+", nil], [:NUMBER, 3], [")", nil], ["*", nil], [:NUMBER, 4]])) end + test "generated parsers expose lookahead and reduction traces" do + parser = compile("%token NUMBER\n%%\nstart: NUMBER;").new + assert_equal(7, parser.parse([[:NUMBER, 7]])) + assert_equal(:read, parser.parse_trace.first.first) + assert_equal(:shift, parser.parse_trace[1].first) + assert_include(parser.parse_trace.map(&:first), :reduce) + assert_equal(7, parser.parse([[:NUMBER, 7]])) + assert_not_equal(parser.parse_trace.object_id, nil) + assert_nil(parser.lookahead_token) + assert_nil(parser.lookahead_value) + assert_nil(parser.lookahead_state) + end + test "numeric token IDs, aliases, and explicit EOF" do parser = compile('%token NUMBER 300 "number"' + "\n%%\nstart: NUMBER;").new assert_equal(false, parser.parse([[300, false], [0, nil]])) diff --git a/tool/ruby/actions.rb b/tool/ruby/actions.rb index 6717b9d..2c252b4 100644 --- a/tool/ruby/actions.rb +++ b/tool/ruby/actions.rb @@ -118,7 +118,7 @@ module RubyGrammarActions 412 => ["pass_through", "nil"], 419 => ["pass_through", "nil"], 420 => ["pass_through", "nil"], 421 => ["pass_through", "nil"], 426 => ["empty", "nil"], - 95 => ["pass_through", "$1"], + 95 => ["ported", "$2"], 94 => ["pass_through", "$1"], 82 => ["pass_through", "$1"], 77 => ["pass_through", "$1"], @@ -162,8 +162,8 @@ module RubyGrammarActions 731 => ["pass_through", "$1"], 728 => ["pass_through", "$1"], 725 => ["pass_through", "$1"], - 689 => ["pass_through", "$1"], - 679 => ["pass_through", "$7"], + 689 => ["ported", "@builder.symbol(@builder.join_strings($2))"], + 679 => ["ported", "@builder.interpolation($6)"], 674 => ["pass_through", "$1"], 665 => ["pass_through", "$1"], 628 => ["pass_through", "$1"], @@ -209,15 +209,15 @@ module RubyGrammarActions 520 => ["pass_through", "$7"], 518 => ["pass_through", "$7"], 516 => ["pass_through", "$2"], - 511 => ["pass_through", "$1"], - 510 => ["pass_through", "$1"], - 509 => ["pass_through", "$1"], - 508 => ["pass_through", "$1"], + 511 => ["ported", "@builder.receiver_call($1, $2, :call, $3)"], + 510 => ["ported", "@builder.receiver_call($1, :\"::\", $3, [])"], + 509 => ["ported", "@builder.receiver_call($1, :\"::\", $3, $4)"], + 508 => ["ported", "@builder.receiver_call($1, $2, $3, $4)"], 506 => ["pass_through", "$1"], 505 => ["pass_through", "$1"], 504 => ["pass_through", "$1"], 503 => ["pass_through", "$1"], - 502 => ["pass_through", "$1"], + 502 => ["ported", "@builder.block_call($1, $2)"], 501 => ["pass_through", "$2"], 488 => ["pass_through", "$1"], 487 => ["pass_through", "$1"], @@ -254,8 +254,8 @@ module RubyGrammarActions 451 => ["pass_through", "$1"], 448 => ["pass_through", "$1"], 445 => ["pass_through", "$1"], - 368 => ["pass_through", "$1"], - 366 => ["pass_through", "$1"], + 368 => ["ported", "@builder.block_call($1, $2)"], + 366 => ["ported", "@builder.block_call(@builder.call($1, []), $2)"], 356 => ["pass_through", "$2"], 313 => ["pass_through", "$1"], 280 => ["pass_through", "$1"], @@ -275,19 +275,19 @@ module RubyGrammarActions 136 => ["pass_through", "$1"], 135 => ["pass_through", "$1"], 134 => ["pass_through", "$1"], - 105 => ["pass_through", "$1"], - 104 => ["pass_through", "$1"], - 103 => ["pass_through", "$1"], - 102 => ["pass_through", "$1"], - 100 => ["pass_through", "$1"], - 101 => ["pass_through", "$1"], + 105 => ["ported", "@builder.block_call(@builder.receiver_call($1, :\"::\", $3, []), $5)"], + 104 => ["ported", "@builder.block_call(@builder.receiver_call($1, :\"::\", $3, $4), $5)"], + 103 => ["ported", "@builder.receiver_call($1, :\"::\", $3, $4)"], + 102 => ["ported", "@builder.block_call(@builder.receiver_call($1, $2, $3, $4), $5)"], + 100 => ["ported", "@builder.block_call(@builder.call($1, $2 || []), $3)"], + 101 => ["ported", "@builder.receiver_call($1, $2, $3, $4)"], 606 => ["ported", "@builder.range(:\"...\", nil, $2)"], 605 => ["ported", "@builder.range(:\"..\", nil, $2)"], 604 => ["ported", "@builder.range(:\"...\", $1, nil)"], 810 => ["pass_through", "$2"], 820 => ["pass_through", "$3"], 828 => ["pass_through", "$4"], - 806 => ["pass_through", "$2"], + 806 => ["ported", "[$2].freeze"], 802 => ["pass_through", "$2"], 835 => ["pass_through", "$1"], 831 => ["pass_through", "$1"], @@ -351,8 +351,8 @@ module RubyGrammarActions 320 => ["ported", "(($1 || []) + ($3 || [])).freeze"], 324 => ["pass_through", "$2"], 326 => ["pass_through", "$2"], - 331 => ["ported", "($1 + [$3]).freeze"], - 332 => ["pass_through", "$2"], + 331 => ["ported", "@builder.append_arguments($1, $3)"], + 332 => ["ported", "[$2].freeze"], 336 => ["ported", "($1 + [$3]).freeze"], 337 => ["ported", "($1 + [$4]).freeze"], 338 => ["ported", "[$2].freeze"], @@ -380,10 +380,10 @@ module RubyGrammarActions 114 => ["pass_through", "$2"], 128 => ["pass_through", "$2"], 129 => ["ported", "[$1].freeze"], - 116 => ["ported", "($1 + [$2]).freeze"], - 117 => ["ported", "($1 + [$3]).freeze"], - 119 => ["ported", "($1 + [$3]).freeze"], - 120 => ["ported", "($1 + [$3] + $5).freeze"], + 116 => ["ported", "(Array($1) + [$2]).freeze"], + 117 => ["ported", "(Array($1) + [$3]).freeze"], + 119 => ["ported", "(Array($1) + [$3]).freeze"], + 120 => ["ported", "(Array($1) + [$3] + Array($5)).freeze"], 122 => ["pass_through", "$1"], 121 => ["pass_through", "$1"], 123 => ["pass_through", "$2"], @@ -523,7 +523,7 @@ module RubyGrammarActions 301 => ["ported", "($1 + $3).freeze"], 302 => ["pass_through", "$1"], 328 => ["ported", "[$1].freeze"], - 330 => ["ported", "($1 + [$3]).freeze"], + 330 => ["ported", "@builder.append_arguments($1, $3)"], 347 => ["pass_through", "$1"], 357 => ["ported", "@builder.array($2)"], 358 => ["ported", "@builder.hash($2)"], @@ -548,7 +548,7 @@ module RubyGrammarActions 425 => ["ported", "@builder.elsif_node($2, $4, $5)"], 426 => ["empty", "nil"], 445 => ["pass_through", "$1"], - 100 => ["pass_through", "$1"], 427 => ["pass_through", "$2"], + 427 => ["pass_through", "$2"], 17 => ["empty", "nil"], 18 => ["ported", "@builder.body_nodes($1)"], 79 => ["pass_through", "$1"], 80 => ["pass_through", "$2"], 407 => ["empty", "nil"], 388 => ["empty", "nil"], @@ -558,7 +558,7 @@ module RubyGrammarActions 389 => ["ported", "@builder.def_node($1, $2, @builder.body_nodes($4))"], 629 => ["empty", "nil"], 636 => ["empty", "nil"], 149 => ["pass_through", "$1"], 150 => ["pass_through", "$1"], - 151 => ["pass_through", "$2"], 152 => ["pass_through", "$1"], 153 => ["pass_through", "$2"], + 151 => ["pass_through", "$2"], 152 => ["pass_through", "$1"], 153 => ["pass_through", "$3"], 382 => ["empty", "nil"], 384 => ["empty", "nil"], 386 => ["empty", "nil"], 383 => ["ported", "@builder.class_node($2, $3, $5)"], 387 => ["ported", "@builder.module_node($2, $4)"], @@ -623,7 +623,7 @@ module RubyGrammarActions 796 => ["pass_through", "$1"], 798 => ["pass_through", "$1"], 799 => ["empty", "nil"], 800 => ["empty", "nil"], 801 => ["pass_through", "$1"], 803 => ["pass_through", "$1"], 804 => ["empty", "nil"], 805 => ["empty", "nil"], - 807 => ["pass_through", "$1"], 808 => ["empty", "nil"], 809 => ["empty", "nil"], + 807 => ["empty", "[]"], 808 => ["empty", "nil"], 809 => ["empty", "nil"], 812 => ["pass_through", "$1"], 813 => ["empty", "nil"], 814 => ["pass_through", "$1"], 815 => ["pass_through", "$1"], 816 => ["pass_through", "$1"], 817 => ["pass_through", "$1"], 818 => ["pass_through", "$1"], diff --git a/tool/ruby/grammar.rb b/tool/ruby/grammar.rb index 147d136..bdef3fb 100644 --- a/tool/ruby/grammar.rb +++ b/tool/ruby/grammar.rb @@ -11,6 +11,9 @@ module RubyGrammar ROOT = File.expand_path("../..", __dir__) DESTINATION = File.join(ROOT, "lib/lrama/ruby/languages/ruby") REVISION = "37d60dd3241d5fdb10ca43f36077f5d556ae1b8d" + EXTENDED_RULES = { + ["value_expr_command", ["command"]] => "tLBRACE_ARG" + }.freeze def self.parse(source, filename) grammar = Lrama::Parser.new(source, filename).parse @@ -82,17 +85,28 @@ def self.artifacts grammar.precedences.group_by(&:precedence).sort.each do |_level, group| lines << "%#{group.first.type} #{group.map(&:s_value).join(' ')}" end + lines << "%left keyword_do_block" lines << "%start #{name.call(grammar.rules.first.rhs.first)}" lines << "%%" metadata = inventory.to_h { |rule| [rule.fetch("id"), rule] } rules = grammar.rules.reject(&:initial_rule?) rules.each do |rule| rhs = rule.rhs.empty? ? "%empty" : rule.rhs.map(&name).join(" ") - precedence = rule.precedence_sym ? " %prec #{name.call(rule.precedence_sym)}" : "" + extended_precedence = EXTENDED_RULES[[rule.lhs.id.s_value, rule.rhs.map { |symbol| symbol.id.s_value }]] + precedence = if rule.precedence_sym + " %prec #{name.call(rule.precedence_sym)}" + elsif extended_precedence + " %prec #{extended_precedence}" + else + "" + end expression = RubyGrammarActions::ACTIONS[rule.id]&.last || "@builder.unsupported(#{rule.id})" lines << "/* upstream parse.y:#{metadata.fetch(rule.id).fetch('line')}: #{rule.as_comment} */" lines << "#{name.call(rule.lhs)}: #{rhs}#{precedence} { $$ = #{expression} };" end + lines << "/* arpaka extension: block call as a parenthesized argument */" + lines << "call_args: block_call { $$ = [$1].freeze };" + lines << "call_args: args ',' block_call %prec tLOWEST { $$ = ($1 + [$3]).freeze };" source = lines.join("\n") + "\n" verify_structure(grammar, source, names) compact = rules.map do |rule| @@ -112,11 +126,33 @@ def self.verify_structure(original, source, names) [name.call(rule.lhs), rule.rhs.map(&name), rule.precedence_sym && name.call(rule.precedence_sym)] end.sort_by(&:inspect) end - raise "Production structure changed" unless signature.call(original, names) == signature.call(port, {}) + expected_rules = signature.call(original, names).map do |rule| + if rule[0] == names.fetch("value_expr_command") && rule[1] == [names.fetch("command")] + rule[0, 2] + ["tLBRACE_ARG"] + else + rule + end + end + expected_rules << ["call_args", ["block_call"], nil] + expected_rules << ["call_args", ["args", "','", "block_call"], "tLOWEST"] + actual_rules = signature.call(port, {}) + unless expected_rules.sort_by(&:inspect) == actual_rules.sort_by(&:inspect) + raise "Production structure changed" + end terms = lambda do |grammar| grammar.terms.map { |symbol| [symbol.id.s_value, symbol.token_id, symbol.precedence&.type, symbol.precedence&.precedence] }.sort_by(&:inspect) end - raise "Tokens or precedence changed" unless terms.call(original) == terms.call(port) + expected_terms = terms.call(original).map do |term| + if term[0] == "keyword_do_block" + [term[0], term[1], :left, original.precedences.map(&:precedence).max + 1] + else + term + end + end + actual_terms = terms.call(port) + unless expected_terms.sort_by(&:inspect) == actual_terms + raise "Tokens or precedence changed: expected=#{expected_terms.sort_by(&:inspect).inspect} actual=#{actual_terms.inspect}" + end end def self.run(check:) @@ -128,7 +164,7 @@ def self.run(check:) File.write(path, contents) end end - puts(check ? "Ruby grammar verified (853 productions)" : "Ruby grammar regenerated") + puts(check ? "Ruby grammar verified (853 upstream productions + extensions)" : "Ruby grammar regenerated") end end diff --git a/tool/ruby/rails_snapshot.rb b/tool/ruby/rails_snapshot.rb new file mode 100644 index 0000000..3cd0baf --- /dev/null +++ b/tool/ruby/rails_snapshot.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +# Local-only health snapshot for the Rails checkout. This intentionally uses +# RubyVM::AbstractSyntaxTree as an oracle; it is not part of Arpaka's runtime. + +require "json" +require "lrama/ruby/action_scanner" + +rails = ARGV[0] || [ + File.expand_path("~/Program/Github/rails"), + File.expand_path("~/Program/Ruby/rails") +].find { |path| File.directory?(path) } +abort "Rails checkout not found" unless rails + +files = Dir.glob(File.join(rails, "**/*.rb"), File::FNM_EXTGLOB).sort +snapshot = { + "rails" => rails, + "files" => files.length, + "scanner_failures" => [], + "ast_failures" => [], + "scanner_successes" => 0, + "ast_successes" => 0 +} + +files.each do |path| + source = File.binread(path) + relative = path.delete_prefix("#{rails}/") + + begin + Lrama::Ruby::ActionScanner.new(source, filename: relative).scan + snapshot["scanner_successes"] += 1 + rescue Lrama::Ruby::ActionScanner::Error => error + snapshot["scanner_failures"] << {"file" => relative, "error" => error.message} + end + + begin + RubyVM::AbstractSyntaxTree.parse(source) + snapshot["ast_successes"] += 1 + rescue SyntaxError => error + snapshot["ast_failures"] << {"file" => relative, "error" => error.message.lines.first&.chomp} + end +end + +puts JSON.pretty_generate(snapshot)