diff --git a/arpaka.gemspec b/arpaka.gemspec index f3bfd1a..9df9315 100644 --- a/arpaka.gemspec +++ b/arpaka.gemspec @@ -28,4 +28,5 @@ Gem::Specification.new do |spec| spec.require_paths = ["lib"] spec.add_dependency "lrama", ">= 0.8.0" + spec.add_dependency "racc" end diff --git a/lib/arpaka/ruby_grammar.rb b/lib/arpaka/ruby_grammar.rb index f57d668..13c017f 100644 --- a/lib/arpaka/ruby_grammar.rb +++ b/lib/arpaka/ruby_grammar.rb @@ -3,8 +3,6 @@ require "lrama" require "json" require "digest" -require "open3" -require "rbconfig" require_relative "actions" module Arpaka::RubyGrammar @@ -35,9 +33,8 @@ def self.load_upstream(ruby_source) warn "Arpaka: Ruby source differs from the recorded profile: #{path}; " \ "expected SHA256 #{digest}, got #{actual}." end - source, error, result = Open3.capture3(RbConfig.ruby, - File.join(vendor, "tool/id2token.rb"), File.join(vendor, "parse.y")) - raise ::Arpaka::Error, "Ruby preprocessing failed: #{error}" unless result.success? + source = preprocess(File.read(File.join(vendor, "parse.y")), + File.join(vendor, "defs/id.def")) grammar = parse(source, File.join(vendor, "parse.y")) inventory = grammar.rules.reject(&:initial_rule?).map do |rule| { "id" => rule.id, "lhs" => rule.lhs.id.s_value, @@ -52,6 +49,29 @@ def self.load_upstream(ruby_source) [grammar, inventory] end + # Ruby's id.def is Ruby code. Evaluate it in a Box so its temporary locals, + # methods and constants cannot affect Arpaka or Lrama's process. + def self.preprocess(source, id_def) + box = ::Ruby::Box.new + ids = box.eval(File.read(id_def)) + tokens = ids.fetch(:token_op).each_with_object({}) do |(_id, _op, token, id), result| + result[token] = id if token + end + return source unless tokens.any? + + names = tokens.keys.map { |name| Regexp.escape(name) }.join("|") + token_pattern = /\bRUBY_TOKEN\((#{names})\)\s*(?=\s)/ + source.lines.map do |line| + if line.start_with?("%token") + line.gsub(token_pattern) { tokens.fetch(Regexp.last_match(1)).to_s } + else + line + end + end.join + rescue KeyError, TypeError, SyntaxError => error + raise ::Arpaka::Error, "Ruby id.def preprocessing failed: #{error.message}" + end + def self.symbol_names(grammar) used = grammar.terms.map { |symbol| symbol.id.s_value } grammar.nterms.to_h do |symbol| diff --git a/test/lrama/frontend_generator_test.rb b/test/lrama/frontend_generator_test.rb index 976b4fc..92fd3e6 100644 --- a/test/lrama/frontend_generator_test.rb +++ b/test/lrama/frontend_generator_test.rb @@ -76,6 +76,16 @@ def generated_source end end + test "preprocesses id.def inside a Ruby Box" do + source = "%token RUBY_TOKEN(tIDENTIFIER) RUBY_TOKEN(tINTEGER)\n%token RUBY_TOKEN(tUNKNOWN)\n" + Dir.mktmpdir do |directory| + id_def = File.join(directory, "id.def") + File.write(id_def, '{ token_op: [["identifier", "", "tIDENTIFIER", 1], ["integer", "", "tINTEGER", 2]] }') + assert_equal("%token 1 2\n%token RUBY_TOKEN(tUNKNOWN)\n", + Arpaka::RubyGrammar.preprocess(source, id_def)) + end + end + test "CLI requires output and handles help" do out, err = StringIO.new, StringIO.new assert_equal(0, Arpaka::CLI.run(["--help"], out: out, err: err))