Load Ruby token definitions in Arpaka

arpaka.gemspec

@@ -28,4 +28,5 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_dependency "lrama", ">= 0.8.0"
+  spec.add_dependency "racc"
 end

lib/arpaka/ruby_grammar.rb

@@ -3,8 +3,6 @@
 require "lrama"
 require "json"
 require "digest"
-require "open3"
-require "rbconfig"
 require_relative "actions"
 
 module Arpaka::RubyGrammar
@@ -35,9 +33,8 @@ def self.load_upstream(ruby_source)
       warn "Arpaka: Ruby source differs from the recorded profile: #{path}; " \
         "expected SHA256 #{digest}, got #{actual}."
     end
-    source, error, result = Open3.capture3(RbConfig.ruby,
-      File.join(vendor, "tool/id2token.rb"), File.join(vendor, "parse.y"))
-    raise ::Arpaka::Error, "Ruby preprocessing failed: #{error}" unless result.success?
+    source = preprocess(File.read(File.join(vendor, "parse.y")),
+      File.join(vendor, "defs/id.def"))
     grammar = parse(source, File.join(vendor, "parse.y"))
     inventory = grammar.rules.reject(&:initial_rule?).map do |rule|
       { "id" => rule.id, "lhs" => rule.lhs.id.s_value,
@@ -52,6 +49,29 @@ def self.load_upstream(ruby_source)
     [grammar, inventory]
   end
 
+  # Ruby's id.def is Ruby code. Evaluate it in a Box so its temporary locals,
+  # methods and constants cannot affect Arpaka or Lrama's process.
+  def self.preprocess(source, id_def)
+    box = ::Ruby::Box.new
+    ids = box.eval(File.read(id_def))
+    tokens = ids.fetch(:token_op).each_with_object({}) do |(_id, _op, token, id), result|
+      result[token] = id if token
+    end
+    return source unless tokens.any?
+
+    names = tokens.keys.map { |name| Regexp.escape(name) }.join("|")
+    token_pattern = /\bRUBY_TOKEN\((#{names})\)\s*(?=\s)/
+    source.lines.map do |line|
+      if line.start_with?("%token")
+        line.gsub(token_pattern) { tokens.fetch(Regexp.last_match(1)).to_s }
+      else
+        line
+      end
+    end.join
+  rescue KeyError, TypeError, SyntaxError => error
+    raise ::Arpaka::Error, "Ruby id.def preprocessing failed: #{error.message}"
+  end
+
   def self.symbol_names(grammar)
     used = grammar.terms.map { |symbol| symbol.id.s_value }
     grammar.nterms.to_h do |symbol|

test/lrama/frontend_generator_test.rb

@@ -76,6 +76,16 @@ def generated_source
     end
   end
 
+  test "preprocesses id.def inside a Ruby Box" do
+    source = "%token RUBY_TOKEN(tIDENTIFIER) RUBY_TOKEN(tINTEGER)\n%token RUBY_TOKEN(tUNKNOWN)\n"
+    Dir.mktmpdir do |directory|
+      id_def = File.join(directory, "id.def")
+      File.write(id_def, '{ token_op: [["identifier", "", "tIDENTIFIER", 1], ["integer", "", "tINTEGER", 2]] }')
+      assert_equal("%token 1 2\n%token RUBY_TOKEN(tUNKNOWN)\n",
+        Arpaka::RubyGrammar.preprocess(source, id_def))
+    end
+  end
+
   test "CLI requires output and handles help" do
     out, err = StringIO.new, StringIO.new
     assert_equal(0, Arpaka::CLI.run(["--help"], out: out, err: err))