Refactor Ruby frontend boundary and add boxed plugin command

AGENTS.md

@@ -26,8 +26,8 @@ versions is not currently promised.
 - `lib/lrama/ruby/parser.rb.erb`: standalone parser template.
 - `lib/lrama/ruby/backend.rb` and `action_code.rb`: internal files loaded in the generator's box.
 - `examples/calculator.y`: executable example grammar.
-- `lib/arpaka/`: frontend generator, CLI, source profile and Action mappings.
-- `lib/arpaka/runtime/`: AST, builder, lexer and frontend source templates.
+- `lib/arpaka/`: public frontend entry points and compatibility CLI.
+- `lib/arpaka/ruby/`: Ruby frontend generator, CLI, source profiles, Action mappings and runtime templates.
 - `exe/arpaka`: installed generation command.
 - `tool/ruby/`: development entry point for frontend generation and checks.
 - `vendor/ruby/`: pinned upstream grammar, preprocessing tools and license files.

README.md

@@ -164,6 +164,17 @@ RUBY_BOX=1 bundle exec arpaka generate \
 This mode embeds Arpaka's own license notices. The `--ruby-source` mode remains
 the compatibility-preserving path when the Ruby source tree is available.
 
+The experimental sample plugin demonstrates the plugin command shape. It is
+selected by the first command argument and runs in its own Ruby Box; the host
+process writes the generated source:
+
+```sh
+RUBY_BOX=1 bundle exec arpaka sample generate \
+  -y /path/to/parser.y \
+  -c MyParser \
+  -o lib/my_library/parser.rb
+```
+
 The consumer only needs the generated file and Ruby 4.0 or later:
 
 ```ruby
@@ -273,10 +284,10 @@ requires an explicit Ruby source tree. `ruby:check` verifies the source profile,
 Action inventory, grammar transformation and generated frontend without editing
 artifacts. `ruby:generate` writes the standalone frontend to `OUTPUT`.
 
-`lib/arpaka/actions.rb` holds the Ruby Action mappings;
-`lib/arpaka/upstream_rules.json` records the expected expanded upstream rules.
+`lib/arpaka/ruby/actions.rb` holds the Ruby Action mappings;
+`lib/arpaka/ruby/upstream_rules.json` records the expected expanded upstream rules.
 Update these and the source checksums together when adding support for a changed
-Ruby grammar. Runtime templates live under `lib/arpaka/runtime/`.
+Ruby grammar. Runtime templates live under `lib/arpaka/ruby/runtime/`.
 
 Tests exercise the generated frontend and the generic backend. Package checks
 install the gem, generate from external Ruby sources, and execute the resulting

lib/arpaka.rb

@@ -5,16 +5,13 @@
 module Arpaka
   class Error < Lrama::Ruby::Error; end
 
+  require_relative "arpaka/ruby"
+
   def self.generate(ruby_source: nil, parse_y: nil, class_name: "RubyParser")
-    Generator.new.generate(ruby_source: ruby_source, parse_y: parse_y, class_name: class_name)
+    Ruby.generate(ruby_source: ruby_source, parse_y: parse_y, class_name: class_name)
   end
 
   def self.compile(ruby_source: nil, parse_y: nil, class_name: "RubyParser")
-    source = generate(ruby_source: ruby_source, parse_y: parse_y, class_name: class_name)
-    box = ::Ruby::Box.new
-    box.eval(source)
-    box.const_get(class_name, false)
+    Ruby.compile(ruby_source: ruby_source, parse_y: parse_y, class_name: class_name)
   end
 end
-
-require_relative "arpaka/generator"

lib/arpaka/cli.rb

@@ -1,63 +1,15 @@
 # frozen_string_literal: true
 
-require "optparse"
-require "tempfile"
 require_relative "../arpaka"
+require_relative "ruby/cli"
+require_relative "plugins/cli"
 
 module Arpaka
   class CLI
     def self.run(argv, out: $stdout, err: $stderr)
-      options = { class_name: "RubyParser" }
-      force = false
-      help = false
-      parser = OptionParser.new do |opts|
-        opts.banner = "Usage: arpaka generate (--ruby-source DIR | --parse-y FILE) --output FILE [--class-name NAME] [--force]"
-        opts.on("--ruby-source DIR", "Ruby source tree (provided by the user)") { |v| options[:ruby_source] = v }
-        opts.on("--parse-y FILE", "parse.y file (use Arpaka's token definitions)") { |v| options[:parse_y] = v }
-        opts.on("--output FILE", "Write a standalone Ruby frontend") { |v| options[:output] = v }
-        opts.on("--class-name NAME", "Generated class name (default: RubyParser)") { |v| options[:class_name] = v }
-        opts.on("--force", "Replace an existing output file") { force = true }
-        opts.on("-h", "--help", "Show usage") { help = true }
-      end
-      args = argv.dup
-      command = args.shift
-      if ["--help", "-h"].include?(command)
-        out.puts(parser)
-        return 0
-      end
-      raise OptionParser::InvalidArgument, "expected generate" unless command == "generate"
-      parser.parse!(args)
-      if help
-        out.puts(parser)
-        return 0
-      end
-      raise OptionParser::InvalidArgument, args.join(" ") unless args.empty?
-      raise OptionParser::InvalidArgument, "specify exactly one of --ruby-source or --parse-y" if options[:ruby_source] == options[:parse_y]
-      raise OptionParser::MissingArgument, "--output" unless options[:output]
-      %i[output].each do |key|
-        raise OptionParser::MissingArgument, "--#{key.to_s.tr('_', '-')}" unless options[key]
-      end
-      output = File.expand_path(options.delete(:output))
-      if !force && (File.exist?(output) || File.symlink?(output))
-        raise Error, "Output already exists: #{output}; use --force to replace it"
-      end
-      source = Arpaka.generate(**options)
-      Tempfile.create([".arpaka-", ".rb"], File.dirname(output)) do |file|
-        file.write(source)
-        file.flush
-        file.chmod(0o644)
-        if force
-          File.rename(file.path, output)
-        else
-          # Publish only the complete file, without overwriting a concurrent writer.
-          File.link(file.path, output)
-        end
-      end
-      out.puts("Generated #{output}")
-      0
-    rescue OptionParser::ParseError, Lrama::Ruby::Error, SystemCallError => error
-      err.puts("arpaka: #{error.message}")
-      1
+      return Plugins::CLI.run(argv.drop(1), out: out, err: err) if argv.first == "sample"
+
+      Ruby::CLI.run(argv, out: out, err: err)
     end
   end
 end

lib/arpaka/plugins/cli.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require "optparse"
+require "tempfile"
+require_relative "sample_runner"
+
+module Arpaka
+  module Plugins
+    class CLI
+      def self.run(argv, out: $stdout, err: $stderr)
+        options = { class_name: "SampleParser" }
+        force = false
+        parser = OptionParser.new do |opts|
+          opts.banner = "Usage: arpaka sample generate -y FILE -o FILE [--class-name NAME] [--force]"
+          opts.on("-y FILE", "--grammar FILE", "Grammar file") { |v| options[:grammar] = v }
+          opts.on("-c NAME", "--class-name NAME", "Generated class name (default: SampleParser)") do |v|
+            options[:class_name] = v
+          end
+          opts.on("-o FILE", "--output FILE", "Write the generated parser") { |v| options[:output] = v }
+          opts.on("--force", "Replace an existing output file") { force = true }
+          opts.on("-h", "--help", "Show usage") { out.puts(parser); return 0 }
+        end
+
+        args = argv.dup
+        command = args.shift
+        if ["--help", "-h"].include?(command)
+          out.puts(parser)
+          return 0
+        end
+        raise OptionParser::InvalidArgument, "expected generate" unless command == "generate"
+        parser.parse!(args)
+        raise OptionParser::InvalidArgument, args.join(" ") unless args.empty?
+        raise OptionParser::MissingArgument, "--grammar" unless options[:grammar]
+        raise OptionParser::MissingArgument, "--output" unless options[:output]
+
+        output = File.expand_path(options.delete(:output))
+        if !force && (File.exist?(output) || File.symlink?(output))
+          raise ::Arpaka::Error, "Output already exists: #{output}; use --force to replace it"
+        end
+
+        source = SampleRunner.generate(**options)
+        Tempfile.create([".arpaka-", ".rb"], File.dirname(output)) do |file|
+          file.write(source)
+          file.flush
+          file.chmod(0o644)
+          if force
+            File.rename(file.path, output)
+          else
+            # Publish only the complete file, without overwriting a concurrent writer.
+            File.link(file.path, output)
+          end
+        end
+        out.puts("Generated #{output}")
+        0
+      rescue OptionParser::ParseError, Lrama::Ruby::Error, SystemCallError => error
+        err.puts("arpaka: #{error.message}")
+        1
+      end
+    end
+  end
+end

lib/arpaka/plugins/sample.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+require "lrama/ruby"
+
+module Arpaka
+  module Plugins
+    module Sample
+      def self.generate(grammar_path, class_name: "SampleParser")
+        grammar = File.read(grammar_path)
+        Lrama::Ruby.generate(grammar, filename: grammar_path, class_name: class_name)
+      end
+    end
+  end
+end

lib/arpaka/plugins/sample_runner.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Arpaka
+  module Plugins
+    class SampleRunner
+      PLUGIN = File.expand_path("sample.rb", __dir__)
+
+      def self.generate(grammar:, class_name:)
+        unless ::Ruby::Box.enabled?
+          raise ::Arpaka::Error, "Ruby Box is required for plugins. Start Ruby with RUBY_BOX=1."
+        end
+
+        box = ::Ruby::Box.new
+        # A new box does not inherit Bundler's activated load paths.
+        box.load_path.replace($LOAD_PATH)
+        box.require(PLUGIN)
+        box.eval("Arpaka::Plugins::Sample.generate(#{grammar.dump}, class_name: #{class_name.dump})")
+      end
+    end
+  end
+end

lib/arpaka/ruby.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Arpaka
+  module Ruby
+    def self.generate(ruby_source: nil, parse_y: nil, class_name: "RubyParser")
+      Generator.new.generate(ruby_source: ruby_source, parse_y: parse_y, class_name: class_name)
+    end
+
+    def self.compile(ruby_source: nil, parse_y: nil, class_name: "RubyParser")
+      source = generate(ruby_source: ruby_source, parse_y: parse_y, class_name: class_name)
+      box = ::Ruby::Box.new
+      box.eval(source)
+      box.const_get(class_name, false)
+    end
+  end
+end
+
+require_relative "ruby/generator"

lib/arpaka/actions.rb → lib/arpaka/ruby/actions.rb

@@ -2,7 +2,7 @@
 
 # Rule IDs refer to upstream_rules.json, validated against the pinned source.
 # Each entry is [classification, Ruby expression used as the reduction value].
-module Arpaka::RubyGrammarActions
+module Arpaka::Ruby::Actions
   ACTIONS = {
     1 => ["empty", "nil"], 2 => ["empty", "nil"], 3 => ["empty", "nil"],
     4 => ["pass_through", "$1"], 5 => ["ported", "@builder.program($2)"],

lib/arpaka/ruby/cli.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require "optparse"
+require "tempfile"
+require_relative "../../arpaka"
+
+module Arpaka::Ruby
+  class CLI
+    def self.run(argv, out: $stdout, err: $stderr)
+      options = { class_name: "RubyParser" }
+      force = false
+      help = false
+      parser = OptionParser.new do |opts|
+        opts.banner = "Usage: arpaka generate (--ruby-source DIR | --parse-y FILE) --output FILE [--class-name NAME] [--force]"
+        opts.on("--ruby-source DIR", "Ruby source tree (provided by the user)") { |v| options[:ruby_source] = v }
+        opts.on("--parse-y FILE", "parse.y file (use Arpaka's token definitions)") { |v| options[:parse_y] = v }
+        opts.on("--output FILE", "Write a standalone Ruby frontend") { |v| options[:output] = v }
+        opts.on("--class-name NAME", "Generated class name (default: RubyParser)") { |v| options[:class_name] = v }
+        opts.on("--force", "Replace an existing output file") { force = true }
+        opts.on("-h", "--help", "Show usage") { help = true }
+      end
+      args = argv.dup
+      command = args.shift
+      if ["--help", "-h"].include?(command)
+        out.puts(parser)
+        return 0
+      end
+      raise OptionParser::InvalidArgument, "expected generate" unless command == "generate"
+      parser.parse!(args)
+      if help
+        out.puts(parser)
+        return 0
+      end
+      raise OptionParser::InvalidArgument, args.join(" ") unless args.empty?
+      raise OptionParser::InvalidArgument, "specify exactly one of --ruby-source or --parse-y" if options[:ruby_source] == options[:parse_y]
+      raise OptionParser::MissingArgument, "--output" unless options[:output]
+      %i[output].each do |key|
+        raise OptionParser::MissingArgument, "--#{key.to_s.tr('_', '-')}" unless options[key]
+      end
+      output = File.expand_path(options.delete(:output))
+      if !force && (File.exist?(output) || File.symlink?(output))
+        raise ::Arpaka::Error, "Output already exists: #{output}; use --force to replace it"
+      end
+      source = Arpaka::Ruby.generate(**options)
+      Tempfile.create([".arpaka-", ".rb"], File.dirname(output)) do |file|
+        file.write(source)
+        file.flush
+        file.chmod(0o644)
+        if force
+          File.rename(file.path, output)
+        else
+          # Publish only the complete file, without overwriting a concurrent writer.
+          File.link(file.path, output)
+        end
+      end
+      out.puts("Generated #{output}")
+      0
+    rescue OptionParser::ParseError, Lrama::Ruby::Error, SystemCallError => error
+      err.puts("arpaka: #{error.message}")
+      1
+    end
+  end
+end

lib/arpaka/generator.rb → lib/arpaka/ruby/generator.rb

@@ -2,21 +2,21 @@
 
 require "erb"
 require "json"
-require_relative "ruby_grammar"
+require_relative "grammar"
 
-module Arpaka
+module Arpaka::Ruby
   class Generator
     def generate(ruby_source: nil, parse_y: nil, class_name:)
       if ruby_source.nil? == parse_y.nil?
-        raise Error, "specify exactly one of ruby_source or parse_y"
+        raise ::Arpaka::Error, "specify exactly one of ruby_source or parse_y"
       end
       unless /\A[A-Z][a-zA-Z0-9_]*\z/.match?(class_name)
-        raise Error, "class_name must be a single Ruby constant name"
+        raise ::Arpaka::Error, "class_name must be a single Ruby constant name"
       end
       unless ::Ruby::Box.enabled?
-        raise Error, "Ruby Box is required for generation. Start Ruby with RUBY_BOX=1."
+        raise ::Arpaka::Error, "Ruby Box is required for generation. Start Ruby with RUBY_BOX=1."
       end
-      artifacts = RubyGrammar.artifacts(ruby_source: ruby_source, parse_y: parse_y)
+      artifacts = Grammar.artifacts(ruby_source: ruby_source, parse_y: parse_y)
       grammar_filename = parse_y || File.join(ruby_source, "parse.y")
       parser = Lrama::Ruby.generate(artifacts.fetch("parse.y"),
         filename: grammar_filename, class_name: "Parser", allow_error_rules: true)
@@ -27,7 +27,7 @@ def generate(ruby_source: nil, parse_y: nil, class_name:)
         template = File.read(File.join(__dir__, "runtime", "#{name}.rb.erb"))
         ERB.new(template, trim_mode: "-").result_with_hash(rules: rules, runtime: runtime_options)
       end.join("\n")
-      notices = [File.read(File.expand_path("../../LICENSE.txt", __dir__))]
+      notices = [File.read(File.expand_path("../../../LICENSE.txt", __dir__))]
       %w[COPYING BSDL].each do |name|
         path = ruby_source ? File.join(ruby_source, name) : File.join(__dir__, name)
         notices << File.read(path)
@@ -43,9 +43,9 @@ def generate(ruby_source: nil, parse_y: nil, class_name:)
       box.eval(source)
       source
     rescue SyntaxError => error
-      raise Error, "Invalid generated frontend: #{error.message}"
+      raise ::Arpaka::Error, "Invalid generated frontend: #{error.message}"
     rescue SystemCallError => error
-      raise Error, "Cannot read Ruby frontend inputs: #{error.message}"
+      raise ::Arpaka::Error, "Cannot read Ruby frontend inputs: #{error.message}"
     end
   end
 end

lib/arpaka/ruby_grammar.rb → lib/arpaka/ruby/grammar.rb

@@ -4,9 +4,9 @@
 require "json"
 require "digest"
 require_relative "actions"
-require_relative "profile/ruby"
+require_relative "profile"
 
-module Arpaka::RubyGrammar
+module Arpaka::Ruby::Grammar
   REVISION = "37d60dd3241d5fdb10ca43f36077f5d556ae1b8d"
   EXTENDED_RULES = {
     ["value_expr_command", ["command"]] => "tLBRACE_ARG"
@@ -110,7 +110,7 @@ def self.artifacts(ruby_source: nil, parse_y: nil)
     end
     names = symbol_names(grammar)
     name = ->(symbol) { symbol.term ? symbol.id.s_value : names.fetch(symbol.id.s_value) }
-    profile = ::Arpaka::Profile::RUBY.select(grammar)
+    profile = ::Arpaka::Ruby::Profile.select(grammar)
     expected_conflicts = (grammar.expect || 0) + profile.fetch("additional_expected_conflicts", 0)
     lines = ["/* Generated from ruby/ruby #{REVISION}. See Arpaka Action mappings. */",
       "%no-stdlib", "%expect #{expected_conflicts}"]