From 76508e8e14bddbee118d099c4e6c25829ef67cba Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Sun, 13 Sep 2026 09:59:25 +0200 Subject: [PATCH 1/5] [ruby/erb] Add escape test coverage for strings as long as 128 bytes Extracted from https://github.com/ruby/erb/pull/141 and https://github.com/ruby/erb/pull/144 https://github.com/ruby/erb/commit/182850a467 Co-Authored-By: Scott Myron --- test/erb/test_erb_escape.rb | 79 +++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/test/erb/test_erb_escape.rb b/test/erb/test_erb_escape.rb index c351feb862cbbc..e5299438cc94a1 100644 --- a/test/erb/test_erb_escape.rb +++ b/test/erb/test_erb_escape.rb @@ -58,6 +58,85 @@ def test_simd_coverage assert_equal '<' * 32, h('<' * 32) end + def test_html_escape_simd_block_boundary + # Ensure we only escape the characters that need to be escaped. + (0...128).each do |pos| + s = "a" * 128 + s[pos] = "<" + expected = "a" * pos + "<" + "a" * (128 - pos - 1) + assert_equal(expected, ERB::Util.html_escape(s), "escape at position #{pos}") + end + end + + HTML_ESCAPE_ENTITIES = {"'" => "'", '"' => """, "&" => "&", "<" => "<", ">" => ">"} + + def test_html_escape_simd_multiple_matches_per_block + chars = ["'", '"', '&', '<', '>'] + (0..15).each do |a| + (0..15).each do |b| + next if a == b + s = "a" * 32 + s[a] = chars[a % chars.size] + s[b] = chars[b % chars.size] + expected = Array.new(32, "a") + expected[a] = HTML_ESCAPE_ENTITIES[chars[a % chars.size]] + expected[b] = HTML_ESCAPE_ENTITIES[chars[b % chars.size]] + assert_equal(expected.join, ERB::Util.html_escape(s), "positions #{a}, #{b}") + end + end + end + + def test_html_escape_simd_tail_lengths + (1..128).each do |len| + (0...len).each do |pos| + s = "a" * len + s[pos] = ">" + expected = "a" * pos + ">" + "a" * (len - pos - 1) + assert_equal(expected, ERB::Util.html_escape(s), "len=#{len} pos=#{pos}") + end + end + end + + def test_html_escape_simd_wide_block_boundary + # Ensure a 64-byte-wide SIMD fast path correctly locates a match + # at every byte position, including the last byte of the block + # (which is special-cased in find_next_match_neon). + (0...128).each do |pos| + s = "a" * 128 + s[pos] = "<" + expected = "a" * pos + "<" + "a" * (128 - pos - 1) + assert_equal(expected, ERB::Util.html_escape(s), "escape at position #{pos}") + end + end + + def test_html_escape_simd_wide_block_multiple_matches + chars = ["'", '"', '&', '<', '>'] + boundary_positions = [0, 1, 15, 16, 17, 31, 32, 33, 47, 48, 49, 62, 63] + boundary_positions.each do |a| + boundary_positions.each do |b| + next if a == b + s = "a" * 64 + s[a] = chars[a % chars.size] + s[b] = chars[b % chars.size] + expected = Array.new(64, "a") + expected[a] = HTML_ESCAPE_ENTITIES[chars[a % chars.size]] + expected[b] = HTML_ESCAPE_ENTITIES[chars[b % chars.size]] + assert_equal(expected.join, ERB::Util.html_escape(s), "positions #{a}, #{b}") + end + end + end + + def test_html_escape_simd_wide_block_tail_lengths + ([*56..72] + [*120..136]).each do |len| + (0...len).each do |pos| + s = "a" * len + s[pos] = ">" + expected = "a" * pos + ">" + "a" * (len - pos - 1) + assert_equal(expected, ERB::Util.html_escape(s), "len=#{len} pos=#{pos}") + end + end + end + private def h(...) From 4ca48a192c225b887ac2e5d6d30486bb6850887f Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Sun, 13 Sep 2026 10:14:28 +0200 Subject: [PATCH 2/5] [ruby/erb] escape.c: refactor `find_next_match_*` to not consume the match Now the caller is responsible for consuming the match using `consume_match`, which means the undefined behavior `matches_bitmap >>= 64` is no longer a concern, eliminating a condition. https://github.com/ruby/erb/commit/40a8edea18 --- ext/erb/escape/escape.c | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/ext/erb/escape/escape.c b/ext/erb/escape/escape.c index e905bb54bf6d27..66cb77ed332a7d 100644 --- a/ext/erb/escape/escape.c +++ b/ext/erb/escape/escape.c @@ -77,7 +77,7 @@ find_next_basic(search_state *search) #ifdef HAVE_SIMD_SSE2 -static inline int trailing_zeros(int input) +static inline int trailing_zeros32(int input) { RUBY_ASSERT(input > 0); // __builtin_ctz(0) is undefined behavior @@ -97,9 +97,9 @@ static inline int trailing_zeros(int input) static inline bool find_next_match_sse2(search_state *search) { - int next_match_offset = trailing_zeros(search->matches_bitmap); - search->matches_bitmap >>= (next_match_offset + 1); - search->cstr += next_match_offset; + uint32_t trailing_zeros = trailing_zeros32(search->matches_bitmap); + search->matches_bitmap >>= trailing_zeros; + search->cstr += trailing_zeros; RUBY_ASSERT(search->cstr <= search->end); return true; } @@ -170,14 +170,10 @@ find_next_match_neon(search_state *search) uint32_t trailing_zeros = trailing_zeros64(search->matches_bitmap); // uint64_t >>= 64 is undefined behaviour - if (trailing_zeros >= 63) { - search->matches_bitmap = 0; - search->cstr += 15; - } - else { - search->matches_bitmap >>= (trailing_zeros + 1); - search->cstr += trailing_zeros / 4; - } + RUBY_ASSERT(trailing_zeros < 64); + search->matches_bitmap >>= trailing_zeros; + search->cstr += trailing_zeros / 4; + RUBY_ASSERT(search->cstr <= search->end); return true; } @@ -223,6 +219,15 @@ find_next_neon(search_state *search) #define find_next find_next_neon #endif // HAVE_SIMD_NEON +static inline void +consume_match(search_state *search) +{ +#ifdef HAVE_SIMD + search->matches_bitmap >>= 1; +#endif + search->cstr++; +} + #ifndef find_next #define find_next find_next_basic #endif @@ -242,9 +247,8 @@ optimized_escape_html(VALUE str) while (find_next(&search)) { const unsigned char c = *search.cstr; - size_t segment_len = search.cstr - segment_start; - search.cstr++; + size_t segment_len = search.cstr - segment_start; if (!buf) { buf = ALLOCV_N(char, vbuf, escaped_length(str)); dest = buf; @@ -253,7 +257,6 @@ optimized_escape_html(VALUE str) memcpy(dest, segment_start, segment_len); dest += segment_len; } - segment_start = search.cstr; switch(c) { #define HTML_ESCAPE(c, str) \ @@ -272,6 +275,8 @@ optimized_escape_html(VALUE str) #undef HTML_ESCAPE } + consume_match(&search); + segment_start = search.cstr; } VALUE escaped = str; From fa7e8c848d25dc9893ee093e815d783801c74def Mon Sep 17 00:00:00 2001 From: Earlopain <14981592+Earlopain@users.noreply.github.com> Date: Sun, 13 Sep 2026 11:41:35 +0200 Subject: [PATCH 3/5] [ruby/prism] Make `SymbolNode#value_loc` non-optional (https://github.com/ruby/prism/pull/4220) Sort of a followup for https://github.com/ruby/prism/commit/929aec16504f799dae358757784741ec8b2b1c6a It changed the value for `:''` into nil/null, which is a bit inconvenient. For syntax-valid code I expect it to always be present. It's also inconsistent with `StringNode`. In effect, this reverts the snapshots changes and the two changes in ruby for the ripper/parser compiler. https://github.com/ruby/prism/commit/531cd5e557 --- lib/prism/translation/parser/compiler.rb | 12 ++++++------ lib/prism/translation/ripper.rb | 12 +++++++----- prism/config.yml | 2 +- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/lib/prism/translation/parser/compiler.rb b/lib/prism/translation/parser/compiler.rb index 1ef30dab1beac8..256d873f666918 100644 --- a/lib/prism/translation/parser/compiler.rb +++ b/lib/prism/translation/parser/compiler.rb @@ -165,13 +165,13 @@ def visit_assoc_node(node) else parts = if key.is_a?(SymbolNode) - value_loc = key.value_loc - if value_loc.nil? + value = key.value + if value == "" [] - elsif value_loc.slice.include?("\n") - string_nodes_from_line_continuations(key.unescaped, value_loc.slice, value_loc.start_offset, key.opening) + elsif value.include?("\n") + string_nodes_from_line_continuations(key.unescaped, value, key.value_loc.start_offset, key.opening) else - [builder.string_internal([key.unescaped, srange(value_loc)])] + [builder.string_internal([key.unescaped, srange(key.value_loc)])] end else visit_all(key.parts) @@ -1775,7 +1775,7 @@ def visit_symbol_node(node) end else parts = - if node.value_loc.nil? + if node.value == "" [] elsif node.value.include?("\n") string_nodes_from_line_continuations(node.unescaped, node.value, node.value_loc.start_offset, node.opening) diff --git a/lib/prism/translation/ripper.rb b/lib/prism/translation/ripper.rb index d9c1e69f920d48..de178edd2c58a8 100644 --- a/lib/prism/translation/ripper.rb +++ b/lib/prism/translation/ripper.rb @@ -3840,12 +3840,14 @@ def visit_super_node(node) # ^^^^ def visit_symbol_node(node) with_string_bounds(node) do - if node.value_loc.nil? - bounds(node.location) - on_dyna_symbol(on_string_content) - elsif (opening = node.opening)&.match?(/^%s|['"]:?$/) + if (opening = node.opening)&.match?(/^%s|['"]:?$/) bounds(node.value_loc) - content = on_string_add(on_string_content, on_tstring_content(node.value)) + content = on_string_content + + if !(value = node.value).empty? + content = on_string_add(content, on_tstring_content(value)) + end + bounds(node.location) on_dyna_symbol(content) elsif (closing = node.closing) == ":" diff --git a/prism/config.yml b/prism/config.yml index 4892089c031ab8..f5ff264693d963 100644 --- a/prism/config.yml +++ b/prism/config.yml @@ -4553,7 +4553,7 @@ nodes: - name: opening_loc type: location? - name: value_loc - type: location? + type: location - name: closing_loc type: location? - name: unescaped From 882f4e3fd58d85922bf745560f063e5c40ffae30 Mon Sep 17 00:00:00 2001 From: Earlopain <14981592+Earlopain@users.noreply.github.com> Date: Sun, 13 Sep 2026 11:44:30 +0200 Subject: [PATCH 4/5] [ruby/prism] Allow path-like objects for `Prism.parse_file` and similar (https://github.com/ruby/prism/pull/4226) This tries to coerce with `to_path` and then `to_str` if it isn't a string already. I had a `Pathname` from some other source and was sad I couldn't simply pass it along https://github.com/ruby/prism/commit/4651adbaeb --- lib/prism.rb | 16 ++++++++-------- lib/prism/ffi.rb | 29 +++++++++++++---------------- prism/extension.c | 6 ++++-- test/prism/api/parse_test.rb | 6 ++++++ 4 files changed, 31 insertions(+), 26 deletions(-) diff --git a/lib/prism.rb b/lib/prism.rb index f4b36b1826eb7d..bff555ffb3f8c6 100644 --- a/lib/prism.rb +++ b/lib/prism.rb @@ -112,14 +112,14 @@ def self.find(callable) # def self.parse_success?: (String source, ?filepath: String, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> bool # def self.parse_failure?: (String source, ?filepath: String, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> bool # def self.parse_stream: (_Stream stream, ?filepath: String, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> ParseResult - # def self.parse_file: (String filepath, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> ParseResult - # def self.profile_file: (String filepath, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> void - # def self.lex_file: (String filepath, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> LexResult - # def self.parse_lex_file: (String filepath, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> ParseLexResult - # def self.dump_file: (String filepath, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> String - # def self.parse_file_comments: (String filepath, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> Array[Comment] - # def self.parse_file_success?: (String filepath, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> bool - # def self.parse_file_failure?: (String filepath, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> bool + # def self.parse_file: (path filepath, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> ParseResult + # def self.profile_file: (path filepath, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> void + # def self.lex_file: (path filepath, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> LexResult + # def self.parse_lex_file: (path filepath, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> ParseLexResult + # def self.dump_file: (path filepath, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> String + # def self.parse_file_comments: (path filepath, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> Array[Comment] + # def self.parse_file_success?: (path filepath, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> bool + # def self.parse_file_failure?: (path filepath, ?command_line: String, ?encoding: Encoding | false, ?freeze: bool, ?frozen_string_literal: bool, ?line: Integer, ?main_script: bool, ?partial_script: bool, ?raise_error: Symbol | true, ?scopes: Array[Array[Symbol]], ?version: String) -> bool end require_relative "prism/polyfill/byteindex" diff --git a/lib/prism/ffi.rb b/lib/prism/ffi.rb index d717925cc0a161..a9442bd26e9282 100644 --- a/lib/prism/ffi.rb +++ b/lib/prism/ffi.rb @@ -214,7 +214,12 @@ def self.with_string(string) end # Yields a PrismSource to the given block, backed by a pm_source_t. - def self.with_file(filepath) + def self.with_file(filepath, options) + unless filepath.is_a?(String) + filepath = filepath.to_path if filepath.respond_to?(:to_path) + filepath = filepath.to_str if filepath.respond_to?(:to_str) + end + options[:filepath] = filepath raise TypeError unless filepath.is_a?(String) # On Windows and Mac, it's expected that filepaths will be encoded in @@ -265,8 +270,7 @@ def dump(source, **options) # Mirror the Prism.dump_file API by using the serialization API. def dump_file(filepath, **options) - options[:filepath] = filepath - LibRubyParser::PrismSource.with_file(filepath) { |string| dump_common(string, options) } + LibRubyParser::PrismSource.with_file(filepath, options) { |string| dump_common(string, options) } end # Mirror the Prism.lex API by using the serialization API. @@ -276,8 +280,7 @@ def lex(code, **options) # Mirror the Prism.lex_file API by using the serialization API. def lex_file(filepath, **options) - options[:filepath] = filepath - LibRubyParser::PrismSource.with_file(filepath) { |string| lex_common(string, string.read, options) } + LibRubyParser::PrismSource.with_file(filepath, options) { |string| lex_common(string, string.read, options) } end # Mirror the Prism.parse API by using the serialization API. @@ -289,8 +292,7 @@ def parse(code, **options) # native strings instead of Ruby strings because it allows us to use mmap # when it is available. def parse_file(filepath, **options) - options[:filepath] = filepath - LibRubyParser::PrismSource.with_file(filepath) { |string| parse_common(string, string.read, options) } + LibRubyParser::PrismSource.with_file(filepath, options) { |string| parse_common(string, string.read, options) } end # Mirror the Prism.parse_stream API by using the serialization API. @@ -349,8 +351,7 @@ def parse_comments(code, **options) # API. This uses native strings instead of Ruby strings because it allows us # to use mmap when it is available. def parse_file_comments(filepath, **options) - options[:filepath] = filepath - LibRubyParser::PrismSource.with_file(filepath) { |string| parse_comments_common(string, string.read, options) } + LibRubyParser::PrismSource.with_file(filepath, options) { |string| parse_comments_common(string, string.read, options) } end # Mirror the Prism.parse_lex API by using the serialization API. @@ -360,8 +361,7 @@ def parse_lex(code, **options) # Mirror the Prism.parse_lex_file API by using the serialization API. def parse_lex_file(filepath, **options) - options[:filepath] = filepath - LibRubyParser::PrismSource.with_file(filepath) { |string| parse_lex_common(string, string.read, options) } + LibRubyParser::PrismSource.with_file(filepath, options) { |string| parse_lex_common(string, string.read, options) } end # Mirror the Prism.parse_success? API by using the serialization API. @@ -376,8 +376,7 @@ def parse_failure?(code, **options) # Mirror the Prism.parse_file_success? API by using the serialization API. def parse_file_success?(filepath, **options) - options[:filepath] = filepath - LibRubyParser::PrismSource.with_file(filepath) { |string| parse_file_success_common(string, options) } + LibRubyParser::PrismSource.with_file(filepath, options) { |string| parse_file_success_common(string, options) } end # Mirror the Prism.parse_file_failure? API by using the serialization API. @@ -401,9 +400,7 @@ def profile(source, **options) # Mirror the Prism.profile_file API by using the serialization API. def profile_file(filepath, **options) - LibRubyParser::PrismSource.with_file(filepath) do |string| - options[:filepath] = filepath - + LibRubyParser::PrismSource.with_file(filepath, options) do |string| if (format_type = raise_error_format_type(options)) raise_error(string, options, format_type) end diff --git a/prism/extension.c b/prism/extension.c index 7a5250f0d53d9b..62979873ed609e 100644 --- a/prism/extension.c +++ b/prism/extension.c @@ -393,9 +393,11 @@ file_options(int argc, VALUE *argv, pm_options_t *options, VALUE *encoded_filepa VALUE keywords; rb_scan_args(argc, argv, "1:", &filepath, &keywords); - if (!RB_TYPE_P(filepath, T_STRING)) { + int state = 0; + filepath = rb_protect(rb_get_path, filepath, &state); + if (state != 0) { pm_options_free(options); - rb_raise(rb_eTypeError, "wrong argument type %"PRIsVALUE" (expected String)", rb_obj_class(filepath)); + rb_jump_tag(state); } *encoded_filepath = rb_str_encode_ospath(filepath); diff --git a/test/prism/api/parse_test.rb b/test/prism/api/parse_test.rb index c9a47c1a61e03b..5ca67d20bd0ca9 100644 --- a/test/prism/api/parse_test.rb +++ b/test/prism/api/parse_test.rb @@ -69,6 +69,12 @@ def test_parse_tempfile end end + def test_parse_pathname + pathname = Pathname.new(__FILE__) + node = Prism.parse_file(pathname).value + assert_kind_of ProgramNode, node + end + if RUBY_ENGINE != "truffleruby" def test_parse_nonascii Dir.mktmpdir do |dir| From 9d54011bffbd95d4f00e542145cf271f3476dcb0 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Wed, 9 Sep 2026 14:15:26 +0900 Subject: [PATCH 5/5] Fix memory leak of TracePoint in Ractor TracePoints that are enabled but never disabled in a Ractor leak memory. For example, the following script leaks memory: 10.times do 1_000.times do Ractor.new { 200.times { TracePoint.new(:call) { }.enable } }.value GC.start end puts `ps -o rss= -p #{$$}` end Before: 32180 47868 63492 79116 94744 110368 125992 141616 157244 172868 After: 16428 16492 16492 16492 16492 16492 16492 16492 16492 16492 --- vm_trace.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/vm_trace.c b/vm_trace.c index 8b99335b95962e..ab677eaef8db4d 100644 --- a/vm_trace.c +++ b/vm_trace.c @@ -92,6 +92,9 @@ static void clean_hooks(rb_hook_list_t *list); void rb_hook_list_free(rb_hook_list_t *hooks) { + for (rb_event_hook_t *hook = hooks->hooks; hook; hook = hook->next) { + hook->hook_flags |= RUBY_EVENT_HOOK_FLAG_DELETED; + } hooks->need_clean = true; if (hooks->running == 0) {