From 0a87db47689c8057caf0c13c00fca21d7032f709 Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Tue, 1 Sep 2026 21:31:30 +0100 Subject: [PATCH 01/24] Release empty heap pages under one pool lock heap_pages_free_unused_pages called heap_page_free once per empty page, and each call took page_pool.lock twice: once in global_page_index_remove and once in page_pool_release. A full drain of N pages cost 2N lock cycles This commit links pages into a local chain using free_next and then releases the whole chain under a single page_pool.lock. --- gc/default/default.c | 103 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 84 insertions(+), 19 deletions(-) diff --git a/gc/default/default.c b/gc/default/default.c index 4e36f6d399b433..0125ea87d2bd0c 100644 --- a/gc/default/default.c +++ b/gc/default/default.c @@ -852,6 +852,9 @@ static void objspace_absorb(rb_objspace_t *dst, rb_objspace_t *src); static struct heap_page_body *page_pool_acquire(struct page_arena **arena_out); static void page_pool_release(struct heap_page_body *body, struct page_arena *arena); +#ifdef HAVE_MMAP +static void page_pool_release_locked(struct heap_page_body *body, struct page_arena *arena); +#endif static void page_pool_reclaim(rb_global_objspace_t *g); static void @@ -2224,6 +2227,12 @@ heap_page_body_free(struct heap_page_body *page_body, struct page_arena *arena) page_pool_release(page_body, arena); } +#if RGENGC_CHECK_MODE && !defined(_WIN32) && !defined(__wasi__) +# define ASSERT_PAGE_POOL_LOCKED(g) GC_ASSERT(rb_native_mutex_trylock(&(g)->page_pool.lock) == EBUSY) +#else +# define ASSERT_PAGE_POOL_LOCKED(g) ((void)0) +#endif + /* Insert into page_index. Writers serialize on page_pool.lock; lomem and himem are a * monotonically growing over-approximation used for a quick reject. */ static void @@ -2259,12 +2268,13 @@ global_page_index_insert(struct heap_page *page) } static void -global_page_index_remove(const struct heap_page *page) +global_page_index_remove_locked(const struct heap_page *page) { rb_global_objspace_t *g = global_objspace; uintptr_t body = (uintptr_t)page->body; - rb_native_mutex_lock(&g->page_pool.lock); + ASSERT_PAGE_POOL_LOCKED(g); + size_t lo = 0, hi = g->page_index.n_pages; while (lo < hi) { size_t mid = (lo + hi) / 2; @@ -2275,6 +2285,15 @@ global_page_index_remove(const struct heap_page *page) memmove(&g->page_index.pages[lo], &g->page_index.pages[lo + 1], (g->page_index.n_pages - lo - 1) * sizeof(struct heap_page *)); g->page_index.n_pages--; +} + +static void +global_page_index_remove(const struct heap_page *page) +{ + rb_global_objspace_t *g = global_objspace; + + rb_native_mutex_lock(&g->page_pool.lock); + global_page_index_remove_locked(page); rb_native_mutex_unlock(&g->page_pool.lock); } @@ -2287,6 +2306,36 @@ heap_page_free(rb_objspace_t *objspace, struct heap_page *page) free(page); } +static void +heap_pages_free_batch(rb_objspace_t *objspace, struct heap_page *pages) +{ + if (HEAP_PAGE_ALLOC_USE_MMAP) { +#ifdef HAVE_MMAP + rb_global_objspace_t *g = global_objspace; + + rb_native_mutex_lock(&g->page_pool.lock); + for (struct heap_page *page = pages; page != NULL; page = page->free_next) { + global_page_index_remove_locked(page); + page_pool_release_locked(page->body, page->arena); + } + rb_native_mutex_unlock(&g->page_pool.lock); +#endif + } + else { + for (struct heap_page *page = pages; page != NULL; page = page->free_next) { + global_page_index_remove(page); + heap_page_body_free(page->body, page->arena); + } + } + + while (pages != NULL) { + struct heap_page *next = pages->free_next; + objspace->heap_pages.freed_pages++; + free(pages); + pages = next; + } +} + static void heap_pages_free_unused_pages(rb_objspace_t *objspace) { @@ -2296,11 +2345,13 @@ heap_pages_free_unused_pages(rb_objspace_t *objspace) objspace->empty_pages_count = 0; size_t i, j; + struct heap_page *to_free = NULL; for (i = j = 0; i < rb_darray_size(objspace->heap_pages.sorted); i++) { struct heap_page *page = rb_darray_get(objspace->heap_pages.sorted, i); if (heap_page_in_global_empty_pages_pool(objspace, page) && heap_pages_freeable_pages > 0) { - heap_page_free(objspace, page); + page->free_next = to_free; + to_free = page; heap_pages_freeable_pages--; } else { @@ -2336,6 +2387,8 @@ heap_pages_free_unused_pages(rb_objspace_t *objspace) heap_pages_lomem = 0; heap_pages_himem = 0; } + + heap_pages_free_batch(objspace, to_free); } } @@ -2529,6 +2582,33 @@ page_pool_acquire(struct page_arena **arena_out) return body; } +#ifdef HAVE_MMAP +static void +page_pool_release_locked(struct heap_page_body *body, struct page_arena *arena) +{ + rb_global_objspace_t *g = global_objspace; + + ASSERT_PAGE_POOL_LOCKED(g); + + /* A body in the empty-pages pool stays fully poisoned (see gc_sweep_page), so + * unpoison the scratch area (link + arena tag) before writing. */ + asan_unpoison_memory_region(body, PAGE_POOL_SCRATCH_SIZE, false); + arena->free_count++; + PAGE_POOL_BODY_ARENA(body) = arena; + if (g->page_pool.hot_count < PAGE_POOL_HOT_MAX) { + *(uintptr_t *)body = (uintptr_t)g->page_pool.hot_list; + g->page_pool.hot_list = body; + g->page_pool.hot_count++; + } + else { + *(uintptr_t *)body = (uintptr_t)arena->cold_freelist; + arena->cold_freelist = body; + arena->cold_count++; + } + asan_poison_memory_region(body, HEAP_PAGE_SIZE); +} +#endif + static void page_pool_release(struct heap_page_body *body, struct page_arena *arena) { @@ -2537,22 +2617,7 @@ page_pool_release(struct heap_page_body *body, struct page_arena *arena) rb_global_objspace_t *g = global_objspace; rb_native_mutex_lock(&g->page_pool.lock); - /* A body in the empty-pages pool stays fully poisoned (see gc_sweep_page), so - * unpoison the scratch area (link + arena tag) before writing. */ - asan_unpoison_memory_region(body, PAGE_POOL_SCRATCH_SIZE, false); - arena->free_count++; - PAGE_POOL_BODY_ARENA(body) = arena; - if (g->page_pool.hot_count < PAGE_POOL_HOT_MAX) { - *(uintptr_t *)body = (uintptr_t)g->page_pool.hot_list; - g->page_pool.hot_list = body; - g->page_pool.hot_count++; - } - else { - *(uintptr_t *)body = (uintptr_t)arena->cold_freelist; - arena->cold_freelist = body; - arena->cold_count++; - } - asan_poison_memory_region(body, HEAP_PAGE_SIZE); + page_pool_release_locked(body, arena); rb_native_mutex_unlock(&g->page_pool.lock); #endif } From ac44788ff55622b4a69fb2f99f37d883daae25ee Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Thu, 3 Sep 2026 21:15:09 +0100 Subject: [PATCH 02/24] Use ERRORCHECK to assert the page pool mutex This commit initializes the page_pool.lock with PTHREAD_MUTEX_ERRORCHECK in RGENGC_CHECK_MODE builds and asserts that the the mutex lock is EDEADLK. --- gc/default/default.c | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/gc/default/default.c b/gc/default/default.c index 0125ea87d2bd0c..b89f1532aee054 100644 --- a/gc/default/default.c +++ b/gc/default/default.c @@ -857,12 +857,30 @@ static void page_pool_release_locked(struct heap_page_body *body, struct page_ar #endif static void page_pool_reclaim(rb_global_objspace_t *g); +#if RGENGC_CHECK_MODE && !defined(_WIN32) && !defined(__wasi__) && defined(HAVE_PTHREAD_H) +# define PAGE_POOL_LOCK_ERRORCHECK 1 +#endif + +static void +page_pool_lock_initialize(rb_nativethread_lock_t *lock) +{ +#ifdef PAGE_POOL_LOCK_ERRORCHECK + pthread_mutexattr_t attr; + pthread_mutexattr_init(&attr); + pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_ERRORCHECK); + pthread_mutex_init(lock, &attr); + pthread_mutexattr_destroy(&attr); +#else + rb_native_mutex_initialize(lock); +#endif +} + static void global_objspace_init(void) { if (global_objspace == NULL) { rb_global_objspace_t *g = &rb_global_objspace_instance; - rb_native_mutex_initialize(&g->page_pool.lock); + page_pool_lock_initialize(&g->page_pool.lock); g->page_pool.hot_list = NULL; g->page_pool.hot_count = 0; g->page_pool.arenas = NULL; @@ -2227,8 +2245,8 @@ heap_page_body_free(struct heap_page_body *page_body, struct page_arena *arena) page_pool_release(page_body, arena); } -#if RGENGC_CHECK_MODE && !defined(_WIN32) && !defined(__wasi__) -# define ASSERT_PAGE_POOL_LOCKED(g) GC_ASSERT(rb_native_mutex_trylock(&(g)->page_pool.lock) == EBUSY) +#ifdef PAGE_POOL_LOCK_ERRORCHECK +# define ASSERT_PAGE_POOL_LOCKED(g) GC_ASSERT(pthread_mutex_lock(&(g)->page_pool.lock) == EDEADLK) #else # define ASSERT_PAGE_POOL_LOCKED(g) ((void)0) #endif @@ -12493,7 +12511,7 @@ rb_gc_impl_after_fork(void *objspace_ptr, rb_pid_t pid) heap_alloc_state_clear(objspace); /* The forking Ractor becomes the child process's main Ractor. */ global_objspace->main_objspace = objspace; - rb_native_mutex_initialize(&rb_global_objspace_instance.page_pool.lock); + page_pool_lock_initialize(&rb_global_objspace_instance.page_pool.lock); } } From ada2be0b85b3980ac7a04af82fcf1cd351419497 Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Mon, 7 Sep 2026 11:41:43 +0100 Subject: [PATCH 03/24] Take the pool lock once for non-mmap batch page release heap_pages_free_batch locked page_pool.lock once per page on the non-mmap path, through global_page_index_remove. --- gc/default/default.c | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/gc/default/default.c b/gc/default/default.c index b89f1532aee054..a6afd9e65c97e9 100644 --- a/gc/default/default.c +++ b/gc/default/default.c @@ -2327,21 +2327,22 @@ heap_page_free(rb_objspace_t *objspace, struct heap_page *page) static void heap_pages_free_batch(rb_objspace_t *objspace, struct heap_page *pages) { - if (HEAP_PAGE_ALLOC_USE_MMAP) { -#ifdef HAVE_MMAP - rb_global_objspace_t *g = global_objspace; + rb_global_objspace_t *g = global_objspace; - rb_native_mutex_lock(&g->page_pool.lock); - for (struct heap_page *page = pages; page != NULL; page = page->free_next) { - global_page_index_remove_locked(page); + rb_native_mutex_lock(&g->page_pool.lock); + for (struct heap_page *page = pages; page != NULL; page = page->free_next) { + global_page_index_remove_locked(page); + if (HEAP_PAGE_ALLOC_USE_MMAP) { +#ifdef HAVE_MMAP page_pool_release_locked(page->body, page->arena); - } - rb_native_mutex_unlock(&g->page_pool.lock); #endif + } } - else { + rb_native_mutex_unlock(&g->page_pool.lock); + + if (!HEAP_PAGE_ALLOC_USE_MMAP) { + /* gc_aligned_free does not need the pool lock. */ for (struct heap_page *page = pages; page != NULL; page = page->free_next) { - global_page_index_remove(page); heap_page_body_free(page->body, page->arena); } } From b6830f5d2be2f96904fe123f333bd3d832d202af Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Mon, 7 Sep 2026 16:57:56 +0900 Subject: [PATCH 04/24] [ruby/rubygems] Reject Bundler redirects that downgrade https to http The redirect handling only compared hosts, so an https source redirecting to http on the same host had the request's user and password copied onto a plaintext connection. Gem::RemoteFetcher already refuses such a redirect. https://github.com/ruby/rubygems/commit/56e1b63333 Co-Authored-By: Claude Opus 5 --- lib/bundler/fetcher/downloader.rb | 9 +++++++++ spec/bundler/bundler/fetcher/downloader_spec.rb | 13 +++++++++++++ 2 files changed, 22 insertions(+) diff --git a/lib/bundler/fetcher/downloader.rb b/lib/bundler/fetcher/downloader.rb index 59e54c819de0bd..8050c0a3f2421b 100644 --- a/lib/bundler/fetcher/downloader.rb +++ b/lib/bundler/fetcher/downloader.rb @@ -51,6 +51,11 @@ def fetch(uri, headers = {}, counter = 0) response when Gem::Net::HTTPRedirection new_uri = Gem::URI.parse(response["location"]) + # Following a downgrade would put the credentials this request carries + # on a plaintext connection, so refuse it like Gem::RemoteFetcher does. + if https?(uri) && !https?(new_uri) + raise HTTPError, "Redirecting to a non-https URI is not allowed: #{URICredentialsFilter.credential_filtered_uri(new_uri)}" + end if new_uri.host == uri.host new_uri.user = uri.user new_uri.password = uri.password @@ -120,6 +125,10 @@ def network_down_error(uri, filtered_uri) "connection and try again.") end + def https?(uri) + uri.scheme&.casecmp("https")&.zero? + end + def validate_uri_scheme!(uri) return if /\Ahttps?\z/.match?(uri.scheme) raise InvalidOption, diff --git a/spec/bundler/bundler/fetcher/downloader_spec.rb b/spec/bundler/bundler/fetcher/downloader_spec.rb index a3390ac5276fc0..a7f642f456e67e 100644 --- a/spec/bundler/bundler/fetcher/downloader_spec.rb +++ b/spec/bundler/bundler/fetcher/downloader_spec.rb @@ -59,6 +59,19 @@ subject.fetch(uri, options, counter) end end + + context "when the redirect uri downgrades https to http" do + let(:uri) { Gem::URI("https://username:password@www.uri-to-fetch.com/api/v2/endpoint") } + + before { http_response["location"] = "http://www.uri-to-fetch.com/api/v2/endpoint" } + + it "should raise a Bundler::HTTPError instead of following the redirect" do + expect(subject).to receive(:fetch).with(uri, options, 0).and_call_original + expect(subject).not_to receive(:fetch).with(Gem::URI("http://username:password@www.uri-to-fetch.com/api/v2/endpoint"), options, 1) + expect { subject.fetch(uri, options, counter) }.to raise_error(Bundler::HTTPError, + "Redirecting to a non-https URI is not allowed: http://www.uri-to-fetch.com/api/v2/endpoint") + end + end end context "when the request response is a Gem::Net::HTTPSuccess" do From db84d039d083096d2d13b5c4feff858d3239045d Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Tue, 8 Sep 2026 08:00:33 +0900 Subject: [PATCH 05/24] [Bug #19383] Fix Time#zone encoding when TZ is set on Windows CRT _tzname is encoded in the active code page, while zone_str() assumed the UTF-8 converted name from GetDynamicTimeZoneInformation. Tag strings from the _tzname fallback with the encoding of GetACP(), and stop asserting the locale encoding on Windows, where the assertion restated the bug. Co-authored-by: Nobuyoshi Nakada --- test/ruby/test_time.rb | 4 +++- time.c | 23 ++++++++++++++++++++--- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/test/ruby/test_time.rb b/test/ruby/test_time.rb index 91bf4f763b1233..fb9864c70e18ce 100644 --- a/test/ruby/test_time.rb +++ b/test/ruby/test_time.rb @@ -722,7 +722,9 @@ def assert_zone_encoding(time) assert_predicate(zone, :valid_encoding?) if zone.ascii_only? assert_equal(Encoding::US_ASCII, zone.encoding) - else + elsif !/mswin|mingw/.match?(RUBY_PLATFORM) + # Windows takes the name from the CRT, which encodes it in the + # active code page rather than the locale one. [Bug #19383] enc = Encoding.default_internal || Encoding.find('locale') assert_equal(enc, zone.encoding) end diff --git a/time.c b/time.c index f57b68cb5d3360..32b7048d8f68fc 100644 --- a/time.c +++ b/time.c @@ -739,8 +739,9 @@ static struct { } w32_tz; static char * -get_tzname(int dst) +get_tzname(int dst, rb_encoding **enc) { + *enc = NULL; if (w32_tz.use_tzkey) { if (w32_tz.name[0]) { return w32_tz.name; @@ -765,6 +766,11 @@ get_tzname(int dst) } } } + /* CRT timezone names are encoded in the active code page, which + * may differ from the locale (console) code page */ + char cp[(sizeof(UINT) * 8 / 3) + 4]; + snprintf(cp, sizeof(cp), "CP%u", GetACP()); + *enc = rb_enc_find(cp); return _tzname[_daylight && dst]; } #endif @@ -1002,7 +1008,7 @@ timegmw_noleapsecond(struct vtm *vtm) } static VALUE -zone_str(const char *zone) +zone_str_enc(const char *zone, rb_encoding *enc) { const char *p; int ascii_only = 1; @@ -1023,6 +1029,9 @@ zone_str(const char *zone) if (ascii_only) { return rb_enc_interned_str(zone, len, rb_usascii_encoding()); } + else if (enc) { + return rb_enc_interned_str(zone, len, enc); + } else { #ifdef _WIN32 VALUE str = rb_utf8_str_new(zone, len); @@ -1035,6 +1044,12 @@ zone_str(const char *zone) } } +static inline VALUE +zone_str(const char *zone) +{ + return zone_str_enc(zone, NULL); +} + static void gmtimew_noleapsecond(wideval_t timew, struct vtm *vtm) { @@ -1730,7 +1745,9 @@ localtime_with_gmtoff_zone(const time_t *t, struct tm *result, long *gmtoff, VAL #if defined(HAVE_TM_ZONE) *zone = zone_str(tm.tm_zone); #elif defined(_WIN32) - *zone = zone_str(get_tzname(tm.tm_isdst)); + rb_encoding *enc; + const char *name = get_tzname(tm.tm_isdst, &enc); + *zone = zone_str_enc(name, enc); #elif defined(HAVE_TZNAME) && defined(HAVE_DAYLIGHT) /* this needs tzset or localtime, instead of localtime_r */ *zone = zone_str(tzname[daylight && tm.tm_isdst]); From 3f2e039ca7290940e03e6b4e09f68481f98fef55 Mon Sep 17 00:00:00 2001 From: BurdetteLamar Date: Mon, 7 Sep 2026 17:10:33 -0500 Subject: [PATCH 06/24] [DOC] Tweaks for File::stat and File::lstat --- file.c | 52 ++++++++++++++++++++++++++++++++++------------------ 1 file changed, 34 insertions(+), 18 deletions(-) diff --git a/file.c b/file.c index 167cc32933b823..7f0967e5c18719 100644 --- a/file.c +++ b/file.c @@ -1505,12 +1505,28 @@ rb_stat(VALUE file, struct stat *st) } /* + * :markup: markdown + * * call-seq: - * File.stat(filepath) -> stat + * File.stat(path) -> file_stat * - * Returns a File::Stat object for the file at +filepath+ (see File::Stat): + * Returns a new File::Stat object for the entry at `path`. + * Follows [symbolic links](file/symbolic_links.md); + * therefore if the entry is a symbolic link, + * the returned object contains information for the target entry, not the symbolic link: * - * File.stat('t.txt').class # => File::Stat + * ```ruby + * filepath = 'README.md' + * linkpath = 'foo' + * File.symlink(filepath, linkpath) + * # Method File.stat follows the symlink, so the birthtimes are the same. + * File.stat(filepath).birthtime # => 2026-09-01 09:09:28.378987388 -0500 + * File.stat(linkpath).birthtime # => 2026-09-01 09:09:28.378987388 -0500 + * # Method File.lstat does not follow the symlink, so the birthtimes are different. + * File.lstat(filepath).birthtime # => 2026-09-01 09:09:28.378987388 -0500 + * File.lstat(linkpath).birthtime # => 2026-09-04 10:29:45.884317953 -0500 + * File.unlink(linkpath) # Clean up. + * ``` * */ @@ -1579,24 +1595,24 @@ lstat_without_gvl(const char *path, struct stat *st) * :markup: markdown * * call-seq: - * File.lstat(path) -> new_stat + * File.lstat(path) -> file_stat * - * Returns a File::Stat object for the entry at `path`; - * does not follow symbolic links, - * and therefore returns the stat object for `path`, - * regardless of whether it is a symbolic link: + * Returns a new File::Stat object for the entry at `path`. + * Does not follow [symbolic links](file/symbolic_links.md); + * therefore the returned object contains information for the entry at `path`, + * regardless of whether is a symbolic link: * * ```ruby - * File.write('t.tmp', '') - * sleep(1) - * File.symlink('t.tmp', 'link') - * file = File.new('link', 'r') - * # Method stat: follows link to 't.tmp'. - * file.stat.ctime # => 2026-06-13 15:05:16.996527996 -0500 - * # Method lstat; does not follow link. - * file.lstat.ctime # => 2026-06-13 15:05:17.997527947 -0500 - * File.delete('t.tmp') - * File.delete('link') + * filepath = 'README.md' + * linkpath = 'foo' + * File.symlink(filepath, linkpath) + * # Method File.stat follows the symlink, so the birthtimes are the same. + * File.stat(filepath).birthtime # => 2026-09-01 09:09:28.378987388 -0500 + * File.stat(linkpath).birthtime # => 2026-09-01 09:09:28.378987388 -0500 + * # Method File.lstat does not follow the symlink, so the birthtimes are different. + * File.lstat(filepath).birthtime # => 2026-09-01 09:09:28.378987388 -0500 + * File.lstat(linkpath).birthtime # => 2026-09-04 10:29:45.884317953 -0500 + * File.unlink(linkpath) # Clean up. * ``` * */ From b6ed8bb1bef20008b9562d3c1fae09bc5c7db3f8 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Fri, 4 Sep 2026 08:40:57 +0900 Subject: [PATCH 07/24] [Bug #21697] Keep revision.h in a source tree without VCS `update-src` truncated revision.h to force the regeneration, but the blank file also defeated the guard in file2lastrev.rb that keeps the existing content when no VCS is available, so `make up` on a released tarball dropped `RUBY_REVISION` and `RUBY_FULL_REVISION`. Removing the timestamp alone is enough to force the regeneration. Co-Authored-By: Claude Opus 5 --- common.mk | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/common.mk b/common.mk index 7b63cbd96db34a..4fbd986118dbf6 100644 --- a/common.mk +++ b/common.mk @@ -1520,9 +1520,10 @@ after-update:: extract-extlibs after-update:: extract-gems after-update:: update-default-gemspecs +# Do not remove or empty revision.h itself, whose content file2lastrev.rb +# keeps when the source tree has no VCS. update-src:: - $(Q) $(RM) $(REVISION_H) revision.h "$(srcdir)/$(REVISION_H)" "$(srcdir)/revision.h" - $(Q) exit > "$(srcdir)/revision.h" + $(Q) $(RM) $(REVISION_H) "$(srcdir)/$(REVISION_H)" # $(REVISION_H) can have been made already in this run, as a prerequisite # of the included dependency file, and make does not make it twice. From 3c2856d9368f618a38cb7b86d7ac7c8006742d6b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 8 Sep 2026 02:12:41 +0000 Subject: [PATCH 08/24] Bump taiki-e/install-action Bumps the github-actions group with 1 update in the / directory: [taiki-e/install-action](https://github.com/taiki-e/install-action). Updates `taiki-e/install-action` from 2.87.5 to 2.87.6 - [Release notes](https://github.com/taiki-e/install-action/releases) - [Changelog](https://github.com/taiki-e/install-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/taiki-e/install-action/compare/5bf6ce016fd2e72eefc647cbca1e4213f65955b8...7b8d4719ee4aaa279bdf55df38dacb9ebfe12a6c) --- updated-dependencies: - dependency-name: taiki-e/install-action dependency-version: 2.87.6 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: github-actions ... Signed-off-by: dependabot[bot] --- .github/workflows/zjit-macos.yml | 2 +- .github/workflows/zjit-ubuntu.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/zjit-macos.yml b/.github/workflows/zjit-macos.yml index 72b3c200edb4fb..648c642bd7ccbb 100644 --- a/.github/workflows/zjit-macos.yml +++ b/.github/workflows/zjit-macos.yml @@ -98,7 +98,7 @@ jobs: rustup install ${{ matrix.rust_version }} --profile minimal rustup default ${{ matrix.rust_version }} - - uses: taiki-e/install-action@5bf6ce016fd2e72eefc647cbca1e4213f65955b8 # v2.87.5 + - uses: taiki-e/install-action@7b8d4719ee4aaa279bdf55df38dacb9ebfe12a6c # v2.87.6 with: tool: nextest@0.9 if: ${{ matrix.test_task == 'zjit-check' }} diff --git a/.github/workflows/zjit-ubuntu.yml b/.github/workflows/zjit-ubuntu.yml index b0475e62dab611..15427f8b8a5130 100644 --- a/.github/workflows/zjit-ubuntu.yml +++ b/.github/workflows/zjit-ubuntu.yml @@ -152,7 +152,7 @@ jobs: ruby-version: '3.1' bundler: none - - uses: taiki-e/install-action@5bf6ce016fd2e72eefc647cbca1e4213f65955b8 # v2.87.5 + - uses: taiki-e/install-action@7b8d4719ee4aaa279bdf55df38dacb9ebfe12a6c # v2.87.6 with: tool: nextest@0.9 if: ${{ matrix.test_task == 'zjit-check' }} From c0fdf812767c85484db613c92509959404867d3f Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Mon, 7 Sep 2026 16:49:35 +0900 Subject: [PATCH 09/24] Check struct size when a user class relabels a loaded struct TYPE_UCLASS only compared the built-in type before RBASIC_SET_CLASS, so a struct could be relabelled as a struct class with a different number of members, and its accessors then read and wrote past the allocated slots. Apply the member count check that TYPE_STRUCT already has. Co-Authored-By: Claude Opus 5 --- marshal.c | 5 +++++ test/ruby/test_marshal.rb | 13 +++++++++++++ 2 files changed, 18 insertions(+) diff --git a/marshal.c b/marshal.c index a9fd5aeb918d47..ec081e78b9e6b5 100644 --- a/marshal.c +++ b/marshal.c @@ -1971,6 +1971,11 @@ r_object_for(struct load_arg *arg, bool partial, int *ivp, VALUE klass, VALUE ex if (TYPE(v) != TYPE(tmp)) goto format_error; } + if (RB_TYPE_P(v, T_STRUCT) && + RSTRUCT_LEN_RAW(v) != RARRAY_LEN(rb_struct_s_members(c))) { + rb_raise(rb_eTypeError, "struct %"PRIsVALUE" not compatible (struct size differs)", + rb_class_name(c)); + } RBASIC_SET_CLASS(v, c); } break; diff --git a/test/ruby/test_marshal.rb b/test/ruby/test_marshal.rb index b7e40cb2d3600d..d3654a59e7be1a 100644 --- a/test/ruby/test_marshal.rb +++ b/test/ruby/test_marshal.rb @@ -212,6 +212,19 @@ def test_change_struct self.class.__send__(:remove_const, :C3) if self.class.const_defined?(:C3) end + UserClassWideStruct = Struct.new(:a, :b, :c, :d, :e) + UserClassNarrowStruct = Struct.new(:a) + + def test_user_class_struct_size + wide = Marshal.dump(UserClassWideStruct.new(1, 2, 3, 4, 5)).byteslice(2..) + narrow = Marshal.dump(UserClassNarrowStruct.new(6)).byteslice(2..) + # An array of both structs, then a user class marker relabelling the + # narrow struct (object link "@\a") as the wide class (symbol link ";\0"). + data = "\x04\b[\b".b + wide + narrow + "C;\0@\a".b + message = /\Astruct #{UserClassWideStruct.name} not compatible \(struct size differs\)/ + assert_raise_with_message(TypeError, message) {Marshal.load(data)} + end + class C4 def initialize(gc) @gc = gc From 21bc49cf7358c2f0540624e534e0fe81a1b12a16 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Tue, 8 Sep 2026 08:27:55 +0900 Subject: [PATCH 10/24] [ruby/net-http] Require chunked to be the final transfer coding Net::HTTPHeader#chunked? matched the token anywhere in Transfer-Encoding, so a response carrying `chunked, gzip` was chunk-framed and the bytes after the terminating chunk were left unread on the connection. See RFC 9112 Section 6.3: https://www.rfc-editor.org/rfc/rfc9112.html#section-6.3 If a Transfer-Encoding header field is present in a response and the chunked transfer coding is not the final encoding, the message body length is determined by reading the connection until it is closed by the server. https://github.com/ruby/net-http/commit/e70762aaf9 Co-Authored-By: Claude Opus 5 --- lib/net/http/header.rb | 7 +++-- test/net/http/test_httpheader.rb | 10 +++++++ test/net/http/test_httpresponse.rb | 46 ++++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 2 deletions(-) diff --git a/lib/net/http/header.rb b/lib/net/http/header.rb index 291cc333819df7..a468827bd5d206 100644 --- a/lib/net/http/header.rb +++ b/lib/net/http/header.rb @@ -685,7 +685,7 @@ def content_length=(len) end # Returns +true+ if field 'Transfer-Encoding' - # exists and has value 'chunked', + # exists and has 'chunked' as its final transfer coding, # +false+ otherwise; # see {Transfer-Encoding response header}[https://en.wikipedia.org/wiki/List_of_HTTP_header_fields#transfer-encoding-response-header]: # @@ -693,10 +693,13 @@ def content_length=(len) # res['Transfer-Encoding'] # => "chunked" # res.chunked? # => true # + # 'gzip, chunked' is chunked but 'chunked, gzip' is not, + # because only the final transfer coding frames the message. + # See {RFC 9112 Section 6.3}[https://www.rfc-editor.org/rfc/rfc9112.html#section-6.3]. def chunked? return false unless @header['transfer-encoding'] field = self['Transfer-Encoding'] - (/(?:\A|[^\-\w])chunked(?![\-\w])/i =~ field) ? true : false + (/(?:\A|[^\-\w])chunked(?![\-\w])\s*(?:,\s*)*\z/i =~ field) ? true : false end # Returns a Range object representing the value of field diff --git a/test/net/http/test_httpheader.rb b/test/net/http/test_httpheader.rb index 3c09f4c194a41e..b86d48681e1cb5 100644 --- a/test/net/http/test_httpheader.rb +++ b/test/net/http/test_httpheader.rb @@ -384,6 +384,16 @@ def test_chunked? try_chunked false, 'chunked-but-not-chunked' end + def test_chunked_final_transfer_coding + try_chunked true, 'gzip, chunked' + try_chunked true, 'chunked,' + try_chunked true, 'gzip , chunked , ' + + try_chunked false, 'chunked, gzip' + try_chunked false, 'chunked, identity' + try_chunked false, 'gzip' + end + def try_chunked(bool, str) @c['transfer-encoding'] = str assert_equal bool, @c.chunked? diff --git a/test/net/http/test_httpresponse.rb b/test/net/http/test_httpresponse.rb index e9ed1213516210..e22bc229862ac0 100644 --- a/test/net/http/test_httpresponse.rb +++ b/test/net/http/test_httpresponse.rb @@ -113,6 +113,52 @@ def test_read_body assert_equal 'hello', body end + def test_read_body_chunked_is_final_transfer_coding + io = dummy_io(< Date: Tue, 8 Sep 2026 08:28:42 +0900 Subject: [PATCH 11/24] [ruby/net-http] Let Transfer-Encoding override Content-Length Once chunked is not the final transfer coding, read_body_0 fell through to Content-Length and framed the body by it, which leaves the client and the server disagreeing about where the response ends. See RFC 9112 Section 6.3: https://www.rfc-editor.org/rfc/rfc9112.html#section-6.3 If a message is received with both a Transfer-Encoding and a Content-Length header field, the Transfer-Encoding overrides the Content-Length. https://github.com/ruby/net-http/commit/9fb5b93b52 Co-Authored-By: Claude Opus 5 --- lib/net/http/response.rb | 23 ++++++++++++++--------- test/net/http/test_httpresponse.rb | 21 +++++++++++++++++++++ 2 files changed, 35 insertions(+), 9 deletions(-) diff --git a/lib/net/http/response.rb b/lib/net/http/response.rb index db719161752e91..4395e11188810f 100644 --- a/lib/net/http/response.rb +++ b/lib/net/http/response.rb @@ -616,15 +616,20 @@ def read_body_0(dest) @socket = inflate_body_io - clen = content_length() - if clen - @socket.read clen, dest, @ignore_eof - return - end - clen = range_length() - if clen - @socket.read clen, dest - return + # Transfer-Encoding overrides Content-Length, so a body that is not + # chunk-framed runs until the server closes the connection. + # See RFC 9112 Section 6.3. + unless @header['transfer-encoding'] + clen = content_length() + if clen + @socket.read clen, dest, @ignore_eof + return + end + clen = range_length() + if clen + @socket.read clen, dest + return + end end @socket.read_all dest end diff --git a/test/net/http/test_httpresponse.rb b/test/net/http/test_httpresponse.rb index e22bc229862ac0..e845f80fb5af25 100644 --- a/test/net/http/test_httpresponse.rb +++ b/test/net/http/test_httpresponse.rb @@ -159,6 +159,27 @@ def test_read_body_chunked_is_not_final_transfer_coding assert_equal "5\r\nhello\r\n0\r\n\r\n", body end + def test_read_body_transfer_encoding_overrides_content_length + io = dummy_io(< Date: Mon, 7 Sep 2026 05:19:50 +0000 Subject: [PATCH 12/24] Wait for terminating Ractors through the scheduler rb_ractor_terminate_all() interrupts the other Ractors and waits on a condvar of its own for them to finish, dropping the VM lock for the wait and taking it back after. That was rb_vm_cond_timedwait(), whose only caller it was; rb_vm_cond_wait() had none at all. Move the wait to rb_ractor_sched_wait_terminate() in thread_sched.c and give the native thread back for its duration, as a blocking region does. Waiting natively holds whatever native thread the caller runs on, and an M:N caller runs on one from the shared pool -- so it would wait, holding the pool, for Ractors that need the pool to finish. It does not happen today, the caller being the main Ractor's main thread with a native thread of its own, but a scheduler-blind wait in the scheduler's way is worth keeping out of vm_sync.c. Co-Authored-By: Claude Opus 5 --- ractor.c | 2 +- thread_none.c | 6 ++++++ thread_sched.c | 25 +++++++++++++++++++++++++ thread_sched.h | 1 + vm_core.h | 4 ---- vm_sync.c | 31 ------------------------------- 6 files changed, 33 insertions(+), 36 deletions(-) diff --git a/ractor.c b/ractor.c index 5311c4b96ba861..6d60d6e6af6869 100644 --- a/ractor.c +++ b/ractor.c @@ -1181,7 +1181,7 @@ rb_ractor_terminate_all(void) // wait for 1sec rb_vm_ractor_blocking_cnt_inc(vm, cr, __FILE__, __LINE__); rb_del_running_thread(rb_ec_thread_ptr(cr->threads.running_ec)); - rb_vm_cond_timedwait(vm, &vm->ractor.sync.terminate_cond, 1000 /* ms */); + rb_ractor_sched_wait_terminate(vm, &vm->ractor.sync.terminate_cond, 1000 /* ms */); while (vm->ractor.sched.barrier_is_waiting) { // A barrier is waiting. Threads relinquish the VM lock before joining the barrier and // since we just acquired the VM lock back, we're blocking other threads from joining it. diff --git a/thread_none.c b/thread_none.c index 7de59b6f9af6f2..58f43796eb06d4 100644 --- a/thread_none.c +++ b/thread_none.c @@ -291,6 +291,12 @@ rb_del_running_thread(rb_thread_t *th) // do nothing } +void +rb_ractor_sched_wait_terminate(rb_vm_t *vm, rb_nativethread_cond_t *cond, unsigned long msec) +{ + // do nothing: with no threads there is no other Ractor to wait for +} + void rb_threadptr_sched_free(rb_thread_t *th) { diff --git a/thread_sched.c b/thread_sched.c index a7d89cc1316eef..5cdd33c90e4910 100644 --- a/thread_sched.c +++ b/thread_sched.c @@ -2469,6 +2469,31 @@ rb_thread_lock_native_thread(void) return is_snt; } +// rb_ractor_terminate_all() waits for the Ractors it interrupted on a condvar +// of its own, holding the VM lock, which it drops for the wait and takes back +// after. An M:N thread would hold the shared native thread it runs on for the +// whole wait as well, leaving those Ractors with nothing to run on, so give +// that native thread back the way a blocking region does. +void +rb_ractor_sched_wait_terminate(rb_vm_t *vm, rb_nativethread_cond_t *cond, unsigned long msec) +{ + ASSERT_vm_locking(); + + rb_thread_t *th = GET_THREAD(); + unsigned int lock_rec = vm->ractor.sync.lock_rec; + rb_ractor_t *lock_owner = vm->ractor.sync.lock_owner; + + vm->ractor.sync.lock_rec = 0; + vm->ractor.sync.lock_owner = NULL; + + native_thread_dedicated_inc(vm, th->ractor, th->nt); + rb_native_cond_timedwait(cond, &vm->ractor.sync.lock, msec); + native_thread_dedicated_dec(vm, th->ractor, th->nt); + + vm->ractor.sync.lock_rec = lock_rec; + vm->ractor.sync.lock_owner = lock_owner; +} + void rb_thread_malloc_stack_set(rb_thread_t *th, void *stack, size_t stack_size) { diff --git a/thread_sched.h b/thread_sched.h index 63ca7d003b6204..e56f5d7dd19c32 100644 --- a/thread_sched.h +++ b/thread_sched.h @@ -262,6 +262,7 @@ struct rb_ractor_sched { void rb_ractor_sched_wait(struct rb_execution_context_struct *ec, struct rb_ractor_struct *cr, rb_unblock_function_t *ptr, void *arg); void rb_ractor_sched_wakeup(struct rb_ractor_struct *r, struct rb_thread_struct *th); +void rb_ractor_sched_wait_terminate(struct rb_vm_struct *vm, rb_nativethread_cond_t *cond, unsigned long msec); void rb_thread_wake_fence(struct rb_thread_struct *th); #endif /* RUBY_THREAD_SCHED_H */ diff --git a/vm_core.h b/vm_core.h index 8843b7ac124ead..75d6323236169a 100644 --- a/vm_core.h +++ b/vm_core.h @@ -2354,10 +2354,6 @@ void rb_fiber_close(rb_fiber_t *fib); void Init_native_thread(rb_thread_t *th); int rb_vm_check_ints_blocking(rb_execution_context_t *ec); -// vm_sync.h -void rb_vm_cond_wait(rb_vm_t *vm, rb_nativethread_cond_t *cond); -void rb_vm_cond_timedwait(rb_vm_t *vm, rb_nativethread_cond_t *cond, unsigned long msec); - #define RUBY_VM_CHECK_INTS(ec) rb_vm_check_ints(ec) static inline void rb_vm_check_ints(rb_execution_context_t *ec) diff --git a/vm_sync.c b/vm_sync.c index 7b6a587a1442f6..73335699a01626 100644 --- a/vm_sync.c +++ b/vm_sync.c @@ -214,37 +214,6 @@ rb_vm_unlock_body(LOCATION_ARGS) vm_lock_leave(vm, false, &vm->ractor.sync.lock_rec APPEND_LOCATION_PARAMS); } -static void -vm_cond_wait(rb_vm_t *vm, rb_nativethread_cond_t *cond, unsigned long msec) -{ - ASSERT_vm_locking(); - unsigned int lock_rec = vm->ractor.sync.lock_rec; - rb_ractor_t *cr = vm->ractor.sync.lock_owner; - - vm->ractor.sync.lock_rec = 0; - vm->ractor.sync.lock_owner = NULL; - if (msec > 0) { - rb_native_cond_timedwait(cond, &vm->ractor.sync.lock, msec); - } - else { - rb_native_cond_wait(cond, &vm->ractor.sync.lock); - } - vm->ractor.sync.lock_rec = lock_rec; - vm->ractor.sync.lock_owner = cr; -} - -void -rb_vm_cond_wait(rb_vm_t *vm, rb_nativethread_cond_t *cond) -{ - vm_cond_wait(vm, cond, 0); -} - -void -rb_vm_cond_timedwait(rb_vm_t *vm, rb_nativethread_cond_t *cond, unsigned long msec) -{ - vm_cond_wait(vm, cond, msec); -} - static bool vm_barrier_acquired_p(const rb_vm_t *vm, const rb_ractor_t *cr) { From ce6f2009724406081d409308bdce3eacd5f91db2 Mon Sep 17 00:00:00 2001 From: Koichi Sasada Date: Mon, 7 Sep 2026 04:49:41 +0000 Subject: [PATCH 13/24] RUBY_MN_THREADS=2: run the main thread as an M:N thread too `RUBY_MN_THREADS=1` only affects threads created after it is read: the main thread keeps a native thread of its own and is woken through its condvar, so handing control to the main thread costs an order of magnitude more than handing it between any two other threads (6.4us against 0.56us for a Queue round trip here). With `RUBY_MN_THREADS=2` the running main thread is turned into an M:N thread in place (thread_sched_main_to_shared), rather than spawning a thread for the program body and leaving the main native thread idle: * the process stack becomes the main thread's coroutine context (coroutine_initialize_main), the way nt_start's own stack is a shared thread's context; * the main native thread joins the shared pool with a stack of its own for its scheduling loop (nt_loop_co), which starts on the first transfer into it -- the main thread's first park. From then on any shared native thread may resume the main thread, and this one serves any Ractor. nt_start's shared loop is split out as nt_shared_loop so that both entries share it. The main native thread never retires, its loop being on a stack only it could free: ractor_sched_deq takes a can_retire flag and native_thread_dedicated_dec always lets it rejoin. The wait in rb_ractor_terminate_all had to give its native thread back first, or exit hung with the Ractors it waits for having nothing to run on; that is the commit before this one. The VM barrier waits natively in the same way and has the same exposure, and is not touched here, being no more reachable under `=2` than before. Several places assumed the main thread owns the process's initial native thread: * native_thread_init_stack set the process stack's range for any thread starting on that native thread, over the pool stack an M:N thread already owns; the case is now keyed on th->sched.context. * thread_sched_switch passed to_dead for a thread whose status is THREAD_KILLED, which the main thread already is while rb_ractor_terminate_all parks it. A thread ends only through co_start's epilogue transfer. * thread_sched_atfork left the forker's nt->running_th and nt->retiring as the parent had them; retire eligibility is decided against the process's main native thread, refreshed there. * rb_thread_free_native_thread (RUBY_FREE_AT_EXIT) destroyed the native thread hosting the main thread even when shared, along with the altstack registered on the thread running it. RUBY_FREE_AT_EXIT now leaves more behind: the main native thread and its context and stack, since a thread is parked in its loop until the process ends, and the main thread's own coroutine context, since the free-at-exit path does not reach rb_threadptr_sched_free. Leak-checker baselines move by that much. The OS thread name is no longer set from the Ruby thread for M:N threads. One shared native thread runs many Ruby threads over its life, so the name described whichever one happened to start on it; under `=2` that renamed the process itself, the thread group leader's comm being what ps and pkill show. Thread#name= was already skipped for M:N threads for the same reason. Co-Authored-By: Claude Opus 5 --- thread.c | 7 +- thread_pthread.c | 53 +++++-- thread_sched.c | 352 +++++++++++++++++++++++++++++++++-------------- thread_win32.c | 6 + 4 files changed, 306 insertions(+), 112 deletions(-) diff --git a/thread.c b/thread.c index 4736a8cd67ae75..8310529a38b941 100644 --- a/thread.c +++ b/thread.c @@ -573,7 +573,12 @@ rb_thread_free_native_thread(void *th_ptr) { rb_thread_t *th = th_ptr; - native_thread_destroy_atfork(th->nt); + // A thread with a coroutine context does not own its native thread: that + // one is in the shared pool, listed there and with its altstack registered + // on whichever pthread is running this. See rb_threadptr_sched_free(). + if (th->sched.context == NULL) { + native_thread_destroy_atfork(th->nt); + } th->nt = NULL; } diff --git a/thread_pthread.c b/thread_pthread.c index 8f6a7b11667b4b..0aaaf5de6f852f 100644 --- a/thread_pthread.c +++ b/thread_pthread.c @@ -637,6 +637,29 @@ static struct { VALUE *stack_start; } native_main_thread; +// Whether the calling native thread may leave the shared pool. The process's +// main pthread cannot: under RUBY_MN_THREADS=2 its loop runs on a stack only +// it could free (thread_sched_main_to_shared). By pthread_self(), not +// nt->thread_id, which pthread_create may still be writing. +static bool +native_thread_self_can_retire_p(void) +{ + return !pthread_equal(pthread_self(), native_main_thread.id); +} + +#if defined(HAVE_WORKING_FORK) +// The forking thread's pthread is the child's only one, hence its main one +static void +native_main_thread_atfork(void) +{ + native_main_thread.id = pthread_self(); + // The stack recorded here is the parent's initial one; this thread's is + // another. Nothing reads it for a thread that is already running, and + // saying "unknown" beats saying the wrong bounds. + native_main_thread.stack_maxsize = 0; +} +#endif + #ifdef STACK_END_ADDRESS extern void *STACK_END_ADDRESS; #endif @@ -739,21 +762,24 @@ native_thread_init_stack(rb_thread_t *th, void *local_in_parent_frame) native_thread_init_main_thread_stack(local_in_parent_frame); } - if (pthread_equal(curr, native_main_thread.id)) { + if (th->sched.context != NULL) { + // an M:N thread runs on the pool stack native_thread_create_shared + // recorded, whichever native thread (the main one included) hosts it. + // Not by nt->dedicated: a RESUMED hook may have pinned the nt already. + } + else if (pthread_equal(curr, native_main_thread.id)) { th->ec->machine.stack_start = native_main_thread.stack_start; th->ec->machine.stack_maxsize = native_main_thread.stack_maxsize; } else { #ifdef STACKADDR_AVAILABLE - if (th_has_dedicated_nt(th)) { - void *start; - size_t size; - - if (get_stack(&start, &size) == 0) { - uintptr_t diff = (uintptr_t)start - (uintptr_t)local_in_parent_frame; - th->ec->machine.stack_start = local_in_parent_frame; - th->ec->machine.stack_maxsize = size - diff; - } + void *start; + size_t size; + + if (get_stack(&start, &size) == 0) { + uintptr_t diff = (uintptr_t)start - (uintptr_t)local_in_parent_frame; + th->ec->machine.stack_start = local_in_parent_frame; + th->ec->machine.stack_maxsize = size - diff; } #else rb_raise(rb_eNotImpError, "ruby engine can initialize only in the main thread"); @@ -1031,6 +1057,13 @@ native_set_thread_name(rb_thread_t *th) { #ifdef SET_CURRENT_THREAD_NAME VALUE loc; + + // An M:N thread does not own the native thread it runs on: naming it here + // would name whichever nt started it (under RUBY_MN_THREADS=2 that can be + // the process's main one, whose name is the process's). Thread#name= is + // skipped for the same reason (rb_thread_setname). + if (!th->has_dedicated_nt) return; + if (!NIL_P(loc = th->name)) { SET_CURRENT_THREAD_NAME(RSTRING_PTR(loc)); } diff --git a/thread_sched.c b/thread_sched.c index 5cdd33c90e4910..dffc59a60f8af7 100644 --- a/thread_sched.c +++ b/thread_sched.c @@ -104,9 +104,17 @@ static bool timeslice_scan(rb_vm_t *vm, bool interrupt); static void timer_thread_wakeup(void); static void timer_thread_wakeup_locked(rb_vm_t *vm); static void timer_thread_wakeup_force(void); +static void nt_snts_join(rb_vm_t *vm, struct rb_native_thread *nt); +static void nt_snts_leave(rb_vm_t *vm, struct rb_native_thread *nt); +static bool nt_shared_loop(struct rb_native_thread *nt); +static bool native_thread_self_can_retire_p(void); #include THREAD_IMPL_SRC +#if USE_MN_THREADS +static void thread_sched_main_to_shared(rb_thread_t *th); +#endif + // Defaults for what the platform above did not opt out of. #ifndef RB_NATIVE_MUTEX_TRYLOCK_DETECTS_SELF @@ -1206,7 +1214,9 @@ thread_sched_switch(rb_thread_t *cth, rb_thread_t *next_th) struct rb_native_thread *nt = cth->nt; native_thread_assign(NULL, cth); RUBY_DEBUG_LOG("th:%u->%u on nt:%d", rb_th_serial(cth), rb_th_serial(next_th), nt->serial); - thread_sched_switch0(cth->sched.context, next_th, nt, cth->status == THREAD_KILLED); + // never final: a thread ends only through co_start's epilogue transfer. The + // main thread is THREAD_KILLED (rb_ec_cleanup) while it still parks here. + thread_sched_switch0(cth->sched.context, next_th, nt, false); } #if VM_CHECK_MODE > 0 @@ -1344,7 +1354,7 @@ ractor_sched_enq(rb_vm_t *vm, rb_ractor_t *r) #define SNT_KEEP_MINIMUM (MINIMUM_SNT > 1 ? MINIMUM_SNT : 1) static rb_ractor_t * -ractor_sched_deq(rb_vm_t *vm, rb_ractor_t *cr) +ractor_sched_deq(rb_vm_t *vm, rb_ractor_t *cr, bool can_retire) { rb_ractor_t *r; int idle_streak = 0; // consecutive pops that found the queue empty @@ -1360,7 +1370,7 @@ ractor_sched_deq(rb_vm_t *vm, rb_ractor_t *cr) while ((r = ccan_list_pop(&vm->ractor.sched.grq, rb_ractor_t, threads.sched.grq_node)) == NULL) { RUBY_DEBUG_LOG("wait grq_cnt:%d", (int)vm->ractor.sched.grq_cnt); - if (SNT_IDLE_RETIRE >= 0 && ++idle_streak > SNT_IDLE_RETIRE && + if (can_retire && SNT_IDLE_RETIRE >= 0 && ++idle_streak > SNT_IDLE_RETIRE && (int)RUBY_ATOMIC_LOAD(vm->ractor.sched.snt_cnt) > SNT_KEEP_MINIMUM) { RUBY_ATOMIC_DEC(vm->ractor.sched.snt_cnt); RUBY_DEBUG_LOG("retire, snt_cnt:%d", (int)vm->ractor.sched.snt_cnt); @@ -1753,6 +1763,9 @@ thread_sched_atfork(struct rb_thread_sched *sched) rb_native_mutex_initialize(&vm->ractor.sched.timeslice.lock); ccan_list_head_init(&vm->ractor.sched.timeslice.scheds); rb_native_mutex_initialize(&th->nt->running_th_lock); // a scan could hold it at fork + th->nt->running_th = NULL; // th re-records itself below + th->nt->retiring = false; // the pool it had no room in is gone + native_main_thread_atfork(); // this pthread is the process's main one now // Fork can copy nodes linked (or torn mid-link); re-init every sched's // node so rb_thread_sched_destroy's del_init stays a no-op for them. rb_ractor_t *r; @@ -1800,14 +1813,17 @@ ruby_mn_threads_params(void) rb_vm_t *vm = GET_VM(); rb_ractor_t *main_ractor = GET_RACTOR(); + // RUBY_MN_THREADS: 1 = threads created in the main Ractor are M:N, + // 2 = the main thread too (see thread_sched_main_to_shared) const char *mn_threads_cstr = getenv("RUBY_MN_THREADS"); - bool enable_mn_threads = false; + int mn_threads = (USE_MN_THREADS && mn_threads_cstr) ? atoi(mn_threads_cstr) : 0; - if (USE_MN_THREADS && mn_threads_cstr && (enable_mn_threads = atoi(mn_threads_cstr) > 0)) { - // enabled - ruby_mn_threads_enabled = 1; + if (mn_threads > 0) { + ruby_mn_threads_enabled = mn_threads; } - main_ractor->threads.sched.enable_mn_threads = enable_mn_threads; + // =2 publishes it from thread_sched_main_to_shared, with the main nt's + // pool entry, so that the timer thread cannot mint an snt in between + main_ractor->threads.sched.enable_mn_threads = mn_threads == 1; const char *max_cpu_cstr = getenv("RUBY_MAX_CPU"); int max_cpu = native_thread_default_max_cpu(); @@ -1820,6 +1836,12 @@ ruby_mn_threads_params(void) } vm->ractor.sched.max_cpu = max_cpu; + +#if USE_MN_THREADS + if (mn_threads >= 2) { + thread_sched_main_to_shared(GET_THREAD()); + } +#endif } static void @@ -1857,10 +1879,13 @@ native_thread_dedicated_dec(rb_vm_t *vm, rb_ractor_t *cr, struct rb_native_threa if (nt->dedicated == 0) { // Rejoin under the max_cpu cap; with no room this nt retires and - // belongs to neither count until it ends. + // belongs to neither count until it ends. The process's main nt + // cannot end (its loop runs on a stack it would have to free), so + // it always rejoins. + const bool can_retire = native_thread_self_can_retire_p(); while (1) { rb_atomic_t snt = RUBY_ATOMIC_LOAD(vm->ractor.sched.snt_cnt); - if (snt < vm->ractor.sched.max_cpu || (int)snt <= MINIMUM_SNT) { + if (!can_retire || snt < vm->ractor.sched.max_cpu || (int)snt <= MINIMUM_SNT) { if (RUBY_ATOMIC_CAS(vm->ractor.sched.snt_cnt, snt, snt + 1) == snt) break; } else { @@ -1958,122 +1983,239 @@ nt_start(void *ptr) RUBY_DEBUG_LOG("nt:%u", nt->serial); - bool in_snts = false; + if (nt->dedicated) { + // wait running turn + rb_thread_t *th = nt->running_thread; + struct rb_thread_sched *sched = TH_SCHED(th); - if (!nt->dedicated) { - coroutine_initialize_main(nt->nt_context); + RUBY_DEBUG_LOG("on dedicated th:%u", rb_th_serial(th)); + ruby_thread_set_native(th); - // join the snt list that the barrier/timeslice scans walk - rb_native_mutex_lock(&vm->ractor.sched.ntlist.lock); + thread_sched_lock(sched, th); { - ccan_list_add(&vm->ractor.sched.ntlist.snts, &nt->snts_node); + if (sched->running == th) { + thread_sched_add_running_thread(sched, th); + } + thread_sched_wait_running_turn(sched, th, false, NULL); } - rb_native_mutex_unlock(&vm->ractor.sched.ntlist.lock); - in_snts = true; + thread_sched_unlock(sched, th); + + // start threads + call_thread_start_func_2(th); + // TODO: allow to change to the SNT } + else { + coroutine_initialize_main(nt->nt_context); + nt_snts_join(vm, nt); - bool retired = false; + bool retired = nt_shared_loop(nt); + nt_snts_leave(vm, nt); - while (1) { - if (nt->dedicated) { - // wait running turn - rb_thread_t *th = nt->running_thread; - struct rb_thread_sched *sched = TH_SCHED(th); + if (retired) { + // The counts dropped this nt already; nothing can reference it now. + RUBY_DEBUG_LOG("retired nt:%u", nt->serial); + native_thread_destroy_self(nt); + } + } - RUBY_DEBUG_LOG("on dedicated th:%u", rb_th_serial(th)); - ruby_thread_set_native(th); + return NULL; +} - thread_sched_lock(sched, th); - { - if (sched->running == th) { - thread_sched_add_running_thread(sched, th); - } - thread_sched_wait_running_turn(sched, th, false, NULL); - } - thread_sched_unlock(sched, th); +// join the snt list that the barrier/timeslice scans walk +static void +nt_snts_join(rb_vm_t *vm, struct rb_native_thread *nt) +{ + rb_native_mutex_lock(&vm->ractor.sched.ntlist.lock); + { + ccan_list_add(&vm->ractor.sched.ntlist.snts, &nt->snts_node); + } + rb_native_mutex_unlock(&vm->ractor.sched.ntlist.lock); +} + +// Leaving the shared loop: every path back here deregistered first +// (park and death both precede the transfer), so only the snts entry +// is left to remove. +static void +nt_snts_leave(rb_vm_t *vm, struct rb_native_thread *nt) +{ + VM_ASSERT(nt->running_th == NULL); + rb_native_mutex_lock(&vm->ractor.sched.ntlist.lock); + { + ccan_list_del_init(&nt->snts_node); + } + rb_native_mutex_unlock(&vm->ractor.sched.ntlist.lock); +} - // start threads - call_thread_start_func_2(th); - break; // TODO: allow to change to the SNT +// The shared nt's scheduling loop: serve Ractors from the grq until this nt +// retires (returns true) or goes dedicated while running (returns false). +static bool +nt_shared_loop(struct rb_native_thread *nt) +{ + rb_vm_t *vm = nt->vm; + + while (1) { + RUBY_DEBUG_LOG("check next"); + if (nt->retiring) { // came back with no room in the shared pool + return true; } - else { - RUBY_DEBUG_LOG("check next"); - if (nt->retiring) { // came back with no room in the shared pool - retired = true; - break; - } - rb_ractor_t *r = ractor_sched_deq(vm, NULL); + // asked every time: a fork leaves this loop's native thread as the + // child's main one, which may not retire + rb_ractor_t *r = ractor_sched_deq(vm, NULL, native_thread_self_can_retire_p()); - if (r) { - struct rb_thread_sched *sched = &r->threads.sched; + if (r) { + struct rb_thread_sched *sched = &r->threads.sched; - bool locked = true; + bool locked = true; - thread_sched_lock(sched, NULL); - { - rb_thread_t *next_th = sched->running; + thread_sched_lock(sched, NULL); + { + rb_thread_t *next_th = sched->running; - if (next_th && next_th->nt == NULL) { - RUBY_DEBUG_LOG("nt:%d next_th:%d", (int)nt->serial, (int)next_th->serial); + if (next_th && next_th->nt == NULL) { + RUBY_DEBUG_LOG("nt:%d next_th:%d", (int)nt->serial, (int)next_th->serial); #if USE_MN_THREADS - thread_sched_switch0(nt->nt_context, next_th, nt, false); - - // If a coroutine terminated during the transfer, co_start - // recorded it in nt->dead_co (switch0's return value is - // backend-dependent, unusable; see thread_pthread.h). - struct coroutine_context *dead_co = nt->dead_co; - nt->dead_co = NULL; - if (thread_sched_reclaim(dead_co)) { - // it already released the sched lock before its - // transfer (its Ractor may be gone): leave sched be. - locked = false; - } + thread_sched_switch0(nt->nt_context, next_th, nt, false); + + // If a coroutine terminated during the transfer, co_start + // recorded it in nt->dead_co (switch0's return value is + // backend-dependent, unusable; see thread_pthread.h). + struct coroutine_context *dead_co = nt->dead_co; + nt->dead_co = NULL; + if (thread_sched_reclaim(dead_co)) { + // it already released the sched lock before its + // transfer (its Ractor may be gone): leave sched be. + locked = false; + } #else - thread_sched_switch0(nt->nt_context, next_th, nt, false); + thread_sched_switch0(nt->nt_context, next_th, nt, false); #endif - } - else { - RUBY_DEBUG_LOG("no schedulable threads -- next_th:%p", next_th); - } } - if (locked) { - thread_sched_unlock(sched, NULL); + else { + RUBY_DEBUG_LOG("no schedulable threads -- next_th:%p", next_th); } } - else { - // ractor_sched_deq retired this nt. - retired = true; - break; + if (locked) { + thread_sched_unlock(sched, NULL); } + } + else { + // ractor_sched_deq retired this nt. + return true; + } - if (nt->dedicated) { - // SNT becomes DNT while running - break; - } + if (nt->dedicated) { + // SNT becomes DNT while running + return false; } } +} - if (in_snts) { - // Leaving the shared loop: every path back here deregistered first - // (park and death both precede the transfer), so only the snts entry - // is left to remove. - VM_ASSERT(nt->running_th == NULL); - rb_native_mutex_lock(&vm->ractor.sched.ntlist.lock); - { - ccan_list_del_init(&nt->snts_node); - } - rb_native_mutex_unlock(&vm->ractor.sched.ntlist.lock); +#if USE_MN_THREADS +// The scheduling loop of the process's main nt, entered when RUBY_MN_THREADS=2 +// turned it shared: the process stack belongs to the main thread's context, +// so this loop runs on a coroutine of its own (thread_sched_main_to_shared). +static COROUTINE +nt_loop_co(struct coroutine_context *from, struct coroutine_context *self) +{ +#ifdef RUBY_ASAN_ENABLED + __sanitizer_finish_switch_fiber(self->fake_stack, + (const void**)&from->stack_base, &from->stack_size); +#endif + struct rb_native_thread *nt = (struct rb_native_thread *)self->argument; + + // The first entry is a transfer that in nt_shared_loop would return from + // thread_sched_switch0: a parked thread left its sched lock held for the + // loop to release, or a terminated one (dead_co) released it itself. + struct coroutine_context *dead_co = nt->dead_co; + nt->dead_co = NULL; + if (!thread_sched_reclaim(dead_co)) { + rb_thread_t *parked_th = (rb_thread_t *)from->argument; + thread_sched_unlock(TH_SCHED(parked_th), NULL); } - if (retired) { - // The counts dropped this nt already; nothing can reference it now. - RUBY_DEBUG_LOG("retired nt:%u", nt->serial); - native_thread_destroy_self(nt); + // as after a switch0 return: the thread may have pinned this nt + // (rb_thread_lock_native_thread) before it ended + if (!nt->dedicated) { + if (nt_shared_loop(nt)) rb_bug("main nt retired"); // native_thread_self_can_retire_p() is false here } + nt_snts_leave(nt->vm, nt); - return NULL; + // Went dedicated while running (rb_thread_lock_native_thread) and that + // thread ended. Nothing can resume this context; sleep out the process. + while (1) { + pause(); + } +} + +// RUBY_MN_THREADS=2: make the running main thread an M:N thread in place. +// Its context becomes the process stack (as nt_start's own stack is an snt's +// context) and its nt joins the shared pool with a fresh stack for its loop. +// Nothing changes for the thread until it first blocks: that transfer +// starts nt_loop_co on this nt, and any snt may resume the thread later. +static void +thread_sched_main_to_shared(rb_thread_t *th) +{ + rb_vm_t *vm = th->vm; + struct rb_native_thread *nt = th->nt; + struct rb_thread_sched *sched = TH_SCHED(th); + + VM_ASSERT(th == vm->ractor.main_thread); + VM_ASSERT(nt->dedicated == 1 && th->has_dedicated_nt); + VM_ASSERT(sched->running == th); + + // the loop's stack (the vm stack half of the pool slot goes unused) + void *vm_stack, *machine_stack; + int err = nt_alloc_stack(vm, &vm_stack, &machine_stack); + if (err) { + rb_warn("RUBY_MN_THREADS=2: cannot allocate the main nt's stack (%s); the main thread stays dedicated", strerror(err)); + th->ractor->threads.sched.enable_mn_threads = true; // as =1 + return; + } + size_t machine_stack_size = vm->default_params.thread_machine_stack_size - sizeof(struct nt_machine_stack_footer); + // the main nt is ZALLOC'd by Init_BareVM, not native_thread_alloc: no context yet + nt->nt_context = ruby_xmalloc(sizeof(struct coroutine_context)); + coroutine_initialize(nt->nt_context, nt_loop_co, machine_stack, machine_stack_size); + nt->nt_context->argument = nt; + + // the thread's context is the process stack it already runs on + struct rb_thread_context *tctx = ruby_xmalloc(sizeof(struct rb_thread_context)); + tctx->stack = NULL; // not a pool stack: never freed + tctx->dead = false; + tctx->nt = NULL; + coroutine_initialize_main(&tctx->co); + tctx->co.argument = th; + th->sched.context = &tctx->co; + + thread_sched_lock(sched, th); + { + // re-record the running thread as an snt's (running_dnts -> nt->running_th) + thread_sched_del_running_thread(sched, th); + nt->dedicated = 0; + th->has_dedicated_nt = 0; + nt_snts_join(vm, nt); + // the pool's first snt; under the lock native_thread_check_and_create_shared + // takes, so that it never mints one for the main Ractor alongside + ractor_sched_lock(vm, th->ractor); + { + // The pool is still empty: the timer thread has been up since + // Init_Thread and would mint one here, but its timeout branch is + // the only path there and it sleeps untimed with nothing waiting. + VM_ASSERT(RUBY_ATOMIC_LOAD(vm->ractor.sched.snt_cnt) == 0); + RUBY_ATOMIC_INC(vm->ractor.sched.snt_cnt); + th->ractor->threads.sched.enable_mn_threads = true; + } + ractor_sched_unlock(vm, th->ractor); +#if USE_RUBY_DEBUG_LOG + vm->ractor.sched.dnt_cnt--; +#endif + thread_sched_add_running_thread(sched, th); + } + thread_sched_unlock(sched, th); + + RUBY_DEBUG_LOG("main th:%u on nt:%d is now shared", rb_th_serial(th), nt->serial); } +#endif static int native_thread_create_shared(rb_thread_t *th); @@ -2115,13 +2257,15 @@ rb_threadptr_sched_free(rb_thread_t *th) { timer_thread_wake_fence(th); #if USE_MN_THREADS - if (th->sched.malloc_stack) { - // has dedicated - SIZED_FREE_N((VALUE *)th->sched.context_stack, th->sched.context_stack_size); - native_thread_destroy(th->nt); - } - else if (th->sched.context != NULL) { - // a coroutine thread that never reached its epilogue (never started); + // A thread with a coroutine context runs on a native thread it shares and + // does not own: one made for the shared pool, or the main thread made + // shared. nt->dedicated does not say so, rb_thread_lock_native_thread() + // raising it on a shared native thread that stays in the pool. + const bool owns_nt = th->sched.context == NULL; + + if (th->sched.context != NULL) { + // a coroutine thread that never reached its epilogue (never started), + // or the main thread made shared (its stack is the process stack); // a terminated one is reclaimed by whoever resumed from its final // transfer (thread_sched_reclaim), and cleared this pointer. struct rb_thread_context *tctx = (struct rb_thread_context *)th->sched.context; @@ -2130,6 +2274,12 @@ rb_threadptr_sched_free(rb_thread_t *th) th->sched.context = NULL; // TODO: how to free nt and nt->altstack? } + if (th->sched.malloc_stack) { + SIZED_FREE_N((VALUE *)th->sched.context_stack, th->sched.context_stack_size); + if (th->nt && owns_nt) { + native_thread_destroy(th->nt); + } + } #else SIZED_FREE_N((VALUE *)th->sched.context_stack, th->sched.context_stack_size); native_thread_destroy(th->nt); diff --git a/thread_win32.c b/thread_win32.c index 456ad8114bd86d..943744a9bcf023 100644 --- a/thread_win32.c +++ b/thread_win32.c @@ -872,6 +872,12 @@ native_reset_timer_thread(void) * the ones that rb_bug(). * ------------------------------------------------------------------------- */ +static bool +native_thread_self_can_retire_p(void) +{ + return true; +} + static int native_thread_create_shared(rb_thread_t *th) { From c5777a3ac5c2114b9aaa6c5d2806ea58bb794c95 Mon Sep 17 00:00:00 2001 From: Koichi Sasada Date: Mon, 7 Sep 2026 04:49:42 +0000 Subject: [PATCH 14/24] RUBY_MN_THREADS=-1: do not use the M:N scheduler at all A Ractor's threads have been on the M:N scheduler since it was added, with no way to turn that off: RUBY_MN_THREADS only ever decided whether the main Ractor joined them. There is no way, then, to tell an M:N bug from a Ractor bug, and a C extension that keeps state per native thread cannot be used from a Ractor at all. RUBY_MN_THREADS=-1 gives every Ractor's threads a native thread of their own, as the main Ractor's have by default. With it the setting reads as one ladder: -1 nothing, 0 (the default) a Ractor's threads, 1 the main Ractor's threads too, 2 the main thread as well. The shared pool is left unmade when nothing is M:N, rather than minting native threads that no thread can park on. Co-Authored-By: Claude Opus 5 --- thread_sched.c | 23 ++++++++++++++++++++--- thread_sched_mn.c | 2 ++ 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/thread_sched.c b/thread_sched.c index dffc59a60f8af7..7bceb1f66cc934 100644 --- a/thread_sched.c +++ b/thread_sched.c @@ -104,6 +104,18 @@ static bool timeslice_scan(rb_vm_t *vm, bool interrupt); static void timer_thread_wakeup(void); static void timer_thread_wakeup_locked(rb_vm_t *vm); static void timer_thread_wakeup_force(void); +// RUBY_MN_THREADS: -1 = nothing is M:N, not even a Ractor's threads; +// 0 = the default (a Ractor's threads are, the main Ractor's are not); +// 1 = the main Ractor's threads too; 2 = the main thread as well. +static int mn_threads_mode = 0; + +// Only consulted from USE_MN_THREADS code; the platform gate is not defined yet here. +static bool +mn_threads_enabled_p(void) +{ + return mn_threads_mode >= 0; +} + static void nt_snts_join(rb_vm_t *vm, struct rb_native_thread *nt); static void nt_snts_leave(rb_vm_t *vm, struct rb_native_thread *nt); static bool nt_shared_loop(struct rb_native_thread *nt); @@ -1159,7 +1171,9 @@ rb_thread_sched_init(struct rb_thread_sched *sched, bool atfork) ccan_list_node_init(&sched->timeslice_node); #if USE_MN_THREADS - if (!atfork) sched->enable_mn_threads = true; // MN is enabled on Ractors + // A Ractor's threads are M:N unless RUBY_MN_THREADS turns it off entirely; + // the main Ractor's setting is decided in ruby_mn_threads_params(). + if (!atfork) sched->enable_mn_threads = mn_threads_enabled_p(); #endif } @@ -1813,11 +1827,14 @@ ruby_mn_threads_params(void) rb_vm_t *vm = GET_VM(); rb_ractor_t *main_ractor = GET_RACTOR(); - // RUBY_MN_THREADS: 1 = threads created in the main Ractor are M:N, - // 2 = the main thread too (see thread_sched_main_to_shared) + // RUBY_MN_THREADS: -1 = nothing is M:N, 0 = the default, 1 = the main + // Ractor's threads too, 2 = the main thread as well (see + // thread_sched_main_to_shared). The main Ractor's sched already exists + // here, so it is set rather than defaulted. const char *mn_threads_cstr = getenv("RUBY_MN_THREADS"); int mn_threads = (USE_MN_THREADS && mn_threads_cstr) ? atoi(mn_threads_cstr) : 0; + mn_threads_mode = mn_threads; if (mn_threads > 0) { ruby_mn_threads_enabled = mn_threads; } diff --git a/thread_sched_mn.c b/thread_sched_mn.c index bac0e9eeaf63c4..73a195c557f3ae 100644 --- a/thread_sched_mn.c +++ b/thread_sched_mn.c @@ -925,6 +925,8 @@ native_thread_check_and_create_shared(rb_vm_t *vm) { bool need_to_make = false; + if (!mn_threads_enabled_p()) return 0; // no thread is M:N: the pool serves nobody + ractor_sched_lock(vm, NULL); // NULL: the timer thread also calls this { unsigned int schedulable_ractor_cnt = vm->ractor.cnt; From 09f163dbba3cd59137e71b8aa5d087733d785ffa Mon Sep 17 00:00:00 2001 From: Koichi Sasada Date: Mon, 7 Sep 2026 04:49:42 +0000 Subject: [PATCH 15/24] NEWS: RUBY_MN_THREADS Co-Authored-By: Claude Opus 5 --- NEWS.md | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/NEWS.md b/NEWS.md index 8478a47caa3637..4e9038baf77d17 100644 --- a/NEWS.md +++ b/NEWS.md @@ -356,6 +356,52 @@ A lot of work has gone into making Ractors more stable, performant, and usable. * `ObjectSpace.define_finalizer` on another Ractor's object raises `Ractor::IsolationError`. +### M:N thread scheduler + +* The scheduler scales with the number of waiters and of Ractors, where it + used to walk a list or take one lock for all of them: + + * A timed wait sits in a hierarchical timer wheel rather than on a list + sorted by deadline, which was inserted into by a linear scan. + * An fd stays armed in the backend between waits, instead of being added + before each wait and removed after each wake. + * The io-wait bookkeeping is sharded by fd, rather than serialized on one + lock across every fd. + * A timed wait on an fd rides the scheduler instead of going to a blocking + region, which cost a native thread handoff per wait. + * A context switch, and leaving or rejoining the shared pool, no longer take + the scheduler's global lock. + +* The `RUBY_MN_THREADS` environment variable now runs from no M:N scheduling + at all to all of it. `-1` is new: a Ractor's threads have been M:N since + the scheduler was added, with no way to turn that off. `0` and `1` are + unchanged. + + | | main thread | the main Ractor's other threads | a Ractor's threads | + |---|---|---|---| + | `-1` | 1:1 | 1:1 | 1:1 | + | `0` or unset | 1:1 | 1:1 | M:N | + | `1` | 1:1 | M:N | M:N | + | `2` | M:N | M:N | M:N | + +* `RUBY_MN_THREADS=2` is new. The main thread is resumed like any other M:N + thread rather than woken on a native thread of its own, which costs an order + of magnitude more. It pays off when the main thread drives the work, and + does nothing for one that only starts other threads and waits. + + The main thread is then no longer bound to one OS thread, which is what the + M:N scheduler already meant for every other thread: + + * A C extension that keeps state per OS thread has to call + `rb_thread_lock_native_thread()`. + * What must run on the process's initial thread does not work at all, + pinning included: macOS AppKit and CFRunLoop, and hosts that embed Ruby + and return into their own main loop. + +* The OS thread name is no longer set from the Ruby thread for M:N threads: + one native thread runs many of them over its life. `Thread#name=` was + already skipped for the same reason. + ## JIT [Bug #18947]: https://bugs.ruby-lang.org/issues/18947 From eb9df91618428695e6e8a236f4230c82eb7b8e8a Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Wed, 15 Apr 2026 18:31:28 +0900 Subject: [PATCH 16/24] Fix boundary check at broken widechar string --- string.c | 11 +++++++---- test/ruby/test_string.rb | 8 ++++++++ 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/string.c b/string.c index 9b5667f3a26813..f38bef34367048 100644 --- a/string.c +++ b/string.c @@ -12309,10 +12309,12 @@ rb_str_partition(VALUE str, VALUE sep) pos = rb_str_index(str, sep, 0); if (pos < 0) goto failed; } + + long rpos = pos + RSTRING_LEN(sep); + if (rpos > RSTRING_LEN(str)) goto failed; return rb_ary_new3(3, rb_str_subseq(str, 0, pos), sep, - rb_str_subseq(str, pos+RSTRING_LEN(sep), - RSTRING_LEN(str)-pos-RSTRING_LEN(sep))); + rb_str_subseq(str, rpos, RSTRING_LEN(str)-rpos)); failed: return rb_ary_new3(3, str_duplicate(rb_cString, str), str_new_empty_String(str), str_new_empty_String(str)); @@ -12349,10 +12351,11 @@ rb_str_rpartition(VALUE str, VALUE sep) } } + long rpos = pos + RSTRING_LEN(sep); + if (rpos > RSTRING_LEN(str)) goto failed; return rb_ary_new3(3, rb_str_subseq(str, 0, pos), sep, - rb_str_subseq(str, pos+RSTRING_LEN(sep), - RSTRING_LEN(str)-pos-RSTRING_LEN(sep))); + rb_str_subseq(str, rpos, RSTRING_LEN(str)-rpos)); failed: return rb_ary_new3(3, str_new_empty_String(str), str_new_empty_String(str), str_duplicate(rb_cString, str)); } diff --git a/test/ruby/test_string.rb b/test/ruby/test_string.rb index e00d4c0522dea7..41bf4ce2cc9fdf 100644 --- a/test/ruby/test_string.rb +++ b/test/ruby/test_string.rb @@ -3021,6 +3021,10 @@ def (hyphen = Object.new).to_str; "-"; end assert_equal(["", "", "foo"], S("foo").partition(/^=*/)) assert_equal([S("ab"), S("c"), S("dbce")], S("abcdbce").partition(/b\Kc/)) + + s = S("A").force_encoding(Encoding::UTF_16LE) + sep = S("A\x00").force_encoding(s.encoding) + assert_equal([s, "", ""], s.partition(sep)) end def test_rpartition @@ -3047,6 +3051,10 @@ def (hyphen = Object.new).to_str; "-"; end assert_equal("hello", hello, bug) assert_equal([S("abcdb"), S("c"), S("e")], S("abcdbce").rpartition(/b\Kc/)) + + s = S("A").force_encoding(Encoding::UTF_16LE) + sep = S("A\x00").force_encoding(s.encoding) + assert_equal(["", "", s], s.rpartition(sep)) end def test_rs From 4cf8d4f96e3561780aef526e639907f826a7558a Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Wed, 15 Apr 2026 18:38:28 +0900 Subject: [PATCH 17/24] Fix `lines` at broken wchar string --- string.c | 2 +- test/ruby/test_string.rb | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/string.c b/string.c index f38bef34367048..8ea9508beb1298 100644 --- a/string.c +++ b/string.c @@ -10604,7 +10604,7 @@ rb_str_enumerate_lines(int argc, VALUE *argv, VALUE str, VALUE ary) subptr = hit; } - if (subptr != pend) { + if (subptr < pend) { if (chomp) { if (rsnewline) { pend = chomp_newline(subptr, pend, enc); diff --git a/test/ruby/test_string.rb b/test/ruby/test_string.rb index 41bf4ce2cc9fdf..32cb62ce8050b5 100644 --- a/test/ruby/test_string.rb +++ b/test/ruby/test_string.rb @@ -1394,6 +1394,11 @@ def test_lines assert_equal s.object_id, s.lines {|x| res << x }.object_id assert_equal(S("hello\n"), res[0]) assert_equal(S("world"), res[1]) + + s = S("AB").force_encoding(Encoding::UTF_32LE) + sep = S("B").force_encoding(Encoding::UTF_32LE) + + assert_empty(s.lines(sep).to_a) end def test_empty? From ac9cc92a49dbdf821750ff3b8be3a0ed6109c789 Mon Sep 17 00:00:00 2001 From: Nobuyoshi Nakada Date: Wed, 15 Apr 2026 19:03:47 +0900 Subject: [PATCH 18/24] Fix boundary check in rindex searching multibyte string --- string.c | 2 +- test/ruby/test_string.rb | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/string.c b/string.c index 8ea9508beb1298..7940aaf5380ddd 100644 --- a/string.c +++ b/string.c @@ -4846,7 +4846,7 @@ str_rindex(VALUE str, VALUE sub, const char *s, rb_encoding *enc) c = *t & 0xff; searchlen = s - sbeg + 1; - if (memcmp(s, t, slen) == 0) { + if (s + slen <= e && memcmp(s, t, slen) == 0) { return s - sbeg; } diff --git a/test/ruby/test_string.rb b/test/ruby/test_string.rb index 32cb62ce8050b5..2a5633850ef690 100644 --- a/test/ruby/test_string.rb +++ b/test/ruby/test_string.rb @@ -1802,6 +1802,8 @@ def o.to_str; "bar"; end assert_rindex(nil, S("こんにち"), S("こんにちは")) assert_rindex(nil, S("こ"), S("こんにちは")) assert_rindex(nil, S(""), S("こんにちは")) + + assert_rindex(nil, S("A" * 1024), S("\u{3042}")) end def test_rjust From 2cae75ab921092045db293fd3e761ebc6ecc9be3 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Tue, 8 Sep 2026 09:18:19 +0900 Subject: [PATCH 19/24] Fix the remaining rindex boundary checks The same character count clamp reaches the backward scan in str_rindex() and the pos == 0 shortcut in rb_str_rindex(), where a pattern with wider characters than the receiver is still compared past its last byte. Co-Authored-By: Claude Opus 5 --- string.c | 10 +++++++--- test/ruby/test_string.rb | 11 +++++++++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/string.c b/string.c index 7940aaf5380ddd..b2564848051c74 100644 --- a/string.c +++ b/string.c @@ -4858,7 +4858,7 @@ str_rindex(VALUE str, VALUE sub, const char *s, rb_encoding *enc) searchlen = adjusted - sbeg; continue; } - if (memcmp(hit, t, slen) == 0) + if (hit + slen <= e && memcmp(hit, t, slen) == 0) return hit - sbeg; searchlen = adjusted - sbeg; } while (searchlen > 0); @@ -4883,16 +4883,20 @@ rb_str_rindex(VALUE str, VALUE sub, long pos) /* substring longer than string */ if (len < slen) return -1; + /* character counts, so the byte tail can still be shorter than sub */ if (len - pos < slen) pos = len - slen; if (len == 0) return pos; sbeg = RSTRING_PTR(str); if (pos == 0) { - if (memcmp(sbeg, RSTRING_PTR(sub), RSTRING_LEN(sub)) == 0) + if (RSTRING_LEN(sub) <= RSTRING_LEN(str) && + memcmp(sbeg, RSTRING_PTR(sub), RSTRING_LEN(sub)) == 0) { return 0; - else + } + else { return -1; + } } s = str_nth(sbeg, RSTRING_END(str), pos, enc, singlebyte); diff --git a/test/ruby/test_string.rb b/test/ruby/test_string.rb index 2a5633850ef690..10a2bf61ad5fb3 100644 --- a/test/ruby/test_string.rb +++ b/test/ruby/test_string.rb @@ -1804,6 +1804,17 @@ def o.to_str; "bar"; end assert_rindex(nil, S(""), S("こんにちは")) assert_rindex(nil, S("A" * 1024), S("\u{3042}")) + + # exact-size allocations, so a comparison past the last byte leaves them + assert_rindex(nil, S("A" * 1020 + "\u{3042}A"), S("\u{3042}\u{3044}")) + assert_rindex(nil, S("\u{3042}" + "A" * 1021), S("\u{3042}" * 1022)) + + WIDE_ENCODINGS.each do |enc| + pattern = "A".encode(enc) + stray = pattern.b[0] + assert_nil(S(stray).force_encoding(enc).rindex(pattern), enc.name) + assert_nil(S("B".encode(enc).b * 2 + stray).force_encoding(enc).rindex(pattern), enc.name) + end end def test_rjust From be00aa83bac2c09ddfc6ba8c5aaed5654b3ecc40 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Tue, 8 Sep 2026 09:18:20 +0900 Subject: [PATCH 20/24] Do not look in front of a receiver shorter than one character in chomp String#chomp steps back by the encoding's minimum character width before testing for a newline, which reads before the first byte of the receiver when it holds less than one whole character. Co-Authored-By: Claude Opus 5 --- string.c | 4 +++- test/ruby/test_string.rb | 13 +++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/string.c b/string.c index b2564848051c74..4327492619f7d7 100644 --- a/string.c +++ b/string.c @@ -11128,6 +11128,8 @@ smart_chomp(VALUE str, const char *e, const char *p) { rb_encoding *enc = rb_enc_get(str); if (rb_enc_mbminlen(enc) > 1) { + /* a receiver shorter than one character has nothing to chomp */ + if (e - p < rb_enc_mbminlen(enc)) return e - p; const char *pp = rb_enc_left_char_head(p, e-rb_enc_mbminlen(enc), e, enc); if (rb_enc_is_newline(pp, e, enc)) { e = pp; @@ -11175,7 +11177,7 @@ chompped_length(VALUE str, VALUE rs) RSTRING_GETMEM(rs, rsptr, rslen); if (rslen == 0) { if (rb_enc_mbminlen(enc) > 1) { - while (e > p) { + while (e - p >= rb_enc_mbminlen(enc)) { pp = rb_enc_left_char_head(p, e-rb_enc_mbminlen(enc), e, enc); if (!rb_enc_is_newline(pp, e, enc)) break; e = pp; diff --git a/test/ruby/test_string.rb b/test/ruby/test_string.rb index 10a2bf61ad5fb3..18629f50d463cd 100644 --- a/test/ruby/test_string.rb +++ b/test/ruby/test_string.rb @@ -476,6 +476,19 @@ def test_chomp assert_equal("foo", s.chomp("\n")) s = "foo\r" assert_equal("foo", s.chomp("\n")) + + # capacity forces a heap buffer, so a read before the receiver leaves it + WIDE_ENCODINGS.each do |enc| + ["A", "AB", "ABC"].each do |bytes| + s = S(capacity: 4096) + s << bytes + s.force_encoding(enc) + label = "#{enc.name} #{bytes.bytesize}" + assert_equal(bytes.b, s.chomp.b, label) + assert_equal(bytes.b, s.chomp("").b, label) + assert_nil(s.chomp!, label) + end + end ensure $/ = save $VERBOSE = verbose From 87622756af2088dd0a3f77f29db6b397c039937d Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Mon, 7 Sep 2026 16:16:14 +0900 Subject: [PATCH 21/24] Fix max_output of the UTF-16 and UTF-32 encoders The BOM written ahead of the first character was not counted, so that character can need 6 bytes for UTF-16 and 8 for UTF-32 against a declared max_output of 4. Since the engine lets a transcoder write straight into the caller's window once max_output bytes are free, a window of exactly 4 overran it: ec = Encoding::Converter.new("UTF-8", "UTF-16") ec.primitive_convert("\u{1F600}", "X" * 4092, 4092, 4) # -e:1: [BUG] probable buffer overflow: 4098 for 4096 Co-Authored-By: Claude Opus 5 --- enc/trans/utf_16_32.trans | 4 ++-- test/ruby/test_transcode.rb | 20 ++++++++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/enc/trans/utf_16_32.trans b/enc/trans/utf_16_32.trans index 632c8808efc65a..4a0168b17f00b1 100644 --- a/enc/trans/utf_16_32.trans +++ b/enc/trans/utf_16_32.trans @@ -521,7 +521,7 @@ rb_to_UTF_16 = { TRANSCODE_TABLE_INFO, 1, /* input_unit_length */ 4, /* max_input */ - 4, /* max_output */ + 6, /* max_output */ asciicompat_encoder, /* asciicompat_type */ 1, state_init, NULL, /* state_size, state_init, state_fini */ NULL, NULL, NULL, fun_so_to_utf_16 @@ -533,7 +533,7 @@ rb_to_UTF_32 = { TRANSCODE_TABLE_INFO, 1, /* input_unit_length */ 4, /* max_input */ - 4, /* max_output */ + 8, /* max_output */ asciicompat_encoder, /* asciicompat_type */ 1, state_init, NULL, /* state_size, state_init, state_fini */ NULL, NULL, NULL, fun_so_to_utf_32 diff --git a/test/ruby/test_transcode.rb b/test/ruby/test_transcode.rb index 7b0bde91ff2262..3508436a738aec 100644 --- a/test/ruby/test_transcode.rb +++ b/test/ruby/test_transcode.rb @@ -1195,6 +1195,16 @@ def test_utf_16_bom assert_invalid_in(%w/fffeb7df/.pack("H*"), "UTF-16") end + def test_utf_16_bom_partial_output + ec = Encoding::Converter.new("UTF-8", "UTF-16") + src = "\u{1F600}" + dst = "\0" * 64 + assert_equal(:destination_buffer_full, ec.primitive_convert(src, dst, 0, 4)) + assert_equal(4, dst.bytesize) + assert_equal(:finished, ec.primitive_convert(src, dst, 4, 4)) + assert_equal("\xFE\xFF\xD8\x3D\xDE\x00", dst.b) + end + def test_utf_32_bom expected = "\u{3042}\u{3044}\u{20bb7}" assert_equal(expected, %w/fffe00004230000044300000b70b0200/.pack("H*").encode("UTF-8","UTF-32")) @@ -1202,6 +1212,16 @@ def test_utf_32_bom assert_invalid_in(%w/0000feff00110000/.pack("H*"), "UTF-32") end + def test_utf_32_bom_partial_output + ec = Encoding::Converter.new("UTF-8", "UTF-32") + src = "A" + dst = "\0" * 64 + assert_equal(:destination_buffer_full, ec.primitive_convert(src, dst, 0, 4)) + assert_equal(4, dst.bytesize) + assert_equal(:finished, ec.primitive_convert(src, dst, 4, 4)) + assert_equal("\x00\x00\xFE\xFF\x00\x00\x00A", dst.b) + end + def check_utf_32_both_ways(utf8, raw) copy = raw.dup 0.step(copy.length-1, 4) do |i| From 720d9e11547bff07e0800725c1cbf54814de2462 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Mon, 7 Sep 2026 16:16:20 +0900 Subject: [PATCH 22/24] Fix max_output of the CP50220 encoder A katakana held back for a possible sound mark is flushed by the next character, costing a designation and two data bytes before that character's own designation and data, so one call can write 9 bytes against a declared max_output of 5: ec = Encoding::Converter.new("CP51932", "CP50220") ec.primitive_convert("\x8e\xb6\x8e\xe0".b, "X" * 4088, 4088, 8) # -e:1: [BUG] probable buffer overflow: 4097 for 4096 9 is past the 8 byte inline write buffer in transcode.c, so this transcoder now allocates one. Co-Authored-By: Claude Opus 5 --- enc/trans/iso2022.trans | 2 +- test/ruby/test_transcode.rb | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/enc/trans/iso2022.trans b/enc/trans/iso2022.trans index bc42bbc19c3c05..6e426d51686242 100644 --- a/enc/trans/iso2022.trans +++ b/enc/trans/iso2022.trans @@ -544,7 +544,7 @@ rb_cp50220_encoder = { TRANSCODE_TABLE_INFO, 1, /* input_unit_length */ 3, /* max_input */ - 5, /* max_output */ + 9, /* max_output */ asciicompat_encoder, /* asciicompat_type */ 3, iso2022jp_init, iso2022jp_init, /* state_size, state_init, state_fini */ NULL, NULL, NULL, fun_so_cp50220_encoder, diff --git a/test/ruby/test_transcode.rb b/test/ruby/test_transcode.rb index 3508436a738aec..29aff2cb2bbab0 100644 --- a/test/ruby/test_transcode.rb +++ b/test/ruby/test_transcode.rb @@ -1662,6 +1662,27 @@ def test_to_cp50221 "\x8E\xA1\x8E\xFE".encode("cp50220", "cp51932")) end + def test_to_cp50220_partial_output + # A katakana held back for a possible sound mark is flushed with its own + # designation (5 bytes) before the designation and data of the character + # that ended the hold (4 bytes). + ec = Encoding::Converter.new("CP51932", "CP50220") + src = "\x8E\xB6\x8E\xE0" + dst = "\0" * 64 + assert_equal(:destination_buffer_full, ec.primitive_convert(src, dst, 0, 8)) + assert_equal(8, dst.bytesize) + assert_equal(:finished, ec.primitive_convert(src, dst, 8, 8)) + assert_equal("\e$B\x25\x2B\e(I\x60\e(B", dst.b) + + ec = Encoding::Converter.new("CP51932", "CP50220") + src = "\x8E\xB6" + dst = "\0" * 64 + assert_equal(:destination_buffer_full, ec.primitive_convert(src, dst, 0, 5)) + assert_equal(5, dst.bytesize) + assert_equal(:finished, ec.primitive_convert(src, dst, 5, 5)) + assert_equal("\e$B\x25\x2B\e(B", dst.b) + end + def test_iso_2022_jp_1 # check_both_ways("\u9299", "\x1b$(Dd!\x1b(B", "iso-2022-jp-1") # JIS X 0212 区68 点01 銙 end From 4d3263c68cdf9f48f502c608aa09ea35ffdfc861 Mon Sep 17 00:00:00 2001 From: Hiroshi SHIBATA Date: Tue, 8 Sep 2026 12:10:05 +0900 Subject: [PATCH 23/24] [ruby/rubygems] Normalize absolute symlink targets before checking the extraction root `extract_tar_gz` only ran relative link targets through `File.expand_path`, so an absolute target kept its `..` components. A target such as `/../../etc` therefore passed the prefix check and resolved outside the extraction root. https://github.com/ruby/rubygems/commit/a34a62c3a1 Co-Authored-By: Claude Opus 5 --- lib/rubygems/package.rb | 2 +- test/rubygems/test_gem_package.rb | 30 ++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/lib/rubygems/package.rb b/lib/rubygems/package.rb index 8e4ab4a4b8a9ff..035f63e312cdc9 100644 --- a/lib/rubygems/package.rb +++ b/lib/rubygems/package.rb @@ -511,7 +511,7 @@ def extract_tar_gz(io, destination_dir, pattern = "*") # :nodoc: if entry.symlink? link_target = entry.header.linkname - real_destination = link_target.start_with?("/") ? link_target : File.expand_path(link_target, File.dirname(destination)) + real_destination = File.expand_path(link_target, File.dirname(destination)) raise Gem::Package::SymlinkError.new(full_name, real_destination, destination_dir) unless normalize_path(real_destination).start_with? normalize_path(destination_dir + "/") diff --git a/test/rubygems/test_gem_package.rb b/test/rubygems/test_gem_package.rb index f509fdfbc4d00c..b0935693d1c7e6 100644 --- a/test/rubygems/test_gem_package.rb +++ b/test/rubygems/test_gem_package.rb @@ -1147,6 +1147,36 @@ def test_extract_symlink_parent assert_path_not_exist File.join(destination_subdir, "lib/link") end + def test_extract_symlink_parent_absolute_path + package = Gem::Package.new @gem + + # Extract into a subdirectory of @destination; if this test fails it writes + # a file outside destination_subdir, but we want the file to remain inside + # @destination so it will be cleaned up. + destination_subdir = File.join @destination, "subdir" + FileUtils.mkdir_p destination_subdir + + pend "TMPDIR seems too long to add it as symlink into tar" if destination_subdir.size > 90 + + tgz_io = util_tar_gz do |tar| + tar.mkdir "lib", 0o755 + tar.add_symlink "lib/link", File.join(destination_subdir, ".."), 0o644 + tar.add_file "lib/link/outside.txt", 0o644 do |io| + io.write "hi" + end + end + + e = assert_raise(Gem::Package::SymlinkError) do + package.extract_tar_gz tgz_io, destination_subdir + end + + assert_equal("installing symlink 'lib/link' pointing to parent path #{@destination} of " \ + "#{destination_subdir} is not allowed", e.message) + + assert_path_not_exist File.join(@destination, "outside.txt") + assert_path_not_exist File.join(destination_subdir, "lib/link") + end + def test_extract_symlink_parent_doesnt_delete_user_dir package = Gem::Package.new @gem From c9764e85f7ec4a78dd83931f5edf49a7fd463723 Mon Sep 17 00:00:00 2001 From: Hiroya Fujinami Date: Tue, 8 Sep 2026 15:40:20 +0900 Subject: [PATCH 24/24] Avoid quadratic parse time on and/or chains (#18638) Avoid quadratic parse time on long and/or chains logop() builds a left-associative and/or chain into a right-leaning tree, and found the insertion point by walking the whole right spine from the top on every operator. For a chain such as `a && a && ... && a` that walk is O(n) per operator, so parsing the chain is O(n^2): 32k operators took seconds. Cache the tail of the chain most recently built by logop(), so an operator that extends the same chain reaches the insertion point in constant time, making the parse linear. The cached tail is validated before use (it must still be a node of the chain's type whose nd_2nd is not another such node), so a stale entry from an earlier parse falls back to the scan. The resulting tree is unchanged: AST and compiled bytecode are byte-identical to before across left-associative, parenthesized, and mixed and/or chains. --- parse.y | 59 ++++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 56 insertions(+), 3 deletions(-) diff --git a/parse.y b/parse.y index a5bbb047798222..fee9970de988bf 100644 --- a/parse.y +++ b/parse.y @@ -517,6 +517,14 @@ struct parser_params { /* track the nest level of only braces "{}" */ int brace_nest; } lex; + /* Cache the tail of the and/or chain most recently built by logop(), so a + * left-associative chain does not rescan its right branch on every + * operator. head is the chain's top node, tail the node whose nd_2nd is the + * next insertion point. */ + struct { + NODE *head; + NODE *tail; + } logop; stack_type cond_stack; stack_type cmdarg_stack; int tokidx; @@ -14341,25 +14349,70 @@ new_unless(struct parser_params *p, NODE *cc, NODE *left, NODE *right, const YYL #define NEW_AND_OR(type, f, s, loc, op_loc) (type == NODE_AND ? NEW_AND(f,s,loc,op_loc) : NEW_OR(f,s,loc,op_loc)) +/* A cached logop tail is usable only if it is still a node of the chain's type + * whose nd_2nd is the insertion point, i.e. not itself another node of that + * type. This rejects a stale cache left over from an earlier parse. */ +static int +logop_valid_tail(NODE *tail, enum node_type type) +{ + NODE *second; + return tail && nd_type_p(tail, type) && + ((second = RNODE_AND(tail)->nd_2nd) == 0 || !nd_type_p(second, type)); +} + static NODE* logop(struct parser_params *p, ID id, NODE *left, NODE *right, const YYLTYPE *op_loc, const YYLTYPE *loc) { enum node_type type = id == idAND || id == idANDOP ? NODE_AND : NODE_OR; NODE *op; + /* Snapshot the cache from the previous logop() call before overwriting it. */ + NODE *prev_head = p->logop.head; + NODE *prev_tail = p->logop.tail; + NODE *node, *second; value_expr(p, left); if (left && nd_type_p(left, type)) { - NODE *node = left, *second; - while ((second = RNODE_AND(node)->nd_2nd) != 0 && nd_type_p(second, type)) { - node = second; + /* The insertion point is the far end of left's right branch. + * Reusing the cached tail keeps a chain such as `a && a && ... && a` + * linear instead of rescanning the whole branch on every operator. The + * cache is validated (its nd_2nd is not another node of the same type) + * so a stale entry falls back to the scan. */ + if (left == prev_head && logop_valid_tail(prev_tail, type)) { + node = prev_tail; + } + else { + node = left; + while ((second = RNODE_AND(node)->nd_2nd) != 0 && nd_type_p(second, type)) { + node = second; + } } + second = RNODE_AND(node)->nd_2nd; RNODE_AND(node)->nd_2nd = NEW_AND_OR(type, second, right, loc, op_loc); nd_set_line(RNODE_AND(node)->nd_2nd, op_loc->beg_pos.lineno); left->nd_loc.end_pos = loc->end_pos; + p->logop.head = left; + p->logop.tail = RNODE_AND(node)->nd_2nd; return left; } op = NEW_AND_OR(type, left, right, loc, op_loc); nd_set_line(op, op_loc->beg_pos.lineno); + /* Record where the next operator will extend op, mirroring the scan the + * chained branch above would perform: if right is itself a chain of the + * same type (from parentheses), its far end; otherwise op itself. */ + p->logop.head = op; + if (right == prev_head && logop_valid_tail(prev_tail, type)) { + p->logop.tail = prev_tail; + } + else if (right && nd_type_p(right, type)) { + node = right; + while ((second = RNODE_AND(node)->nd_2nd) != 0 && nd_type_p(second, type)) { + node = second; + } + p->logop.tail = node; + } + else { + p->logop.tail = op; + } return op; }