From 12cbd04dc192e49c1424f6aac997d10e06dcfc0f Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Fri, 13 Feb 2026 20:59:20 +0000 Subject: [PATCH 01/17] Use power-of-2 size pool slot sizes Change slot sizes from {40,80,160,320,640} to {64,128,256,512,1024}. BASE_SLOT_SIZE is now 64 (2^6) and all pool sizes are powers of 2 This enables bit-shift slot indexing instead of magic number division. --- gc.rb | 14 ++++++------ gc/default/default.c | 39 +++++++++++++++++----------------- gc/mmtk/mmtk.c | 14 ++++++++---- internal/class.h | 2 +- test/objspace/test_objspace.rb | 6 +++--- test/ruby/test_gc_compact.rb | 2 +- test/ruby/test_object.rb | 7 +++++- test/ruby/test_time.rb | 5 ++++- 8 files changed, 52 insertions(+), 37 deletions(-) diff --git a/gc.rb b/gc.rb index 59adcbc62f64d6..e1eda59c4d82ac 100644 --- a/gc.rb +++ b/gc.rb @@ -269,7 +269,7 @@ def self.stat hash_or_key = nil # GC.stat_heap # # => # {0 => - # {slot_size: 40, + # {slot_size: 64, # heap_eden_pages: 246, # heap_eden_slots: 402802, # total_allocated_pages: 246, @@ -278,7 +278,7 @@ def self.stat hash_or_key = nil # total_allocated_objects: 33867152, # total_freed_objects: 33520523}, # 1 => - # {slot_size: 80, + # {slot_size: 128, # heap_eden_pages: 84, # heap_eden_slots: 68746, # total_allocated_pages: 84, @@ -287,7 +287,7 @@ def self.stat hash_or_key = nil # total_allocated_objects: 147491, # total_freed_objects: 90699}, # 2 => - # {slot_size: 160, + # {slot_size: 256, # heap_eden_pages: 157, # heap_eden_slots: 64182, # total_allocated_pages: 157, @@ -296,7 +296,7 @@ def self.stat hash_or_key = nil # total_allocated_objects: 211460, # total_freed_objects: 190075}, # 3 => - # {slot_size: 320, + # {slot_size: 512, # heap_eden_pages: 8, # heap_eden_slots: 1631, # total_allocated_pages: 8, @@ -305,7 +305,7 @@ def self.stat hash_or_key = nil # total_allocated_objects: 1422, # total_freed_objects: 700}, # 4 => - # {slot_size: 640, + # {slot_size: 1024, # heap_eden_pages: 16, # heap_eden_slots: 1628, # total_allocated_pages: 16, @@ -326,7 +326,7 @@ def self.stat hash_or_key = nil # # GC.stat_heap(2) # # => - # {slot_size: 160, + # {slot_size: 256, # heap_eden_pages: 157, # heap_eden_slots: 64182, # total_allocated_pages: 157, @@ -338,7 +338,7 @@ def self.stat hash_or_key = nil # With arguments +heap_id+ and +key+ given, # returns the value for the given key in the given heap: # - # GC.stat_heap(2, :slot_size) # => 160 + # GC.stat_heap(2, :slot_size) # => 256 # # With arguments +nil+ and +hash+ given, # merges the statistics for all heaps into the given hash: diff --git a/gc/default/default.c b/gc/default/default.c index 1099d6e0dc11e5..9a9d02ad6e9ee8 100644 --- a/gc/default/default.c +++ b/gc/default/default.c @@ -687,7 +687,12 @@ size_t rb_gc_impl_obj_slot_size(VALUE obj); # endif #endif -#define BASE_SLOT_SIZE (sizeof(struct RBasic) + sizeof(VALUE[RBIMPL_RVALUE_EMBED_LEN_MAX]) + RVALUE_OVERHEAD) +#if SIZEOF_VALUE >= 8 +#define BASE_SLOT_SIZE_LOG2 6 +#else +#define BASE_SLOT_SIZE_LOG2 5 +#endif +#define BASE_SLOT_SIZE (1 << BASE_SLOT_SIZE_LOG2) #ifndef MAX # define MAX(a, b) (((a) > (b)) ? (a) : (b)) @@ -764,7 +769,7 @@ struct free_slot { struct heap_page { unsigned short slot_size; - uint32_t slot_div_magic; + unsigned char slot_size_log2; unsigned short total_slots; unsigned short free_slots; unsigned short final_slots; @@ -841,15 +846,13 @@ heap_page_in_global_empty_pages_pool(rb_objspace_t *objspace, struct heap_page * #define GET_PAGE_HEADER(x) (&GET_PAGE_BODY(x)->header) #define GET_HEAP_PAGE(x) (GET_PAGE_HEADER(x)->page) -static uint32_t slot_div_magics[HEAP_COUNT]; - static inline size_t -slot_index_for_offset(size_t offset, uint32_t div_magic) +slot_index_for_offset(size_t offset, unsigned char slot_size_log2) { - return (size_t)(((uint64_t)offset * div_magic) >> 32); + return offset >> slot_size_log2; } -#define SLOT_INDEX(page, p) slot_index_for_offset((uintptr_t)(p) - (page)->start, (page)->slot_div_magic) +#define SLOT_INDEX(page, p) slot_index_for_offset((uintptr_t)(p) - (page)->start, (page)->slot_size_log2) #define SLOT_BITMAP_INDEX(page, p) (SLOT_INDEX(page, p) / BITS_BITLENGTH) #define SLOT_BITMAP_OFFSET(page, p) (SLOT_INDEX(page, p) & (BITS_BITLENGTH - 1)) #define SLOT_BITMAP_BIT(page, p) ((bits_t)1 << SLOT_BITMAP_OFFSET(page, p)) @@ -1977,19 +1980,16 @@ heap_add_page(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *page) GC_ASSERT(!heap->sweeping_page); GC_ASSERT(heap_page_in_global_empty_pages_pool(objspace, page)); - /* Align start to the first slot_size boundary after the page header */ + /* Align start to slot_size boundary (both are powers of 2) */ uintptr_t start = (uintptr_t)page->body + sizeof(struct heap_page_header); - size_t remainder = start % heap->slot_size; - if (remainder != 0) { - start += heap->slot_size - remainder; - } + start = (start + heap->slot_size - 1) & ~((uintptr_t)heap->slot_size - 1); int slot_count = (int)((HEAP_PAGE_SIZE - (start - (uintptr_t)page->body))/heap->slot_size); page->start = start; page->total_slots = slot_count; page->slot_size = heap->slot_size; - page->slot_div_magic = slot_div_magics[heap - heaps]; + page->slot_size_log2 = BASE_SLOT_SIZE_LOG2 + (unsigned char)(heap - heaps); page->heap = heap; memset(&page->wb_unprotected_bits[0], 0, HEAP_PAGE_BITMAP_SIZE); @@ -2237,7 +2237,7 @@ heap_slot_size(unsigned char pool_id) { GC_ASSERT(pool_id < HEAP_COUNT); - size_t slot_size = (1 << pool_id) * BASE_SLOT_SIZE; + size_t slot_size = BASE_SLOT_SIZE << pool_id; #if RGENGC_CHECK_MODE rb_objspace_t *objspace = rb_gc_get_objspace(); @@ -2356,10 +2356,10 @@ heap_idx_for_size(size_t size) { size += RVALUE_OVERHEAD; - size_t slot_count = CEILDIV(size, BASE_SLOT_SIZE); + if (size <= BASE_SLOT_SIZE) return 0; - /* heap_idx is ceil(log2(slot_count)) */ - size_t heap_idx = 64 - nlz_int64(slot_count - 1); + /* ceil(log2(size)) - BASE_SLOT_SIZE_LOG2 */ + size_t heap_idx = 64 - nlz_int64(size - 1) - BASE_SLOT_SIZE_LOG2; if (heap_idx >= HEAP_COUNT) { rb_bug("heap_idx_for_size: allocation size too large " @@ -9515,11 +9515,12 @@ rb_gc_impl_objspace_init(void *objspace_ptr) rb_bug("Could not preregister postponed job for GC"); } + GC_ASSERT(sizeof(struct RBasic) + sizeof(VALUE[RBIMPL_RVALUE_EMBED_LEN_MAX]) + RVALUE_OVERHEAD <= BASE_SLOT_SIZE); + for (int i = 0; i < HEAP_COUNT; i++) { rb_heap_t *heap = &heaps[i]; - heap->slot_size = (1 << i) * BASE_SLOT_SIZE; - slot_div_magics[i] = (uint32_t)((uint64_t)UINT32_MAX / heap->slot_size + 1); + heap->slot_size = BASE_SLOT_SIZE << i; ccan_list_head_init(&heap->pages); } diff --git a/gc/mmtk/mmtk.c b/gc/mmtk/mmtk.c index 4832916ce6ea2f..c74a0c8e105e5c 100644 --- a/gc/mmtk/mmtk.c +++ b/gc/mmtk/mmtk.c @@ -618,17 +618,23 @@ void rb_gc_impl_set_params(void *objspace_ptr) { } static VALUE gc_verify_internal_consistency(VALUE self) { return Qnil; } #define MMTK_HEAP_COUNT 6 -#define MMTK_MAX_OBJ_SIZE 640 - +#if SIZEOF_VALUE >= 8 +#define MMTK_MAX_OBJ_SIZE 1024 static size_t heap_sizes[MMTK_HEAP_COUNT + 1] = { - 32, 40, 80, 160, 320, MMTK_MAX_OBJ_SIZE, 0 + 32, 64, 128, 256, 512, MMTK_MAX_OBJ_SIZE, 0 }; +#else +#define MMTK_MAX_OBJ_SIZE 512 +static size_t heap_sizes[MMTK_HEAP_COUNT + 1] = { + 16, 32, 64, 128, 256, MMTK_MAX_OBJ_SIZE, 0 +}; +#endif void rb_gc_impl_init(void) { VALUE gc_constants = rb_hash_new(); - rb_hash_aset(gc_constants, ID2SYM(rb_intern("BASE_SLOT_SIZE")), SIZET2NUM(sizeof(VALUE) * 5)); + rb_hash_aset(gc_constants, ID2SYM(rb_intern("BASE_SLOT_SIZE")), SIZET2NUM(SIZEOF_VALUE >= 8 ? 64 : 32)); rb_hash_aset(gc_constants, ID2SYM(rb_intern("RBASIC_SIZE")), SIZET2NUM(sizeof(struct RBasic))); rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_OVERHEAD")), INT2NUM(0)); rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVARGC_MAX_ALLOCATE_SIZE")), LONG2FIX(MMTK_MAX_OBJ_SIZE)); diff --git a/internal/class.h b/internal/class.h index ea68b07fc20968..08facfd00726e9 100644 --- a/internal/class.h +++ b/internal/class.h @@ -149,7 +149,7 @@ struct RClass_and_rb_classext_t { }; #if SIZEOF_VALUE >= SIZEOF_LONG_LONG -// Assert that classes can be embedded in heaps[2] (which has 160B slot size) +// Assert that classes can be embedded in heaps[2] (which has 256B slot size) // On 32bit platforms there is no variable width allocation so it doesn't matter. STATIC_ASSERT(sizeof_rb_classext_t, sizeof(struct RClass_and_rb_classext_t) <= 4 * RVALUE_SIZE); #endif diff --git a/test/objspace/test_objspace.rb b/test/objspace/test_objspace.rb index d631f97d1bcad8..f1906bcbc49339 100644 --- a/test/objspace/test_objspace.rb +++ b/test/objspace/test_objspace.rb @@ -473,12 +473,12 @@ def test_dump_object assert_include(info, '"embedded":true') assert_include(info, '"ivars":0') - # Non-embed object + # Non-embed object (needs > 6 ivars to exceed pool 0 embed capacity) obj = klass.new - 5.times { |i| obj.instance_variable_set("@ivar#{i}", 0) } + 7.times { |i| obj.instance_variable_set("@ivar#{i}", 0) } info = ObjectSpace.dump(obj) assert_not_include(info, '"embedded":true') - assert_include(info, '"ivars":5') + assert_include(info, '"ivars":7') end def test_dump_control_char diff --git a/test/ruby/test_gc_compact.rb b/test/ruby/test_gc_compact.rb index f3da8e4e138432..2964a67710657e 100644 --- a/test/ruby/test_gc_compact.rb +++ b/test/ruby/test_gc_compact.rb @@ -315,7 +315,7 @@ def test_moving_arrays_up_heaps GC.verify_compaction_references(expand_heap: true, toward: :empty) Fiber.new { - ary = "hello".chars + ary = "hello world".chars # > 6 elements to exceed pool 0 embed capacity $arys = ARY_COUNT.times.map do x = [] ary.each { |e| x << e } diff --git a/test/ruby/test_object.rb b/test/ruby/test_object.rb index 2f340788be43b2..b4d02b2b3b0fd1 100644 --- a/test/ruby/test_object.rb +++ b/test/ruby/test_object.rb @@ -371,12 +371,17 @@ def initialize o1 = c.new o2 = c.new - o1.instance_variable_set(:@foo, 5) + # Add enough ivars to exceed pool 0 embed capacity (6 on 64-bit) + o1.instance_variable_set(:@d, 3) + o1.instance_variable_set(:@e, 4) + o1.instance_variable_set(:@f, 5) + o1.instance_variable_set(:@foo, 6) o1.instance_variable_set(:@a, 0) o1.instance_variable_set(:@b, 1) o1.instance_variable_set(:@c, 2) refute_includes ObjectSpace.dump(o1), '"embedded":true' o1.remove_instance_variable(:@foo) + o1.remove_instance_variable(:@f) assert_includes ObjectSpace.dump(o1), '"embedded":true' o2.instance_variable_set(:@a, 0) diff --git a/test/ruby/test_time.rb b/test/ruby/test_time.rb index 333edb80218a64..80b637d433ee56 100644 --- a/test/ruby/test_time.rb +++ b/test/ruby/test_time.rb @@ -1433,7 +1433,10 @@ def test_memsize RbConfig::SIZEOF["void*"] # Same size as VALUE end sizeof_vtm = RbConfig::SIZEOF["void*"] * 4 + 8 - expect = GC::INTERNAL_CONSTANTS[:BASE_SLOT_SIZE] + sizeof_timew + sizeof_vtm + data_size = GC::INTERNAL_CONSTANTS[:BASE_SLOT_SIZE] + sizeof_timew + sizeof_vtm + # Round up to the next slot size (pools are powers of 2) + expect = GC::INTERNAL_CONSTANTS[:BASE_SLOT_SIZE] + expect <<= 1 while expect < data_size assert_operator ObjectSpace.memsize_of(t), :<=, expect rescue LoadError => e omit "failed to load objspace: #{e.message}" From 0e6ade3a9e1c858f3498847f8cf475d4bce102a3 Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Mon, 16 Feb 2026 22:36:39 +0000 Subject: [PATCH 02/17] Add a 32 byte size pool for small objects RFLOAT mostly --- gc/default/default.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/gc/default/default.c b/gc/default/default.c index 9a9d02ad6e9ee8..478174f7395abf 100644 --- a/gc/default/default.c +++ b/gc/default/default.c @@ -187,7 +187,7 @@ static RB_THREAD_LOCAL_SPECIFIER int malloc_increase_local; #define USE_TICK_T (PRINT_ENTER_EXIT_TICK || PRINT_ROOT_TICKS) #ifndef HEAP_COUNT -# define HEAP_COUNT 5 +# define HEAP_COUNT 6 #endif typedef struct ractor_newobj_heap_cache { @@ -688,9 +688,9 @@ size_t rb_gc_impl_obj_slot_size(VALUE obj); #endif #if SIZEOF_VALUE >= 8 -#define BASE_SLOT_SIZE_LOG2 6 -#else #define BASE_SLOT_SIZE_LOG2 5 +#else +#define BASE_SLOT_SIZE_LOG2 4 #endif #define BASE_SLOT_SIZE (1 << BASE_SLOT_SIZE_LOG2) @@ -9515,7 +9515,7 @@ rb_gc_impl_objspace_init(void *objspace_ptr) rb_bug("Could not preregister postponed job for GC"); } - GC_ASSERT(sizeof(struct RBasic) + sizeof(VALUE[RBIMPL_RVALUE_EMBED_LEN_MAX]) + RVALUE_OVERHEAD <= BASE_SLOT_SIZE); + GC_ASSERT(sizeof(struct RBasic) + sizeof(VALUE[RBIMPL_RVALUE_EMBED_LEN_MAX]) + RVALUE_OVERHEAD <= (BASE_SLOT_SIZE << 1)); for (int i = 0; i < HEAP_COUNT; i++) { rb_heap_t *heap = &heaps[i]; From aca2778dc224c69cba4810c294abad7c2374b23c Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Mon, 16 Feb 2026 22:37:02 +0000 Subject: [PATCH 03/17] Introduce RVALUE_SIZE to capture the size of most RVALUES because BASE_SLOT_SIZE is now 32 bytes, it's no longer suitable for use in tests that use it to assume the size of most RVALUE objects, like strings --- gc/default/default.c | 5 +++++ gc/mmtk/mmtk.c | 1 + test/-ext-/string/test_capacity.rb | 4 ++-- test/-ext-/string/test_set_len.rb | 2 +- test/objspace/test_objspace.rb | 2 +- test/ruby/test_file_exhaustive.rb | 4 ++-- 6 files changed, 12 insertions(+), 6 deletions(-) diff --git a/gc/default/default.c b/gc/default/default.c index 478174f7395abf..9b474ff59f89e9 100644 --- a/gc/default/default.c +++ b/gc/default/default.c @@ -9558,6 +9558,11 @@ rb_gc_impl_init(void) VALUE gc_constants = rb_hash_new(); rb_hash_aset(gc_constants, ID2SYM(rb_intern("DEBUG")), GC_DEBUG ? Qtrue : Qfalse); rb_hash_aset(gc_constants, ID2SYM(rb_intern("BASE_SLOT_SIZE")), SIZET2NUM(BASE_SLOT_SIZE - RVALUE_OVERHEAD)); + /* Minimum slot size for a standard RVALUE (RBasic + embedded VALUEs) */ + size_t rvalue_min = sizeof(struct RBasic) + sizeof(VALUE[RBIMPL_RVALUE_EMBED_LEN_MAX]) + RVALUE_OVERHEAD; + size_t rvalue_slot = BASE_SLOT_SIZE; + while (rvalue_slot < rvalue_min) rvalue_slot <<= 1; + rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_SIZE")), SIZET2NUM(rvalue_slot - RVALUE_OVERHEAD)); rb_hash_aset(gc_constants, ID2SYM(rb_intern("RBASIC_SIZE")), SIZET2NUM(sizeof(struct RBasic))); rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_OVERHEAD")), SIZET2NUM(RVALUE_OVERHEAD)); rb_hash_aset(gc_constants, ID2SYM(rb_intern("HEAP_PAGE_BITMAP_SIZE")), SIZET2NUM(HEAP_PAGE_BITMAP_SIZE)); diff --git a/gc/mmtk/mmtk.c b/gc/mmtk/mmtk.c index c74a0c8e105e5c..6b38fa9a681c33 100644 --- a/gc/mmtk/mmtk.c +++ b/gc/mmtk/mmtk.c @@ -635,6 +635,7 @@ rb_gc_impl_init(void) { VALUE gc_constants = rb_hash_new(); rb_hash_aset(gc_constants, ID2SYM(rb_intern("BASE_SLOT_SIZE")), SIZET2NUM(SIZEOF_VALUE >= 8 ? 64 : 32)); + rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_SIZE")), SIZET2NUM(SIZEOF_VALUE >= 8 ? 64 : 32)); rb_hash_aset(gc_constants, ID2SYM(rb_intern("RBASIC_SIZE")), SIZET2NUM(sizeof(struct RBasic))); rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_OVERHEAD")), INT2NUM(0)); rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVARGC_MAX_ALLOCATE_SIZE")), LONG2FIX(MMTK_MAX_OBJ_SIZE)); diff --git a/test/-ext-/string/test_capacity.rb b/test/-ext-/string/test_capacity.rb index df000f7cdb8103..80a2d5db91cd45 100644 --- a/test/-ext-/string/test_capacity.rb +++ b/test/-ext-/string/test_capacity.rb @@ -5,13 +5,13 @@ class Test_StringCapacity < Test::Unit::TestCase def test_capacity_embedded - assert_equal GC::INTERNAL_CONSTANTS[:BASE_SLOT_SIZE] - embed_header_size - 1, capa('foo') + assert_equal GC::INTERNAL_CONSTANTS[:RVALUE_SIZE] - embed_header_size - 1, capa('foo') assert_equal max_embed_len, capa('1' * max_embed_len) assert_equal max_embed_len, capa('1' * (max_embed_len - 1)) end def test_capacity_shared - sym = ("a" * GC::INTERNAL_CONSTANTS[:BASE_SLOT_SIZE]).to_sym + sym = ("a" * GC::INTERNAL_CONSTANTS[:RVALUE_SIZE]).to_sym assert_equal 0, capa(sym.to_s) end diff --git a/test/-ext-/string/test_set_len.rb b/test/-ext-/string/test_set_len.rb index 1531d76167c35c..a18bbbc70cd237 100644 --- a/test/-ext-/string/test_set_len.rb +++ b/test/-ext-/string/test_set_len.rb @@ -5,7 +5,7 @@ class Test_StrSetLen < Test::Unit::TestCase def setup # Make string long enough so that it is not embedded - @range_end = ("0".ord + GC::INTERNAL_CONSTANTS[:BASE_SLOT_SIZE]).chr + @range_end = ("0".ord + GC::INTERNAL_CONSTANTS[:RVALUE_SIZE]).chr @s0 = [*"0"..@range_end].join("").freeze @s1 = Bug::String.new(@s0) end diff --git a/test/objspace/test_objspace.rb b/test/objspace/test_objspace.rb index f1906bcbc49339..92e58219d69bdf 100644 --- a/test/objspace/test_objspace.rb +++ b/test/objspace/test_objspace.rb @@ -33,7 +33,7 @@ def test_memsize_of_root_shared_string b = a.dup c = nil ObjectSpace.each_object(String) {|x| break c = x if a == x and x.frozen?} - rv_size = GC::INTERNAL_CONSTANTS[:BASE_SLOT_SIZE] + rv_size = GC::INTERNAL_CONSTANTS[:RVALUE_SIZE] assert_equal([rv_size, rv_size, a.length + 1 + rv_size], [a, b, c].map {|x| ObjectSpace.memsize_of(x)}) end diff --git a/test/ruby/test_file_exhaustive.rb b/test/ruby/test_file_exhaustive.rb index be9e6bd44e702d..3f9a71253331e7 100644 --- a/test/ruby/test_file_exhaustive.rb +++ b/test/ruby/test_file_exhaustive.rb @@ -897,10 +897,10 @@ def test_expand_path_memsize bug9934 = '[ruby-core:63114] [Bug #9934]' require "objspace" path = File.expand_path("/foo") - assert_operator(ObjectSpace.memsize_of(path), :<=, path.bytesize + GC::INTERNAL_CONSTANTS[:BASE_SLOT_SIZE], bug9934) + assert_operator(ObjectSpace.memsize_of(path), :<=, path.bytesize + GC::INTERNAL_CONSTANTS[:RVALUE_SIZE], bug9934) path = File.expand_path("/a"*25) assert_operator(ObjectSpace.memsize_of(path), :<=, - (path.bytesize + 1) * 2 + GC::INTERNAL_CONSTANTS[:BASE_SLOT_SIZE], bug9934) + (path.bytesize + 1) * 2 + GC::INTERNAL_CONSTANTS[:RVALUE_SIZE], bug9934) end def test_expand_path_encoding From f6f96d5080c04482f8cade6b0dd188a690681d9c Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Tue, 17 Feb 2026 12:25:32 +0000 Subject: [PATCH 04/17] Fix unsigned underflow in shape capacity for small size pools MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When RVALUE_OVERHEAD is large (debug builds with RACTOR_CHECK_MODE + GC_DEBUG), the smallest size pool's usable size can be less than sizeof(struct RBasic). The capacity calculation underflows: (8 - 16) / 8 → 0xFFFF (via size_t wraparound, truncated to uint16_t) Since shape_grow_capa iterates capacities from index 0, the garbage 65535 at capacities[0] poisons all ivar capacity growth, causing a buffer overflow in the RUBY_DEBUG assertion that fills unused capacity with Qundef. --- shape.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/shape.c b/shape.c index 90036722f10026..11db2ad56dcffa 100644 --- a/shape.c +++ b/shape.c @@ -1543,7 +1543,12 @@ Init_default_shapes(void) capacities[heaps_count] = 0; size_t index; for (index = 0; index < heaps_count; index++) { - capacities[index] = (heap_sizes[index] - sizeof(struct RBasic)) / sizeof(VALUE); + if (heap_sizes[index] > sizeof(struct RBasic)) { + capacities[index] = (heap_sizes[index] - sizeof(struct RBasic)) / sizeof(VALUE); + } + else { + capacities[index] = 1; + } } rb_shape_tree.capacities = capacities; From 44439e58249bd4e91397887923b6ac2a6075e802 Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Tue, 17 Feb 2026 18:18:30 +0000 Subject: [PATCH 05/17] Fix shape capacity for heaps with no room for embedded IVs When RVALUE_OVERHEAD > 0 (GC_DEBUG, RACTOR_CHECK_MODE), heap[0]'s usable space can equal sizeof(struct RBasic), leaving zero bytes for instance variables. The capacity was incorrectly set to 1, allowing the shape system to embed an IV that overflows into the overhead area. Change the fallback capacity to 0 and switch shape_grow_capa to count-based iteration so that a zero capacity is not confused with the array sentinel. --- shape.c | 9 +++++---- shape.h | 1 + 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/shape.c b/shape.c index 11db2ad56dcffa..93ccd3eb595fdd 100644 --- a/shape.c +++ b/shape.c @@ -477,14 +477,14 @@ static attr_index_t shape_grow_capa(attr_index_t current_capa) { const attr_index_t *capacities = rb_shape_tree.capacities; + size_t heaps_count = rb_shape_tree.heaps_count; // First try to use the next size that will be embeddable in a larger object slot. - attr_index_t capa; - while ((capa = *capacities)) { + for (size_t i = 0; i < heaps_count; i++) { + attr_index_t capa = capacities[i]; if (capa > current_capa) { return capa; } - capacities++; } return (attr_index_t)rb_malloc_grow_capa(current_capa, sizeof(VALUE)); @@ -1547,9 +1547,10 @@ Init_default_shapes(void) capacities[index] = (heap_sizes[index] - sizeof(struct RBasic)) / sizeof(VALUE); } else { - capacities[index] = 1; + capacities[index] = 0; } } + rb_shape_tree.heaps_count = heaps_count; rb_shape_tree.capacities = capacities; #ifdef HAVE_MMAP diff --git a/shape.h b/shape.h index 96c78f2bc1a356..1296e62a1a4d7f 100644 --- a/shape.h +++ b/shape.h @@ -115,6 +115,7 @@ typedef struct { rb_shape_t *shape_list; rb_shape_t *root_shape; const attr_index_t *capacities; + size_t heaps_count; rb_atomic_t next_shape_id; redblack_node_t *shape_cache; From 5e886e8751d56657879d968255054bd2af251264 Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Tue, 17 Feb 2026 20:39:15 +0000 Subject: [PATCH 06/17] Ensure T_OBJECT always has room for external IV storage rb_obj_embedded_size(0) returned sizeof(struct RBasic), which is too small on builds with RVALUE_OVERHEAD (GC_DEBUG) where heap[0] has no usable space beyond RBasic. The as.heap variant needs at least one VALUE of space for the external IV pointer. Clamp the minimum fields_count to 1 so T_OBJECT allocations always request enough space for the as union. --- object.c | 1 + 1 file changed, 1 insertion(+) diff --git a/object.c b/object.c index 4dcd5d615f85a9..d3036c52f0a7c0 100644 --- a/object.c +++ b/object.c @@ -93,6 +93,7 @@ static ID id_instance_variables_to_inspect; size_t rb_obj_embedded_size(uint32_t fields_count) { + if (fields_count < 1) fields_count = 1; return offsetof(struct RObject, as.ary) + (sizeof(VALUE) * fields_count); } From 1a698847b304c97cda7aea8f4f5902ab8cfb6f28 Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Wed, 18 Feb 2026 12:00:45 +0000 Subject: [PATCH 07/17] Scale initial heap size by bytes rather than slot count This assumes that the typical allocation pattern is going to build way more smaller objects than large (>512 byte) objects for most cases. --- gc/default/default.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/gc/default/default.c b/gc/default/default.c index 9b474ff59f89e9..f664d1989475cf 100644 --- a/gc/default/default.c +++ b/gc/default/default.c @@ -9542,8 +9542,10 @@ rb_gc_impl_objspace_init(void *objspace_ptr) #endif /* Set size pools allocatable pages. */ for (int i = 0; i < HEAP_COUNT; i++) { - /* Set the default value of heap_init_slots. */ - gc_params.heap_init_slots[i] = GC_HEAP_INIT_SLOTS; + /* Set the default value of heap_init_slots. + * Scale inversely with slot size so each pool gets an equal byte + * budget (GC_HEAP_INIT_SLOTS * BASE_SLOT_SIZE bytes). */ + gc_params.heap_init_slots[i] = GC_HEAP_INIT_SLOTS >> i; } init_mark_stack(&objspace->mark_stack); From 415720b197a8860e551db9e95030d99c2cf1ccae Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Tue, 24 Feb 2026 15:48:10 +0000 Subject: [PATCH 08/17] Fix the zjit tests. I don't know if this is the correct Fix! --- zjit/src/hir/opt_tests.rs | 36 +++++++++++++++++------------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/zjit/src/hir/opt_tests.rs b/zjit/src/hir/opt_tests.rs index e58e61f3c4e182..c87e4bb2a8e10f 100644 --- a/zjit/src/hir/opt_tests.rs +++ b/zjit/src/hir/opt_tests.rs @@ -5205,12 +5205,8 @@ mod hir_opt_tests { v14:HeapBasicObject = RefineType v6, HeapBasicObject v17:Fixnum[2] = Const Value(2) PatchPoint SingleRactorMode - v36:CShape = LoadField v14, :_shape_id@0x1000 - v37:CShape[0x1003] = GuardBitEquals v36, CShape(0x1003) - StoreField v14, :@bar@0x1004, v17 - WriteBarrier v14, v17 - v40:CShape[0x1005] = Const CShape(0x1005) - StoreField v14, :_shape_id@0x1000, v40 + IncrCounter setivar_fallback_new_shape_needs_extension + SetIvar v14, :@bar, v17 CheckInterrupts Return v17 "); @@ -13368,26 +13364,28 @@ mod hir_opt_tests { v14:HeapBasicObject = RefineType v6, HeapBasicObject v17:Fixnum[2] = Const Value(2) PatchPoint SingleRactorMode - v50:CShape = LoadField v14, :_shape_id@0x1000 - v51:CShape[0x1003] = GuardBitEquals v50, CShape(0x1003) - StoreField v14, :@b@0x1004, v17 - WriteBarrier v14, v17 - v54:CShape[0x1005] = Const CShape(0x1005) - StoreField v14, :_shape_id@0x1000, v54 + IncrCounter setivar_fallback_new_shape_needs_extension + SetIvar v14, :@b, v17 v21:HeapBasicObject = RefineType v14, HeapBasicObject v24:Fixnum[3] = Const Value(3) PatchPoint SingleRactorMode - v57:CShape = LoadField v21, :_shape_id@0x1000 - v58:CShape[0x1005] = GuardBitEquals v57, CShape(0x1005) - StoreField v21, :@c@0x1006, v24 + v51:CShape = LoadField v21, :_shape_id@0x1000 + v52:CShape[0x1004] = GuardBitEquals v51, CShape(0x1004) + v53:CPtr = LoadField v21, :_as_heap@0x1002 + StoreField v53, :@c@0x1002, v24 WriteBarrier v21, v24 - v61:CShape[0x1007] = Const CShape(0x1007) - StoreField v21, :_shape_id@0x1000, v61 + v56:CShape[0x1005] = Const CShape(0x1005) + StoreField v21, :_shape_id@0x1000, v56 v28:HeapBasicObject = RefineType v21, HeapBasicObject v31:Fixnum[4] = Const Value(4) PatchPoint SingleRactorMode - IncrCounter setivar_fallback_new_shape_needs_extension - SetIvar v28, :@d, v31 + v59:CShape = LoadField v28, :_shape_id@0x1000 + v60:CShape[0x1005] = GuardBitEquals v59, CShape(0x1005) + v61:CPtr = LoadField v28, :_as_heap@0x1002 + StoreField v61, :@d@0x1006, v31 + WriteBarrier v28, v31 + v64:CShape[0x1007] = Const CShape(0x1007) + StoreField v28, :_shape_id@0x1000, v64 CheckInterrupts Return v31 "); From 1d62f389a4eddd0e5ed97805bb7bc2127c8938ca Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Tue, 24 Feb 2026 17:40:02 +0000 Subject: [PATCH 09/17] Update gc.rb docs for 6 size pools Add the new 32-byte heap 0 to the stat_heap example, shift all heap indices by 1, and update the keys example to show [0, 1, 2, 3, 4, 5]. --- gc.rb | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/gc.rb b/gc.rb index e1eda59c4d82ac..d65a37e8aac501 100644 --- a/gc.rb +++ b/gc.rb @@ -269,6 +269,15 @@ def self.stat hash_or_key = nil # GC.stat_heap # # => # {0 => + # {slot_size: 32, + # heap_eden_pages: 24, + # heap_eden_slots: 12288, + # total_allocated_pages: 24, + # force_major_gc_count: 0, + # force_incremental_marking_finish_count: 0, + # total_allocated_objects: 8450, + # total_freed_objects: 3120}, + # 1 => # {slot_size: 64, # heap_eden_pages: 246, # heap_eden_slots: 402802, @@ -277,7 +286,7 @@ def self.stat hash_or_key = nil # force_incremental_marking_finish_count: 1, # total_allocated_objects: 33867152, # total_freed_objects: 33520523}, - # 1 => + # 2 => # {slot_size: 128, # heap_eden_pages: 84, # heap_eden_slots: 68746, @@ -286,7 +295,7 @@ def self.stat hash_or_key = nil # force_incremental_marking_finish_count: 4, # total_allocated_objects: 147491, # total_freed_objects: 90699}, - # 2 => + # 3 => # {slot_size: 256, # heap_eden_pages: 157, # heap_eden_slots: 64182, @@ -295,7 +304,7 @@ def self.stat hash_or_key = nil # force_incremental_marking_finish_count: 0, # total_allocated_objects: 211460, # total_freed_objects: 190075}, - # 3 => + # 4 => # {slot_size: 512, # heap_eden_pages: 8, # heap_eden_slots: 1631, @@ -304,7 +313,7 @@ def self.stat hash_or_key = nil # force_incremental_marking_finish_count: 0, # total_allocated_objects: 1422, # total_freed_objects: 700}, - # 4 => + # 5 => # {slot_size: 1024, # heap_eden_pages: 16, # heap_eden_slots: 1628, @@ -316,7 +325,7 @@ def self.stat hash_or_key = nil # # In the example above, the keys in the outer hash are the heap identifiers: # - # GC.stat_heap.keys # => [0, 1, 2, 3, 4] + # GC.stat_heap.keys # => [0, 1, 2, 3, 4, 5] # # On CRuby, each heap identifier is an integer; # on other implementations, a heap identifier may be a string. @@ -324,7 +333,7 @@ def self.stat hash_or_key = nil # With only argument +heap_id+ given, # returns statistics for the given heap identifier: # - # GC.stat_heap(2) + # GC.stat_heap(3) # # => # {slot_size: 256, # heap_eden_pages: 157, @@ -338,7 +347,7 @@ def self.stat hash_or_key = nil # With arguments +heap_id+ and +key+ given, # returns the value for the given key in the given heap: # - # GC.stat_heap(2, :slot_size) # => 256 + # GC.stat_heap(3, :slot_size) # => 256 # # With arguments +nil+ and +hash+ given, # merges the statistics for all heaps into the given hash: From a6d4b97811177393b84ed6d1cec9bb58314ed5e7 Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Tue, 24 Feb 2026 18:00:03 +0000 Subject: [PATCH 10/17] Fix STATIC_ASSERT for class embedding with power-of-2 pools The assertion used 4 * RVALUE_SIZE (160) which was correct for the old heaps[2] = 160B, but no longer matches any pool boundary. Use the literal heaps[3] size (256B) since that is the target pool for class objects on 64-bit. --- internal/class.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/class.h b/internal/class.h index 08facfd00726e9..5dabee3150a50f 100644 --- a/internal/class.h +++ b/internal/class.h @@ -149,9 +149,9 @@ struct RClass_and_rb_classext_t { }; #if SIZEOF_VALUE >= SIZEOF_LONG_LONG -// Assert that classes can be embedded in heaps[2] (which has 256B slot size) +// Assert that classes can be embedded in heaps[3] (256B slot size on 64-bit). // On 32bit platforms there is no variable width allocation so it doesn't matter. -STATIC_ASSERT(sizeof_rb_classext_t, sizeof(struct RClass_and_rb_classext_t) <= 4 * RVALUE_SIZE); +STATIC_ASSERT(sizeof_rb_classext_t, sizeof(struct RClass_and_rb_classext_t) <= 256); #endif struct RClass_boxable { From b08f648320b489bc8337e1ac84f43bad864ec9e8 Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Tue, 24 Feb 2026 23:36:30 +0000 Subject: [PATCH 11/17] Fix MMTk BASE_SLOT_SIZE to match default GC BASE_SLOT_SIZE is now 32 on 64-bit (16 on 32-bit), matching the smallest slot in heap_sizes[]. RVALUE_SIZE stays at 64/32 as the minimum slot for a standard RVALUE. Update stale comment about pool count from 5 to 6. --- gc/mmtk/mmtk.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gc/mmtk/mmtk.c b/gc/mmtk/mmtk.c index 6b38fa9a681c33..88695659efdd66 100644 --- a/gc/mmtk/mmtk.c +++ b/gc/mmtk/mmtk.c @@ -634,12 +634,12 @@ void rb_gc_impl_init(void) { VALUE gc_constants = rb_hash_new(); - rb_hash_aset(gc_constants, ID2SYM(rb_intern("BASE_SLOT_SIZE")), SIZET2NUM(SIZEOF_VALUE >= 8 ? 64 : 32)); + rb_hash_aset(gc_constants, ID2SYM(rb_intern("BASE_SLOT_SIZE")), SIZET2NUM(SIZEOF_VALUE >= 8 ? 32 : 16)); rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_SIZE")), SIZET2NUM(SIZEOF_VALUE >= 8 ? 64 : 32)); rb_hash_aset(gc_constants, ID2SYM(rb_intern("RBASIC_SIZE")), SIZET2NUM(sizeof(struct RBasic))); rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_OVERHEAD")), INT2NUM(0)); rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVARGC_MAX_ALLOCATE_SIZE")), LONG2FIX(MMTK_MAX_OBJ_SIZE)); - // Pretend we have 5 size pools + // Pretend we have 6 size pools rb_hash_aset(gc_constants, ID2SYM(rb_intern("SIZE_POOL_COUNT")), LONG2FIX(MMTK_HEAP_COUNT)); // TODO: correctly set RVALUE_OLD_AGE when we have generational GC support rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_OLD_AGE")), INT2FIX(0)); From 34acfad8877768cadb2093b47662c865c8dad0a6 Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Wed, 25 Feb 2026 09:45:10 +0000 Subject: [PATCH 12/17] Remove stale MMTk test exclusion for test_dump_includes_slot_size BASE_SLOT_SIZE is now 32 bytes, so this exclusion (which was explicitly marked for removal when that happened) can be dropped. --- test/.excludes-mmtk/TestObjSpace.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/test/.excludes-mmtk/TestObjSpace.rb b/test/.excludes-mmtk/TestObjSpace.rb index 703efd79bb8c69..94eb2c436d4435 100644 --- a/test/.excludes-mmtk/TestObjSpace.rb +++ b/test/.excludes-mmtk/TestObjSpace.rb @@ -1,5 +1,4 @@ exclude(:test_dump_all_full, "testing behaviour specific to default GC") exclude(:test_dump_flag_age, "testing behaviour specific to default GC") exclude(:test_dump_flags, "testing behaviour specific to default GC") -exclude(:test_dump_includes_slot_size, "can be removed when BASE_SLOT_SIZE is 32 bytes") exclude(:test_dump_objects_dumps_page_slot_sizes, "testing behaviour specific to default GC") From 01251a13611c6aa047d60bbaee534ded66712402 Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Fri, 27 Feb 2026 18:57:22 +0000 Subject: [PATCH 13/17] Use VALUE alignment for pointer checks Slot sizes are not necessarily powers of BASE_SLOT_SIZE (e.g. the upcoming 48-byte pool). Objects at odd multiples of non-power-of-two slot sizes are VALUE-aligned but not BASE_SLOT_SIZE-aligned. Use sizeof(VALUE) as the minimum alignment for quick-reject in is_pointer_to_heap and related debug assertions. --- gc/default/default.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/gc/default/default.c b/gc/default/default.c index f664d1989475cf..0e862564e806ec 100644 --- a/gc/default/default.c +++ b/gc/default/default.c @@ -1639,7 +1639,7 @@ heap_page_add_freeobj(rb_objspace_t *objspace, struct heap_page *page, VALUE obj /* obj should belong to page */ !(page->start <= (uintptr_t)obj && (uintptr_t)obj < ((uintptr_t)page->start + (page->total_slots * page->slot_size)) && - obj % BASE_SLOT_SIZE == 0)) { + obj % sizeof(VALUE) == 0)) { rb_bug("heap_page_add_freeobj: %p is not rvalue.", (void *)obj); } @@ -2589,7 +2589,7 @@ is_pointer_to_heap(rb_objspace_t *objspace, const void *ptr) if (p < heap_pages_lomem || p > heap_pages_himem) return FALSE; RB_DEBUG_COUNTER_INC(gc_isptr_range); - if (p % BASE_SLOT_SIZE != 0) return FALSE; + if (p % sizeof(VALUE) != 0) return FALSE; RB_DEBUG_COUNTER_INC(gc_isptr_align); page = heap_page_for_ptr(objspace, (uintptr_t)ptr); @@ -3495,7 +3495,7 @@ gc_sweep_plane(rb_objspace_t *objspace, rb_heap_t *heap, uintptr_t p, bits_t bit do { VALUE vp = (VALUE)p; - GC_ASSERT(vp % BASE_SLOT_SIZE == 0); + GC_ASSERT(vp % sizeof(VALUE) == 0); rb_asan_unpoison_object(vp, false); if (bitset & 1) { @@ -5594,7 +5594,7 @@ gc_compact_plane(rb_objspace_t *objspace, rb_heap_t *heap, uintptr_t p, bits_t b do { VALUE vp = (VALUE)p; - GC_ASSERT(vp % BASE_SLOT_SIZE == 0); + GC_ASSERT(vp % sizeof(VALUE) == 0); if (bitset & 1) { objspace->rcompactor.considered_count_table[BUILTIN_TYPE(vp)]++; From 3d9e5b03e1332b215a563f69e3e8a09dc1214ff0 Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Wed, 18 Feb 2026 14:12:39 +0000 Subject: [PATCH 14/17] Introduce slot size lookup table with 48-byte pool Replace the formula-based slot size computation (BASE_SLOT_SIZE << i) with an explicit lookup table that includes a 48-byte pool at index 1. Pool layout: 32, 48, 64, 128, 256, 512, 1024 bytes (7 pools). The 48-byte pool targets the dominant 40-byte RVALUE allocation, reducing internal fragmentation from 37.5% to 16.7%. Init slots are now table-driven: pool 1 (48B) gets the full budget as the busiest pool; pool 0 (32B) gets fewer since little fits there. --- gc/default/default.c | 52 ++++++++++++++++++++++++++++++++------------ 1 file changed, 38 insertions(+), 14 deletions(-) diff --git a/gc/default/default.c b/gc/default/default.c index 0e862564e806ec..dd97a8fde6dcd8 100644 --- a/gc/default/default.c +++ b/gc/default/default.c @@ -187,9 +187,39 @@ static RB_THREAD_LOCAL_SPECIFIER int malloc_increase_local; #define USE_TICK_T (PRINT_ENTER_EXIT_TICK || PRINT_ROOT_TICKS) #ifndef HEAP_COUNT -# define HEAP_COUNT 6 +# define HEAP_COUNT 7 #endif +static const unsigned short heap_slot_size_table[HEAP_COUNT] = { +#if SIZEOF_VALUE >= 8 + 32, 48, 64, 128, 256, 512, 1024 +#else + 16, 24, 32, 64, 128, 256, 512 +#endif +}; + +static const size_t heap_init_slots_table[HEAP_COUNT] = { +#if SIZEOF_VALUE >= 8 + /* [0] 32B */ 2000, + /* [1] 48B */ GC_HEAP_INIT_SLOTS, + /* [2] 64B */ GC_HEAP_INIT_SLOTS / 2, + /* [3] 128B */ GC_HEAP_INIT_SLOTS / 4, + /* [4] 256B */ GC_HEAP_INIT_SLOTS / 8, + /* [5] 512B */ GC_HEAP_INIT_SLOTS / 16, + /* [6] 1024B*/ GC_HEAP_INIT_SLOTS / 32, +#else + 2000, GC_HEAP_INIT_SLOTS, GC_HEAP_INIT_SLOTS / 2, + GC_HEAP_INIT_SLOTS / 4, GC_HEAP_INIT_SLOTS / 8, + GC_HEAP_INIT_SLOTS / 16, GC_HEAP_INIT_SLOTS / 32, +#endif +}; + +static inline bool +slot_size_is_power_of_two(unsigned short slot_size) +{ + return (slot_size & (slot_size - 1)) == 0; +} + typedef struct ractor_newobj_heap_cache { struct free_slot *freelist; struct heap_page *using_page; @@ -2237,13 +2267,7 @@ heap_slot_size(unsigned char pool_id) { GC_ASSERT(pool_id < HEAP_COUNT); - size_t slot_size = BASE_SLOT_SIZE << pool_id; - -#if RGENGC_CHECK_MODE - rb_objspace_t *objspace = rb_gc_get_objspace(); - GC_ASSERT(heaps[pool_id].slot_size == (short)slot_size); -#endif - + size_t slot_size = heap_slot_size_table[pool_id]; slot_size -= RVALUE_OVERHEAD; return slot_size; @@ -9515,12 +9539,12 @@ rb_gc_impl_objspace_init(void *objspace_ptr) rb_bug("Could not preregister postponed job for GC"); } - GC_ASSERT(sizeof(struct RBasic) + sizeof(VALUE[RBIMPL_RVALUE_EMBED_LEN_MAX]) + RVALUE_OVERHEAD <= (BASE_SLOT_SIZE << 1)); + GC_ASSERT(sizeof(struct RBasic) + sizeof(VALUE[RBIMPL_RVALUE_EMBED_LEN_MAX]) + RVALUE_OVERHEAD <= heap_slot_size_table[1]); for (int i = 0; i < HEAP_COUNT; i++) { rb_heap_t *heap = &heaps[i]; - heap->slot_size = BASE_SLOT_SIZE << i; + heap->slot_size = heap_slot_size_table[i]; ccan_list_head_init(&heap->pages); } @@ -9542,10 +9566,10 @@ rb_gc_impl_objspace_init(void *objspace_ptr) #endif /* Set size pools allocatable pages. */ for (int i = 0; i < HEAP_COUNT; i++) { - /* Set the default value of heap_init_slots. - * Scale inversely with slot size so each pool gets an equal byte - * budget (GC_HEAP_INIT_SLOTS * BASE_SLOT_SIZE bytes). */ - gc_params.heap_init_slots[i] = GC_HEAP_INIT_SLOTS >> i; + /* Set the default value of heap_init_slots using a pre-computed + * table. Pool 1 (48B on 64-bit) gets the most slots since it holds + * the busiest objects; larger pools scale down proportionally. */ + gc_params.heap_init_slots[i] = heap_init_slots_table[i]; } init_mark_stack(&objspace->mark_stack); From d70af8c8dc6e9566aeadb1883eb3e66b71091281 Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Wed, 18 Feb 2026 14:27:24 +0000 Subject: [PATCH 15/17] Fix re-embed test for 48-byte RVALUE pool The RVALUE pool is now 48 bytes (capacity 4 embedded IVs), down from 64 bytes (capacity 6). Adjust the test to use 5 ivars to trigger external storage, then remove 2 to test re-embedding with 3. --- test/ruby/test_object.rb | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/test/ruby/test_object.rb b/test/ruby/test_object.rb index b4d02b2b3b0fd1..31e5e60833f154 100644 --- a/test/ruby/test_object.rb +++ b/test/ruby/test_object.rb @@ -371,17 +371,15 @@ def initialize o1 = c.new o2 = c.new - # Add enough ivars to exceed pool 0 embed capacity (6 on 64-bit) + # Add enough ivars to exceed RVALUE pool embed capacity (4 on 64-bit) o1.instance_variable_set(:@d, 3) o1.instance_variable_set(:@e, 4) - o1.instance_variable_set(:@f, 5) - o1.instance_variable_set(:@foo, 6) o1.instance_variable_set(:@a, 0) o1.instance_variable_set(:@b, 1) o1.instance_variable_set(:@c, 2) refute_includes ObjectSpace.dump(o1), '"embedded":true' - o1.remove_instance_variable(:@foo) - o1.remove_instance_variable(:@f) + o1.remove_instance_variable(:@e) + o1.remove_instance_variable(:@d) assert_includes ObjectSpace.dump(o1), '"embedded":true' o2.instance_variable_set(:@a, 0) From 28d69e12621a8e9a3c342088462ba9bbbec30f4b Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Fri, 27 Feb 2026 20:21:37 +0000 Subject: [PATCH 16/17] Fix allocator and bitmap ops for non-power-of-two pools heap_idx_for_size used ceil(log2(size)) to map allocation sizes to pool indices, which assumed every pool was a power-of-two multiple of BASE_SLOT_SIZE. With the 48-byte pool at index 1, objects needing 49-64 bytes were placed in 48-byte slots, causing heap buffer overflows (e.g. embedded TypedData with embed_size=56). Handle the first two pools with direct comparisons, then use the log2 formula with +1 offset for the remaining power-of-two pools. slot_index_for_offset used offset >> slot_size_log2 for bitmap index calculation. Replace with precomputed reciprocal multiplication (offset * reciprocal >> 48) which handles arbitrary slot sizes in two instructions without branching. heap_add_page aligned page start with & ~(slot_size - 1), a bitmask trick that only works for powers of two. Replace with modulo-based round-up on this cold path. Reorder heap_page fields to keep freelist, start, and slot_size_reciprocal in the first cache line for allocation and SLOT_INDEX hot paths. Fix RVALUE_SIZE computation and tests that assumed power-of-two pool progression. --- gc/default/default.c | 58 ++++++++++++++++++++++++++-------- test/objspace/test_objspace.rb | 3 +- test/ruby/test_gc.rb | 2 +- test/ruby/test_time.rb | 6 ++-- 4 files changed, 51 insertions(+), 18 deletions(-) diff --git a/gc/default/default.c b/gc/default/default.c index dd97a8fde6dcd8..03a466fc20596a 100644 --- a/gc/default/default.c +++ b/gc/default/default.c @@ -214,6 +214,32 @@ static const size_t heap_init_slots_table[HEAP_COUNT] = { #endif }; +/* Precomputed reciprocals for fast slot index calculation. + * For slot size d: reciprocal = ceil(2^48 / d). + * Then offset / d == (uint32_t)((offset * reciprocal) >> 48) + * for all offset < HEAP_PAGE_SIZE. */ +#define SLOT_RECIPROCAL_SHIFT 48 + +static const uint64_t heap_slot_reciprocal_table[HEAP_COUNT] = { +#if SIZEOF_VALUE >= 8 + /* 32 */ (1ULL << 48) / 32, + /* 48 */ (1ULL << 48) / 48 + 1, + /* 64 */ (1ULL << 48) / 64, + /* 128 */ (1ULL << 48) / 128, + /* 256 */ (1ULL << 48) / 256, + /* 512 */ (1ULL << 48) / 512, + /* 1024*/ (1ULL << 48) / 1024, +#else + /* 16 */ (1ULL << 48) / 16, + /* 24 */ (1ULL << 48) / 24 + 1, + /* 32 */ (1ULL << 48) / 32, + /* 64 */ (1ULL << 48) / 64, + /* 128 */ (1ULL << 48) / 128, + /* 256 */ (1ULL << 48) / 256, + /* 512 */ (1ULL << 48) / 512, +#endif +}; + static inline bool slot_size_is_power_of_two(unsigned short slot_size) { @@ -798,8 +824,11 @@ struct free_slot { }; struct heap_page { + /* Cache line 0: allocation fast path + SLOT_INDEX */ + struct free_slot *freelist; + uintptr_t start; + uint64_t slot_size_reciprocal; unsigned short slot_size; - unsigned char slot_size_log2; unsigned short total_slots; unsigned short free_slots; unsigned short final_slots; @@ -814,8 +843,6 @@ struct heap_page { struct heap_page *free_next; struct heap_page_body *body; - uintptr_t start; - struct free_slot *freelist; struct ccan_list_node page_node; bits_t wb_unprotected_bits[HEAP_PAGE_BITMAP_LIMIT]; @@ -877,12 +904,12 @@ heap_page_in_global_empty_pages_pool(rb_objspace_t *objspace, struct heap_page * #define GET_HEAP_PAGE(x) (GET_PAGE_HEADER(x)->page) static inline size_t -slot_index_for_offset(size_t offset, unsigned char slot_size_log2) +slot_index_for_offset(size_t offset, uint64_t reciprocal) { - return offset >> slot_size_log2; + return (uint32_t)(((uint64_t)offset * reciprocal) >> SLOT_RECIPROCAL_SHIFT); } -#define SLOT_INDEX(page, p) slot_index_for_offset((uintptr_t)(p) - (page)->start, (page)->slot_size_log2) +#define SLOT_INDEX(page, p) slot_index_for_offset((uintptr_t)(p) - (page)->start, (page)->slot_size_reciprocal) #define SLOT_BITMAP_INDEX(page, p) (SLOT_INDEX(page, p) / BITS_BITLENGTH) #define SLOT_BITMAP_OFFSET(page, p) (SLOT_INDEX(page, p) & (BITS_BITLENGTH - 1)) #define SLOT_BITMAP_BIT(page, p) ((bits_t)1 << SLOT_BITMAP_OFFSET(page, p)) @@ -2010,16 +2037,17 @@ heap_add_page(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *page) GC_ASSERT(!heap->sweeping_page); GC_ASSERT(heap_page_in_global_empty_pages_pool(objspace, page)); - /* Align start to slot_size boundary (both are powers of 2) */ + /* Align start to slot_size boundary */ uintptr_t start = (uintptr_t)page->body + sizeof(struct heap_page_header); - start = (start + heap->slot_size - 1) & ~((uintptr_t)heap->slot_size - 1); + uintptr_t rem = start % heap->slot_size; + if (rem) start += heap->slot_size - rem; int slot_count = (int)((HEAP_PAGE_SIZE - (start - (uintptr_t)page->body))/heap->slot_size); page->start = start; page->total_slots = slot_count; page->slot_size = heap->slot_size; - page->slot_size_log2 = BASE_SLOT_SIZE_LOG2 + (unsigned char)(heap - heaps); + page->slot_size_reciprocal = heap_slot_reciprocal_table[heap - heaps]; page->heap = heap; memset(&page->wb_unprotected_bits[0], 0, HEAP_PAGE_BITMAP_SIZE); @@ -2381,9 +2409,11 @@ heap_idx_for_size(size_t size) size += RVALUE_OVERHEAD; if (size <= BASE_SLOT_SIZE) return 0; + if (size <= heap_slot_size_table[1]) return 1; - /* ceil(log2(size)) - BASE_SLOT_SIZE_LOG2 */ - size_t heap_idx = 64 - nlz_int64(size - 1) - BASE_SLOT_SIZE_LOG2; + /* Pools from index 2 onward are powers of two (64, 128, ...), + * so the log2 formula still works — add 1 to skip the 48B pool. */ + size_t heap_idx = 64 - nlz_int64(size - 1) - BASE_SLOT_SIZE_LOG2 + 1; if (heap_idx >= HEAP_COUNT) { rb_bug("heap_idx_for_size: allocation size too large " @@ -9586,8 +9616,10 @@ rb_gc_impl_init(void) rb_hash_aset(gc_constants, ID2SYM(rb_intern("BASE_SLOT_SIZE")), SIZET2NUM(BASE_SLOT_SIZE - RVALUE_OVERHEAD)); /* Minimum slot size for a standard RVALUE (RBasic + embedded VALUEs) */ size_t rvalue_min = sizeof(struct RBasic) + sizeof(VALUE[RBIMPL_RVALUE_EMBED_LEN_MAX]) + RVALUE_OVERHEAD; - size_t rvalue_slot = BASE_SLOT_SIZE; - while (rvalue_slot < rvalue_min) rvalue_slot <<= 1; + size_t rvalue_slot = heap_slot_size_table[0]; + for (int i = 1; i < HEAP_COUNT && rvalue_slot < rvalue_min; i++) { + rvalue_slot = heap_slot_size_table[i]; + } rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_SIZE")), SIZET2NUM(rvalue_slot - RVALUE_OVERHEAD)); rb_hash_aset(gc_constants, ID2SYM(rb_intern("RBASIC_SIZE")), SIZET2NUM(sizeof(struct RBasic))); rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_OVERHEAD")), SIZET2NUM(RVALUE_OVERHEAD)); diff --git a/test/objspace/test_objspace.rb b/test/objspace/test_objspace.rb index 92e58219d69bdf..39cb6d6734e501 100644 --- a/test/objspace/test_objspace.rb +++ b/test/objspace/test_objspace.rb @@ -648,7 +648,8 @@ def dump_my_heap_please next if obj["type"] == "SHAPE" assert_not_nil obj["slot_size"] - assert_equal 0, obj["slot_size"] % (GC::INTERNAL_CONSTANTS[:BASE_SLOT_SIZE] + GC::INTERNAL_CONSTANTS[:RVALUE_OVERHEAD]) + slot_sizes = GC::INTERNAL_CONSTANTS[:HEAP_COUNT].times.map { |i| GC.stat_heap(i, :slot_size) } + assert_include slot_sizes, obj["slot_size"] } end end diff --git a/test/ruby/test_gc.rb b/test/ruby/test_gc.rb index 60f04f8e10cf11..8ed4c300e46848 100644 --- a/test/ruby/test_gc.rb +++ b/test/ruby/test_gc.rb @@ -230,7 +230,7 @@ def test_stat_heap GC.stat(stat) end - assert_equal (GC::INTERNAL_CONSTANTS[:BASE_SLOT_SIZE] + GC::INTERNAL_CONSTANTS[:RVALUE_OVERHEAD]) * (2**i), stat_heap[:slot_size] + assert_equal GC.stat_heap(i, :slot_size), stat_heap[:slot_size] assert_operator stat_heap[:heap_live_slots], :<=, stat[:heap_live_slots] assert_operator stat_heap[:heap_free_slots], :<=, stat[:heap_free_slots] assert_operator stat_heap[:heap_final_slots], :<=, stat[:heap_final_slots] diff --git a/test/ruby/test_time.rb b/test/ruby/test_time.rb index 80b637d433ee56..3dc757b2805e17 100644 --- a/test/ruby/test_time.rb +++ b/test/ruby/test_time.rb @@ -1434,9 +1434,9 @@ def test_memsize end sizeof_vtm = RbConfig::SIZEOF["void*"] * 4 + 8 data_size = GC::INTERNAL_CONSTANTS[:BASE_SLOT_SIZE] + sizeof_timew + sizeof_vtm - # Round up to the next slot size (pools are powers of 2) - expect = GC::INTERNAL_CONSTANTS[:BASE_SLOT_SIZE] - expect <<= 1 while expect < data_size + # Round up to the smallest slot size that fits + slot_sizes = GC::INTERNAL_CONSTANTS[:HEAP_COUNT].times.map { |i| GC.stat_heap(i, :slot_size) } + expect = slot_sizes.find { |s| s >= data_size } || slot_sizes.last assert_operator ObjectSpace.memsize_of(t), :<=, expect rescue LoadError => e omit "failed to load objspace: #{e.message}" From 9cf3de67f029fa1d9bf53408e928cecd02010e8b Mon Sep 17 00:00:00 2001 From: Matt Valentine-House Date: Fri, 27 Feb 2026 21:26:04 +0000 Subject: [PATCH 17/17] Rename MMTk SIZE_POOL_COUNT to HEAP_COUNT Align MMTk's GC::INTERNAL_CONSTANTS key with the default GC. Update all test references from SIZE_POOL_COUNT to HEAP_COUNT. --- gc/mmtk/mmtk.c | 3 +-- test/ruby/test_gc_compact.rb | 18 ++++++------- test/ruby/test_object.rb | 49 ++++++++++++++++++------------------ test/ruby/test_time.rb | 2 +- 4 files changed, 35 insertions(+), 37 deletions(-) diff --git a/gc/mmtk/mmtk.c b/gc/mmtk/mmtk.c index 88695659efdd66..0b90a8edfbcc56 100644 --- a/gc/mmtk/mmtk.c +++ b/gc/mmtk/mmtk.c @@ -639,8 +639,7 @@ rb_gc_impl_init(void) rb_hash_aset(gc_constants, ID2SYM(rb_intern("RBASIC_SIZE")), SIZET2NUM(sizeof(struct RBasic))); rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_OVERHEAD")), INT2NUM(0)); rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVARGC_MAX_ALLOCATE_SIZE")), LONG2FIX(MMTK_MAX_OBJ_SIZE)); - // Pretend we have 6 size pools - rb_hash_aset(gc_constants, ID2SYM(rb_intern("SIZE_POOL_COUNT")), LONG2FIX(MMTK_HEAP_COUNT)); + rb_hash_aset(gc_constants, ID2SYM(rb_intern("HEAP_COUNT")), LONG2FIX(MMTK_HEAP_COUNT)); // TODO: correctly set RVALUE_OLD_AGE when we have generational GC support rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_OLD_AGE")), INT2FIX(0)); OBJ_FREEZE(gc_constants); diff --git a/test/ruby/test_gc_compact.rb b/test/ruby/test_gc_compact.rb index 2964a67710657e..8f22b49a20a019 100644 --- a/test/ruby/test_gc_compact.rb +++ b/test/ruby/test_gc_compact.rb @@ -207,7 +207,7 @@ def test_updating_references_for_heap_allocated_shared_arrays end def test_updating_references_for_embed_shared_arrays - omit if GC::INTERNAL_CONSTANTS[:SIZE_POOL_COUNT] == 1 + omit if GC::INTERNAL_CONSTANTS[:HEAP_COUNT] == 1 assert_separately(%w[-robjspace], "#{<<~"begin;"}\n#{<<~"end;"}", timeout: 10) begin; @@ -256,7 +256,7 @@ def test_updating_references_for_heap_allocated_frozen_shared_arrays end def test_updating_references_for_embed_frozen_shared_arrays - omit if GC::INTERNAL_CONSTANTS[:SIZE_POOL_COUNT] == 1 + omit if GC::INTERNAL_CONSTANTS[:HEAP_COUNT] == 1 assert_separately(%w[-robjspace], "#{<<~"begin;"}\n#{<<~"end;"}", timeout: 10) begin; @@ -284,7 +284,7 @@ def test_updating_references_for_embed_frozen_shared_arrays end def test_moving_arrays_down_heaps - omit if GC::INTERNAL_CONSTANTS[:SIZE_POOL_COUNT] == 1 + omit if GC::INTERNAL_CONSTANTS[:HEAP_COUNT] == 1 assert_separately(%w[-robjspace], "#{<<~"begin;"}\n#{<<~"end;"}", timeout: 10) begin; @@ -306,7 +306,7 @@ def test_moving_arrays_down_heaps end def test_moving_arrays_up_heaps - omit if GC::INTERNAL_CONSTANTS[:SIZE_POOL_COUNT] == 1 + omit if GC::INTERNAL_CONSTANTS[:HEAP_COUNT] == 1 assert_separately(%w[-robjspace], "#{<<~"begin;"}\n#{<<~"end;"}", timeout: 10) begin; @@ -330,7 +330,7 @@ def test_moving_arrays_up_heaps end def test_moving_objects_between_heaps - omit if GC::INTERNAL_CONSTANTS[:SIZE_POOL_COUNT] == 1 + omit if GC::INTERNAL_CONSTANTS[:HEAP_COUNT] == 1 assert_separately(%w[-robjspace], "#{<<~"begin;"}\n#{<<~"end;"}", timeout: 60) begin; @@ -362,7 +362,7 @@ def add_ivars end def test_compact_objects_of_varying_sizes - omit if GC::INTERNAL_CONSTANTS[:SIZE_POOL_COUNT] == 1 + omit if GC::INTERNAL_CONSTANTS[:HEAP_COUNT] == 1 assert_ruby_status([], "#{<<~"begin;"}\n#{<<~"end;"}", timeout: 10) begin; @@ -378,7 +378,7 @@ def test_compact_objects_of_varying_sizes end def test_moving_strings_up_heaps - omit if GC::INTERNAL_CONSTANTS[:SIZE_POOL_COUNT] == 1 + omit if GC::INTERNAL_CONSTANTS[:HEAP_COUNT] == 1 assert_separately(%w[-robjspace], "#{<<~"begin;"}\n#{<<~"end;"}", timeout: 30) begin; @@ -399,7 +399,7 @@ def test_moving_strings_up_heaps end def test_moving_strings_down_heaps - omit if GC::INTERNAL_CONSTANTS[:SIZE_POOL_COUNT] == 1 + omit if GC::INTERNAL_CONSTANTS[:HEAP_COUNT] == 1 assert_separately(%w[-robjspace], "#{<<~"begin;"}\n#{<<~"end;"}", timeout: 30) begin; @@ -419,7 +419,7 @@ def test_moving_strings_down_heaps end def test_moving_hashes_down_heaps - omit if GC::INTERNAL_CONSTANTS[:SIZE_POOL_COUNT] == 1 + omit if GC::INTERNAL_CONSTANTS[:HEAP_COUNT] == 1 # AR and ST hashes are in the same size pool on 32 bit omit unless RbConfig::SIZEOF["uint64_t"] <= RbConfig::SIZEOF["void*"] diff --git a/test/ruby/test_object.rb b/test/ruby/test_object.rb index 31e5e60833f154..135296447f6ce7 100644 --- a/test/ruby/test_object.rb +++ b/test/ruby/test_object.rb @@ -358,41 +358,40 @@ def test_remove_instance_variable def test_remove_instance_variable_re_embed assert_separately(%w[-robjspace], "#{<<~"begin;"}\n#{<<~'end;'}") begin; - c = Class.new do - attr_reader :a, :b, :c + # Determine the RVALUE pool's embed capacity from GC constants. + rvalue_size = GC::INTERNAL_CONSTANTS[:RVALUE_SIZE] + rbasic_size = GC::INTERNAL_CONSTANTS[:RBASIC_SIZE] + embed_cap = (rvalue_size - rbasic_size) / RbConfig::SIZEOF["void*"] - def initialize - @a = nil - @b = nil - @c = nil - end - end + # Build a class whose initialize sets embed_cap ivars so objects + # are allocated in the RVALUE pool with embedded storage. + init_body = embed_cap.times.map { |i| "@v#{i} = nil" }.join("; ") + c = Class.new { class_eval("def initialize; #{init_body}; end") } o1 = c.new o2 = c.new - # Add enough ivars to exceed RVALUE pool embed capacity (4 on 64-bit) - o1.instance_variable_set(:@d, 3) - o1.instance_variable_set(:@e, 4) - o1.instance_variable_set(:@a, 0) - o1.instance_variable_set(:@b, 1) - o1.instance_variable_set(:@c, 2) + # All embed_cap ivars fit - should be embedded + embed_cap.times { |i| o1.instance_variable_set(:"@v#{i}", i) } + assert_includes ObjectSpace.dump(o1), '"embedded":true' + + # One more ivar overflows embed capacity + o1.instance_variable_set(:@overflow, 99) refute_includes ObjectSpace.dump(o1), '"embedded":true' - o1.remove_instance_variable(:@e) - o1.remove_instance_variable(:@d) + + # Remove the overflow ivar - should re-embed + o1.remove_instance_variable(:@overflow) assert_includes ObjectSpace.dump(o1), '"embedded":true' - o2.instance_variable_set(:@a, 0) - o2.instance_variable_set(:@b, 1) - o2.instance_variable_set(:@c, 2) + # An object that never overflowed is also embedded + embed_cap.times { |i| o2.instance_variable_set(:"@v#{i}", i) } assert_includes ObjectSpace.dump(o2), '"embedded":true' - assert_equal(0, o1.a) - assert_equal(1, o1.b) - assert_equal(2, o1.c) - assert_equal(0, o2.a) - assert_equal(1, o2.b) - assert_equal(2, o2.c) + # Verify values survived re-embedding + embed_cap.times do |i| + assert_equal(i, o1.instance_variable_get(:"@v#{i}")) + assert_equal(i, o2.instance_variable_get(:"@v#{i}")) + end end; end diff --git a/test/ruby/test_time.rb b/test/ruby/test_time.rb index 3dc757b2805e17..bc840eac2757e4 100644 --- a/test/ruby/test_time.rb +++ b/test/ruby/test_time.rb @@ -1421,7 +1421,7 @@ def test_memsize # Time objects are common in some code, try to keep them small omit "Time object size test" if /^(?:i.?86|x86_64)-linux/ !~ RUBY_PLATFORM omit "GC is in debug" if GC::INTERNAL_CONSTANTS[:RVALUE_OVERHEAD] > 0 - omit "memsize is not accurate due to using malloc_usable_size" if GC::INTERNAL_CONSTANTS[:SIZE_POOL_COUNT] == 1 + omit "memsize is not accurate due to using malloc_usable_size" if GC::INTERNAL_CONSTANTS[:HEAP_COUNT] == 1 omit "Only run this test on 64-bit" if RbConfig::SIZEOF["void*"] != 8 require 'objspace'