Power of two size-pools

gc.rb

@@ -269,7 +269,7 @@ def self.stat hash_or_key = nil
   #   GC.stat_heap
   #   # =>
   #   {0 =>
-  #     {slot_size: 40,
+  #     {slot_size: 64,
   #      heap_eden_pages: 246,
   #      heap_eden_slots: 402802,
   #      total_allocated_pages: 246,
@@ -278,7 +278,7 @@ def self.stat hash_or_key = nil
   #      total_allocated_objects: 33867152,
   #      total_freed_objects: 33520523},
   #    1 =>
-  #     {slot_size: 80,
+  #     {slot_size: 128,
   #      heap_eden_pages: 84,
   #      heap_eden_slots: 68746,
   #      total_allocated_pages: 84,
@@ -287,7 +287,7 @@ def self.stat hash_or_key = nil
   #      total_allocated_objects: 147491,
   #      total_freed_objects: 90699},
   #    2 =>
-  #     {slot_size: 160,
+  #     {slot_size: 256,
   #      heap_eden_pages: 157,
   #      heap_eden_slots: 64182,
   #      total_allocated_pages: 157,
@@ -296,7 +296,7 @@ def self.stat hash_or_key = nil
   #      total_allocated_objects: 211460,
   #      total_freed_objects: 190075},
   #    3 =>
-  #     {slot_size: 320,
+  #     {slot_size: 512,
   #      heap_eden_pages: 8,
   #      heap_eden_slots: 1631,
   #      total_allocated_pages: 8,
@@ -305,7 +305,7 @@ def self.stat hash_or_key = nil
   #      total_allocated_objects: 1422,
   #      total_freed_objects: 700},
   #    4 =>
-  #     {slot_size: 640,
+  #     {slot_size: 1024,
   #      heap_eden_pages: 16,
   #      heap_eden_slots: 1628,
   #      total_allocated_pages: 16,
@@ -326,7 +326,7 @@ def self.stat hash_or_key = nil
   #
   #   GC.stat_heap(2)
   #   # =>
-  #   {slot_size: 160,
+  #   {slot_size: 256,
   #    heap_eden_pages: 157,
   #    heap_eden_slots: 64182,
   #    total_allocated_pages: 157,
@@ -338,7 +338,7 @@ def self.stat hash_or_key = nil
   # With arguments +heap_id+ and +key+ given,
   # returns the value for the given key in the given heap:
   #
-  #   GC.stat_heap(2, :slot_size) # => 160
+  #   GC.stat_heap(2, :slot_size) # => 256
   #
   # With arguments +nil+ and +hash+ given,
   # merges the statistics for all heaps into the given hash:

gc/default/default.c

@@ -187,7 +187,7 @@ static RB_THREAD_LOCAL_SPECIFIER int malloc_increase_local;
 #define USE_TICK_T                 (PRINT_ENTER_EXIT_TICK || PRINT_ROOT_TICKS)
 
 #ifndef HEAP_COUNT
-# define HEAP_COUNT 5
+# define HEAP_COUNT 6
 #endif
 
 typedef struct ractor_newobj_heap_cache {
@@ -686,7 +686,12 @@ size_t rb_gc_impl_obj_slot_size(VALUE obj);
 # endif
 #endif
 
-#define BASE_SLOT_SIZE (sizeof(struct RBasic) + sizeof(VALUE[RBIMPL_RVALUE_EMBED_LEN_MAX]) + RVALUE_OVERHEAD)
+#if SIZEOF_VALUE >= 8
+#define BASE_SLOT_SIZE_LOG2 5
+#else
+#define BASE_SLOT_SIZE_LOG2 4
+#endif
+#define BASE_SLOT_SIZE (1 << BASE_SLOT_SIZE_LOG2)
 
 #ifndef MAX
 # define MAX(a, b) (((a) > (b)) ? (a) : (b))
@@ -764,7 +769,7 @@ struct free_slot {
 
 struct heap_page {
     unsigned short slot_size;
-    uint32_t slot_div_magic;
+    unsigned char slot_size_log2;
     unsigned short total_slots;
     unsigned short free_slots;
     unsigned short final_slots;
@@ -841,15 +846,13 @@ heap_page_in_global_empty_pages_pool(rb_objspace_t *objspace, struct heap_page *
 #define GET_PAGE_HEADER(x) (&GET_PAGE_BODY(x)->header)
 #define GET_HEAP_PAGE(x)   (GET_PAGE_HEADER(x)->page)
 
-static uint32_t slot_div_magics[HEAP_COUNT];
-
 static inline size_t
-slot_index_for_offset(size_t offset, uint32_t div_magic)
+slot_index_for_offset(size_t offset, unsigned char slot_size_log2)
 {
-    return (size_t)(((uint64_t)offset * div_magic) >> 32);
+    return offset >> slot_size_log2;
 }
 
-#define SLOT_INDEX(page, p)          slot_index_for_offset((uintptr_t)(p) - (page)->start, (page)->slot_div_magic)
+#define SLOT_INDEX(page, p)          slot_index_for_offset((uintptr_t)(p) - (page)->start, (page)->slot_size_log2)
 #define SLOT_BITMAP_INDEX(page, p)   (SLOT_INDEX(page, p) / BITS_BITLENGTH)
 #define SLOT_BITMAP_OFFSET(page, p)  (SLOT_INDEX(page, p) & (BITS_BITLENGTH - 1))
 #define SLOT_BITMAP_BIT(page, p)     ((bits_t)1 << SLOT_BITMAP_OFFSET(page, p))
@@ -862,7 +865,7 @@ slot_index_for_offset(size_t offset, uint32_t div_magic)
 #define MARK_IN_BITMAP(bits, p)      _MARK_IN_BITMAP(bits, GET_HEAP_PAGE(p), p)
 #define CLEAR_IN_BITMAP(bits, p)     _CLEAR_IN_BITMAP(bits, GET_HEAP_PAGE(p), p)
 
-#define NUM_IN_PAGE(p)   (((bits_t)(p) & HEAP_PAGE_ALIGN_MASK) / BASE_SLOT_SIZE)
+#define NUM_IN_PAGE(p)   (((bits_t)(p) & HEAP_PAGE_ALIGN_MASK) >> BASE_SLOT_SIZE_LOG2)
 
 #define GET_HEAP_MARK_BITS(x)           (&GET_HEAP_PAGE(x)->mark_bits[0])
 #define GET_HEAP_PINNED_BITS(x)         (&GET_HEAP_PAGE(x)->pinned_bits[0])
@@ -1979,30 +1982,16 @@ heap_add_page(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *page)
     GC_ASSERT(!heap->sweeping_page);
     GC_ASSERT(heap_page_in_global_empty_pages_pool(objspace, page));
 
-    /* adjust obj_limit (object number available in this page) */
+    /* Align start to slot_size boundary (both are powers of 2) */
     uintptr_t start = (uintptr_t)page->body + sizeof(struct heap_page_header);
-    if (start % BASE_SLOT_SIZE != 0) {
-        int delta = BASE_SLOT_SIZE - (start % BASE_SLOT_SIZE);
-        start = start + delta;
-        GC_ASSERT(NUM_IN_PAGE(start) == 0 || NUM_IN_PAGE(start) == 1);
-
-        /* Find a num in page that is evenly divisible by `stride`.
-         * This is to ensure that objects are aligned with bit planes.
-         * In other words, ensure there are an even number of objects
-         * per bit plane. */
-        if (NUM_IN_PAGE(start) == 1) {
-            start += heap->slot_size - BASE_SLOT_SIZE;
-        }
-
-        GC_ASSERT(NUM_IN_PAGE(start) * BASE_SLOT_SIZE % heap->slot_size == 0);
-    }
+    start = (start + heap->slot_size - 1) & ~((uintptr_t)heap->slot_size - 1);
 
     int slot_count = (int)((HEAP_PAGE_SIZE - (start - (uintptr_t)page->body))/heap->slot_size);
 
     page->start = start;
     page->total_slots = slot_count;
     page->slot_size = heap->slot_size;
-    page->slot_div_magic = slot_div_magics[heap - heaps];
+    page->slot_size_log2 = BASE_SLOT_SIZE_LOG2 + (unsigned char)(heap - heaps);
     page->heap = heap;
 
     memset(&page->wb_unprotected_bits[0], 0, HEAP_PAGE_BITMAP_SIZE);
@@ -2254,7 +2243,7 @@ heap_slot_size(unsigned char pool_id)
 {
     GC_ASSERT(pool_id < HEAP_COUNT);
 
-    size_t slot_size = (1 << pool_id) * BASE_SLOT_SIZE;
+    size_t slot_size = BASE_SLOT_SIZE << pool_id;
 
 #if RGENGC_CHECK_MODE
     rb_objspace_t *objspace = rb_gc_get_objspace();
@@ -2373,10 +2362,10 @@ heap_idx_for_size(size_t size)
 {
     size += RVALUE_OVERHEAD;
 
-    size_t slot_count = CEILDIV(size, BASE_SLOT_SIZE);
+    if (size <= BASE_SLOT_SIZE) return 0;
 
-    /* heap_idx is ceil(log2(slot_count)) */
-    size_t heap_idx = 64 - nlz_int64(slot_count - 1);
+    /* ceil(log2(size)) - BASE_SLOT_SIZE_LOG2 */
+    size_t heap_idx = 64 - nlz_int64(size - 1) - BASE_SLOT_SIZE_LOG2;
 
     if (heap_idx >= HEAP_COUNT) {
         rb_bug("heap_idx_for_size: allocation size too large "
@@ -3559,7 +3548,7 @@ gc_sweep_plane(rb_objspace_t *objspace, rb_heap_t *heap, uintptr_t p, bits_t bit
                         rb_gc_event_hook(vp, RUBY_INTERNAL_EVENT_FREEOBJ);
                     }
 
-                    (void)VALGRIND_MAKE_MEM_UNDEFINED((void*)p, BASE_SLOT_SIZE);
+                    (void)VALGRIND_MAKE_MEM_UNDEFINED((void*)p, slot_size);
                     heap_page_add_freeobj(objspace, sweep_page, vp);
                     gc_report(3, objspace, "page_sweep: %s (fast path) added to freelist\n", rb_obj_info(vp));
                     ctx->freed_slots++;
@@ -3571,7 +3560,7 @@ gc_sweep_plane(rb_objspace_t *objspace, rb_heap_t *heap, uintptr_t p, bits_t bit
 
                     rb_gc_obj_free_vm_weak_references(vp);
                     if (rb_gc_obj_free(objspace, vp)) {
-                        (void)VALGRIND_MAKE_MEM_UNDEFINED((void*)p, BASE_SLOT_SIZE);
+                        (void)VALGRIND_MAKE_MEM_UNDEFINED((void*)p, slot_size);
                         heap_page_add_freeobj(objspace, sweep_page, vp);
                         gc_report(3, objspace, "page_sweep: %s is added to freelist\n", rb_obj_info(vp));
                         ctx->freed_slots++;
@@ -5455,7 +5444,21 @@ gc_marks_finish(rb_objspace_t *objspace)
         }
 
         if (sweep_slots > max_free_slots) {
-            heap_pages_freeable_pages = (sweep_slots - max_free_slots) / HEAP_PAGE_OBJ_LIMIT;
+            size_t excess_slots = sweep_slots - max_free_slots;
+            size_t total_heap_pages = 0;
+            for (int i = 0; i < HEAP_COUNT; i++) {
+                total_heap_pages += heaps[i].total_pages;
+            }
+            /* Convert excess slots to pages using the actual average slots
+             * per page rather than HEAP_PAGE_OBJ_LIMIT (which is only correct
+             * for pool 0). Larger pools have far fewer slots per page, so using
+             * HEAP_PAGE_OBJ_LIMIT dramatically underestimates freeable pages. */
+            if (total_slots > 0 && total_heap_pages > 0) {
+                heap_pages_freeable_pages = excess_slots * total_heap_pages / total_slots;
+            }
+            else {
+                heap_pages_freeable_pages = 0;
+            }
         }
         else {
             heap_pages_freeable_pages = 0;
@@ -9507,11 +9510,12 @@ rb_gc_impl_objspace_init(void *objspace_ptr)
         rb_bug("Could not preregister postponed job for GC");
     }
 
+    GC_ASSERT(sizeof(struct RBasic) + sizeof(VALUE[RBIMPL_RVALUE_EMBED_LEN_MAX]) + RVALUE_OVERHEAD <= (BASE_SLOT_SIZE << 1));
+
     for (int i = 0; i < HEAP_COUNT; i++) {
         rb_heap_t *heap = &heaps[i];
 
-        heap->slot_size = (1 << i) * BASE_SLOT_SIZE;
-        slot_div_magics[i] = (uint32_t)((uint64_t)UINT32_MAX / heap->slot_size + 1);
+        heap->slot_size = BASE_SLOT_SIZE << i;
 
         ccan_list_head_init(&heap->pages);
     }
@@ -9533,8 +9537,10 @@ rb_gc_impl_objspace_init(void *objspace_ptr)
 #endif
     /* Set size pools allocatable pages. */
     for (int i = 0; i < HEAP_COUNT; i++) {
-        /* Set the default value of heap_init_slots. */
-        gc_params.heap_init_slots[i] = GC_HEAP_INIT_SLOTS;
+        /* Set the default value of heap_init_slots.
+         * Scale inversely with slot size so each pool gets an equal byte
+         * budget (GC_HEAP_INIT_SLOTS * BASE_SLOT_SIZE bytes). */
+        gc_params.heap_init_slots[i] = GC_HEAP_INIT_SLOTS >> i;
     }
 
     init_mark_stack(&objspace->mark_stack);
@@ -9549,6 +9555,11 @@ rb_gc_impl_init(void)
     VALUE gc_constants = rb_hash_new();
     rb_hash_aset(gc_constants, ID2SYM(rb_intern("DEBUG")), GC_DEBUG ? Qtrue : Qfalse);
     rb_hash_aset(gc_constants, ID2SYM(rb_intern("BASE_SLOT_SIZE")), SIZET2NUM(BASE_SLOT_SIZE - RVALUE_OVERHEAD));
+    /* Minimum slot size for a standard RVALUE (RBasic + embedded VALUEs) */
+    size_t rvalue_min = sizeof(struct RBasic) + sizeof(VALUE[RBIMPL_RVALUE_EMBED_LEN_MAX]) + RVALUE_OVERHEAD;
+    size_t rvalue_slot = BASE_SLOT_SIZE;
+    while (rvalue_slot < rvalue_min) rvalue_slot <<= 1;
+    rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_SIZE")), SIZET2NUM(rvalue_slot - RVALUE_OVERHEAD));
     rb_hash_aset(gc_constants, ID2SYM(rb_intern("RBASIC_SIZE")), SIZET2NUM(sizeof(struct RBasic)));
     rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_OVERHEAD")), SIZET2NUM(RVALUE_OVERHEAD));
     rb_hash_aset(gc_constants, ID2SYM(rb_intern("HEAP_PAGE_OBJ_LIMIT")), SIZET2NUM(HEAP_PAGE_OBJ_LIMIT));

gc/mmtk/mmtk.c

@@ -618,17 +618,24 @@ void rb_gc_impl_set_params(void *objspace_ptr) { }
 static VALUE gc_verify_internal_consistency(VALUE self) { return Qnil; }
 
 #define MMTK_HEAP_COUNT 6
-#define MMTK_MAX_OBJ_SIZE 640
-
+#if SIZEOF_VALUE >= 8
+#define MMTK_MAX_OBJ_SIZE 1024
 static size_t heap_sizes[MMTK_HEAP_COUNT + 1] = {
-    32, 40, 80, 160, 320, MMTK_MAX_OBJ_SIZE, 0
+    32, 64, 128, 256, 512, MMTK_MAX_OBJ_SIZE, 0
 };
+#else
+#define MMTK_MAX_OBJ_SIZE 512
+static size_t heap_sizes[MMTK_HEAP_COUNT + 1] = {
+    16, 32, 64, 128, 256, MMTK_MAX_OBJ_SIZE, 0
+};
+#endif
 
 void
 rb_gc_impl_init(void)
 {
     VALUE gc_constants = rb_hash_new();
-    rb_hash_aset(gc_constants, ID2SYM(rb_intern("BASE_SLOT_SIZE")), SIZET2NUM(sizeof(VALUE) * 5));
+    rb_hash_aset(gc_constants, ID2SYM(rb_intern("BASE_SLOT_SIZE")), SIZET2NUM(SIZEOF_VALUE >= 8 ? 64 : 32));
+    rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_SIZE")), SIZET2NUM(SIZEOF_VALUE >= 8 ? 64 : 32));
     rb_hash_aset(gc_constants, ID2SYM(rb_intern("RBASIC_SIZE")), SIZET2NUM(sizeof(struct RBasic)));
     rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_OVERHEAD")), INT2NUM(0));
     rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVARGC_MAX_ALLOCATE_SIZE")), LONG2FIX(MMTK_MAX_OBJ_SIZE));

internal/class.h

@@ -149,7 +149,7 @@ struct RClass_and_rb_classext_t {
 };
 
 #if SIZEOF_VALUE >= SIZEOF_LONG_LONG
-// Assert that classes can be embedded in heaps[2] (which has 160B slot size)
+// Assert that classes can be embedded in heaps[2] (which has 256B slot size)
 // On 32bit platforms there is no variable width allocation so it doesn't matter.
 STATIC_ASSERT(sizeof_rb_classext_t, sizeof(struct RClass_and_rb_classext_t) <= 4 * RVALUE_SIZE);
 #endif

object.c

@@ -93,6 +93,7 @@ static ID id_instance_variables_to_inspect;
 size_t
 rb_obj_embedded_size(uint32_t fields_count)
 {
+    if (fields_count < 1) fields_count = 1;
     return offsetof(struct RObject, as.ary) + (sizeof(VALUE) * fields_count);
 }
 

shape.c

@@ -477,14 +477,14 @@ static attr_index_t
 shape_grow_capa(attr_index_t current_capa)
 {
     const attr_index_t *capacities = rb_shape_tree.capacities;
+    size_t heaps_count = rb_shape_tree.heaps_count;
 
     // First try to use the next size that will be embeddable in a larger object slot.
-    attr_index_t capa;
-    while ((capa = *capacities)) {
+    for (size_t i = 0; i < heaps_count; i++) {
+        attr_index_t capa = capacities[i];
         if (capa > current_capa) {
             return capa;
         }
-        capacities++;
     }
 
     return (attr_index_t)rb_malloc_grow_capa(current_capa, sizeof(VALUE));
@@ -1543,8 +1543,14 @@ Init_default_shapes(void)
     capacities[heaps_count] = 0;
     size_t index;
     for (index = 0; index < heaps_count; index++) {
-        capacities[index] = (heap_sizes[index] - sizeof(struct RBasic)) / sizeof(VALUE);
+        if (heap_sizes[index] > sizeof(struct RBasic)) {
+            capacities[index] = (heap_sizes[index] - sizeof(struct RBasic)) / sizeof(VALUE);
+        }
+        else {
+            capacities[index] = 0;
+        }
     }
+    rb_shape_tree.heaps_count = heaps_count;
     rb_shape_tree.capacities = capacities;
 
 #ifdef HAVE_MMAP

shape.h

@@ -115,6 +115,7 @@ typedef struct {
     rb_shape_t *shape_list;
     rb_shape_t *root_shape;
     const attr_index_t *capacities;
+    size_t heaps_count;
     rb_atomic_t next_shape_id;
 
     redblack_node_t *shape_cache;

string.c

@@ -164,7 +164,7 @@ VALUE rb_cSymbol;
         if (str_embed_capa(str) < capacity + termlen) {\
             char *const tmp = ALLOC_N(char, (size_t)(capacity) + (termlen));\
             const long tlen = RSTRING_LEN(str);\
-            memcpy(tmp, RSTRING_PTR(str), tlen);\
+            memcpy(tmp, RSTRING_PTR(str), str_embed_capa(str));\
             RSTRING(str)->as.heap.ptr = tmp;\
             RSTRING(str)->len = tlen;\
             STR_SET_NOEMBED(str);\

test/-ext-/string/test_capacity.rb

@@ -5,13 +5,13 @@
 
 class Test_StringCapacity < Test::Unit::TestCase
   def test_capacity_embedded
-    assert_equal GC::INTERNAL_CONSTANTS[:BASE_SLOT_SIZE] - embed_header_size - 1, capa('foo')
+    assert_equal GC::INTERNAL_CONSTANTS[:RVALUE_SIZE] - embed_header_size - 1, capa('foo')
     assert_equal max_embed_len, capa('1' * max_embed_len)
     assert_equal max_embed_len, capa('1' * (max_embed_len - 1))
   end
 
   def test_capacity_shared
-    sym = ("a" * GC::INTERNAL_CONSTANTS[:BASE_SLOT_SIZE]).to_sym
+    sym = ("a" * GC::INTERNAL_CONSTANTS[:RVALUE_SIZE]).to_sym
     assert_equal 0, capa(sym.to_s)
   end
 

test/-ext-/string/test_set_len.rb

@@ -5,7 +5,7 @@
 class Test_StrSetLen < Test::Unit::TestCase
   def setup
     # Make string long enough so that it is not embedded
-    @range_end = ("0".ord + GC::INTERNAL_CONSTANTS[:BASE_SLOT_SIZE]).chr
+    @range_end = ("0".ord + GC::INTERNAL_CONSTANTS[:RVALUE_SIZE]).chr
     @s0 = [*"0"..@range_end].join("").freeze
     @s1 = Bug::String.new(@s0)
   end