diff --git a/gc.rb b/gc.rb index 59adcbc62f64d6..e1eda59c4d82ac 100644 --- a/gc.rb +++ b/gc.rb @@ -269,7 +269,7 @@ def self.stat hash_or_key = nil # GC.stat_heap # # => # {0 => - # {slot_size: 40, + # {slot_size: 64, # heap_eden_pages: 246, # heap_eden_slots: 402802, # total_allocated_pages: 246, @@ -278,7 +278,7 @@ def self.stat hash_or_key = nil # total_allocated_objects: 33867152, # total_freed_objects: 33520523}, # 1 => - # {slot_size: 80, + # {slot_size: 128, # heap_eden_pages: 84, # heap_eden_slots: 68746, # total_allocated_pages: 84, @@ -287,7 +287,7 @@ def self.stat hash_or_key = nil # total_allocated_objects: 147491, # total_freed_objects: 90699}, # 2 => - # {slot_size: 160, + # {slot_size: 256, # heap_eden_pages: 157, # heap_eden_slots: 64182, # total_allocated_pages: 157, @@ -296,7 +296,7 @@ def self.stat hash_or_key = nil # total_allocated_objects: 211460, # total_freed_objects: 190075}, # 3 => - # {slot_size: 320, + # {slot_size: 512, # heap_eden_pages: 8, # heap_eden_slots: 1631, # total_allocated_pages: 8, @@ -305,7 +305,7 @@ def self.stat hash_or_key = nil # total_allocated_objects: 1422, # total_freed_objects: 700}, # 4 => - # {slot_size: 640, + # {slot_size: 1024, # heap_eden_pages: 16, # heap_eden_slots: 1628, # total_allocated_pages: 16, @@ -326,7 +326,7 @@ def self.stat hash_or_key = nil # # GC.stat_heap(2) # # => - # {slot_size: 160, + # {slot_size: 256, # heap_eden_pages: 157, # heap_eden_slots: 64182, # total_allocated_pages: 157, @@ -338,7 +338,7 @@ def self.stat hash_or_key = nil # With arguments +heap_id+ and +key+ given, # returns the value for the given key in the given heap: # - # GC.stat_heap(2, :slot_size) # => 160 + # GC.stat_heap(2, :slot_size) # => 256 # # With arguments +nil+ and +hash+ given, # merges the statistics for all heaps into the given hash: diff --git a/gc/default/default.c b/gc/default/default.c index 046aa146f73055..dbfdc43d382d7e 100644 --- a/gc/default/default.c +++ b/gc/default/default.c @@ -187,7 +187,7 @@ static RB_THREAD_LOCAL_SPECIFIER int malloc_increase_local; #define USE_TICK_T (PRINT_ENTER_EXIT_TICK || PRINT_ROOT_TICKS) #ifndef HEAP_COUNT -# define HEAP_COUNT 5 +# define HEAP_COUNT 6 #endif typedef struct ractor_newobj_heap_cache { @@ -687,7 +687,12 @@ size_t rb_gc_impl_obj_slot_size(VALUE obj); # endif #endif -#define BASE_SLOT_SIZE (sizeof(struct RBasic) + sizeof(VALUE[RBIMPL_RVALUE_EMBED_LEN_MAX]) + RVALUE_OVERHEAD) +#if SIZEOF_VALUE >= 8 +#define BASE_SLOT_SIZE_LOG2 5 +#else +#define BASE_SLOT_SIZE_LOG2 4 +#endif +#define BASE_SLOT_SIZE (1 << BASE_SLOT_SIZE_LOG2) #ifndef MAX # define MAX(a, b) (((a) > (b)) ? (a) : (b)) @@ -764,7 +769,7 @@ struct free_slot { struct heap_page { unsigned short slot_size; - uint32_t slot_div_magic; + unsigned char slot_size_log2; unsigned short total_slots; unsigned short free_slots; unsigned short final_slots; @@ -841,15 +846,13 @@ heap_page_in_global_empty_pages_pool(rb_objspace_t *objspace, struct heap_page * #define GET_PAGE_HEADER(x) (&GET_PAGE_BODY(x)->header) #define GET_HEAP_PAGE(x) (GET_PAGE_HEADER(x)->page) -static uint32_t slot_div_magics[HEAP_COUNT]; - static inline size_t -slot_index_for_offset(size_t offset, uint32_t div_magic) +slot_index_for_offset(size_t offset, unsigned char slot_size_log2) { - return (size_t)(((uint64_t)offset * div_magic) >> 32); + return offset >> slot_size_log2; } -#define SLOT_INDEX(page, p) slot_index_for_offset((uintptr_t)(p) - (page)->start, (page)->slot_div_magic) +#define SLOT_INDEX(page, p) slot_index_for_offset((uintptr_t)(p) - (page)->start, (page)->slot_size_log2) #define SLOT_BITMAP_INDEX(page, p) (SLOT_INDEX(page, p) / BITS_BITLENGTH) #define SLOT_BITMAP_OFFSET(page, p) (SLOT_INDEX(page, p) & (BITS_BITLENGTH - 1)) #define SLOT_BITMAP_BIT(page, p) ((bits_t)1 << SLOT_BITMAP_OFFSET(page, p)) @@ -1977,19 +1980,16 @@ heap_add_page(rb_objspace_t *objspace, rb_heap_t *heap, struct heap_page *page) GC_ASSERT(!heap->sweeping_page); GC_ASSERT(heap_page_in_global_empty_pages_pool(objspace, page)); - /* Align start to the first slot_size boundary after the page header */ + /* Align start to slot_size boundary (both are powers of 2) */ uintptr_t start = (uintptr_t)page->body + sizeof(struct heap_page_header); - size_t remainder = start % heap->slot_size; - if (remainder != 0) { - start += heap->slot_size - remainder; - } + start = (start + heap->slot_size - 1) & ~((uintptr_t)heap->slot_size - 1); int slot_count = (int)((HEAP_PAGE_SIZE - (start - (uintptr_t)page->body))/heap->slot_size); page->start = start; page->total_slots = slot_count; page->slot_size = heap->slot_size; - page->slot_div_magic = slot_div_magics[heap - heaps]; + page->slot_size_log2 = BASE_SLOT_SIZE_LOG2 + (unsigned char)(heap - heaps); page->heap = heap; memset(&page->wb_unprotected_bits[0], 0, HEAP_PAGE_BITMAP_SIZE); @@ -2241,7 +2241,7 @@ heap_slot_size(unsigned char pool_id) { GC_ASSERT(pool_id < HEAP_COUNT); - size_t slot_size = (1 << pool_id) * BASE_SLOT_SIZE; + size_t slot_size = BASE_SLOT_SIZE << pool_id; #if RGENGC_CHECK_MODE rb_objspace_t *objspace = rb_gc_get_objspace(); @@ -2360,10 +2360,10 @@ heap_idx_for_size(size_t size) { size += RVALUE_OVERHEAD; - size_t slot_count = CEILDIV(size, BASE_SLOT_SIZE); + if (size <= BASE_SLOT_SIZE) return 0; - /* heap_idx is ceil(log2(slot_count)) */ - size_t heap_idx = 64 - nlz_int64(slot_count - 1); + /* ceil(log2(size)) - BASE_SLOT_SIZE_LOG2 */ + size_t heap_idx = 64 - nlz_int64(size - 1) - BASE_SLOT_SIZE_LOG2; if (heap_idx >= HEAP_COUNT) { rb_bug("heap_idx_for_size: allocation size too large " @@ -5443,11 +5443,31 @@ gc_marks_finish(rb_objspace_t *objspace) /* Approximate freeable pages using the average slots-per-pages across all heaps */ if (sweep_slots > max_free_slots) { +<<<<<<< HEAD size_t excess_slots = sweep_slots - max_free_slots; size_t total_heap_pages = heap_eden_total_pages(objspace); heap_pages_freeable_pages = total_heap_pages > 0 ? excess_slots * total_heap_pages / total_slots : 0; +||||||| parent of a9b735748e (Calculate freeable pages based on how many actual slots are in each page) + heap_pages_freeable_pages = (sweep_slots - max_free_slots) / HEAP_PAGE_OBJ_LIMIT; +======= + size_t excess_slots = sweep_slots - max_free_slots; + size_t total_heap_pages = 0; + for (int i = 0; i < HEAP_COUNT; i++) { + total_heap_pages += heaps[i].total_pages; + } + /* Convert excess slots to pages using the actual average slots + * per page rather than HEAP_PAGE_OBJ_LIMIT (which is only correct + * for pool 0). Larger pools have far fewer slots per page, so using + * HEAP_PAGE_OBJ_LIMIT dramatically underestimates freeable pages. */ + if (total_slots > 0 && total_heap_pages > 0) { + heap_pages_freeable_pages = excess_slots * total_heap_pages / total_slots; + } + else { + heap_pages_freeable_pages = 0; + } +>>>>>>> a9b735748e (Calculate freeable pages based on how many actual slots are in each page) } else { heap_pages_freeable_pages = 0; @@ -9511,11 +9531,12 @@ rb_gc_impl_objspace_init(void *objspace_ptr) rb_bug("Could not preregister postponed job for GC"); } + GC_ASSERT(sizeof(struct RBasic) + sizeof(VALUE[RBIMPL_RVALUE_EMBED_LEN_MAX]) + RVALUE_OVERHEAD <= (BASE_SLOT_SIZE << 1)); + for (int i = 0; i < HEAP_COUNT; i++) { rb_heap_t *heap = &heaps[i]; - heap->slot_size = (1 << i) * BASE_SLOT_SIZE; - slot_div_magics[i] = (uint32_t)((uint64_t)UINT32_MAX / heap->slot_size + 1); + heap->slot_size = BASE_SLOT_SIZE << i; ccan_list_head_init(&heap->pages); } @@ -9537,8 +9558,10 @@ rb_gc_impl_objspace_init(void *objspace_ptr) #endif /* Set size pools allocatable pages. */ for (int i = 0; i < HEAP_COUNT; i++) { - /* Set the default value of heap_init_slots. */ - gc_params.heap_init_slots[i] = GC_HEAP_INIT_SLOTS; + /* Set the default value of heap_init_slots. + * Scale inversely with slot size so each pool gets an equal byte + * budget (GC_HEAP_INIT_SLOTS * BASE_SLOT_SIZE bytes). */ + gc_params.heap_init_slots[i] = GC_HEAP_INIT_SLOTS >> i; } init_mark_stack(&objspace->mark_stack); @@ -9553,6 +9576,11 @@ rb_gc_impl_init(void) VALUE gc_constants = rb_hash_new(); rb_hash_aset(gc_constants, ID2SYM(rb_intern("DEBUG")), GC_DEBUG ? Qtrue : Qfalse); rb_hash_aset(gc_constants, ID2SYM(rb_intern("BASE_SLOT_SIZE")), SIZET2NUM(BASE_SLOT_SIZE - RVALUE_OVERHEAD)); + /* Minimum slot size for a standard RVALUE (RBasic + embedded VALUEs) */ + size_t rvalue_min = sizeof(struct RBasic) + sizeof(VALUE[RBIMPL_RVALUE_EMBED_LEN_MAX]) + RVALUE_OVERHEAD; + size_t rvalue_slot = BASE_SLOT_SIZE; + while (rvalue_slot < rvalue_min) rvalue_slot <<= 1; + rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_SIZE")), SIZET2NUM(rvalue_slot - RVALUE_OVERHEAD)); rb_hash_aset(gc_constants, ID2SYM(rb_intern("RBASIC_SIZE")), SIZET2NUM(sizeof(struct RBasic))); rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_OVERHEAD")), SIZET2NUM(RVALUE_OVERHEAD)); rb_hash_aset(gc_constants, ID2SYM(rb_intern("HEAP_PAGE_BITMAP_SIZE")), SIZET2NUM(HEAP_PAGE_BITMAP_SIZE)); diff --git a/gc/mmtk/mmtk.c b/gc/mmtk/mmtk.c index 4832916ce6ea2f..6b38fa9a681c33 100644 --- a/gc/mmtk/mmtk.c +++ b/gc/mmtk/mmtk.c @@ -618,17 +618,24 @@ void rb_gc_impl_set_params(void *objspace_ptr) { } static VALUE gc_verify_internal_consistency(VALUE self) { return Qnil; } #define MMTK_HEAP_COUNT 6 -#define MMTK_MAX_OBJ_SIZE 640 - +#if SIZEOF_VALUE >= 8 +#define MMTK_MAX_OBJ_SIZE 1024 static size_t heap_sizes[MMTK_HEAP_COUNT + 1] = { - 32, 40, 80, 160, 320, MMTK_MAX_OBJ_SIZE, 0 + 32, 64, 128, 256, 512, MMTK_MAX_OBJ_SIZE, 0 }; +#else +#define MMTK_MAX_OBJ_SIZE 512 +static size_t heap_sizes[MMTK_HEAP_COUNT + 1] = { + 16, 32, 64, 128, 256, MMTK_MAX_OBJ_SIZE, 0 +}; +#endif void rb_gc_impl_init(void) { VALUE gc_constants = rb_hash_new(); - rb_hash_aset(gc_constants, ID2SYM(rb_intern("BASE_SLOT_SIZE")), SIZET2NUM(sizeof(VALUE) * 5)); + rb_hash_aset(gc_constants, ID2SYM(rb_intern("BASE_SLOT_SIZE")), SIZET2NUM(SIZEOF_VALUE >= 8 ? 64 : 32)); + rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_SIZE")), SIZET2NUM(SIZEOF_VALUE >= 8 ? 64 : 32)); rb_hash_aset(gc_constants, ID2SYM(rb_intern("RBASIC_SIZE")), SIZET2NUM(sizeof(struct RBasic))); rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_OVERHEAD")), INT2NUM(0)); rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVARGC_MAX_ALLOCATE_SIZE")), LONG2FIX(MMTK_MAX_OBJ_SIZE)); diff --git a/internal/class.h b/internal/class.h index ea68b07fc20968..08facfd00726e9 100644 --- a/internal/class.h +++ b/internal/class.h @@ -149,7 +149,7 @@ struct RClass_and_rb_classext_t { }; #if SIZEOF_VALUE >= SIZEOF_LONG_LONG -// Assert that classes can be embedded in heaps[2] (which has 160B slot size) +// Assert that classes can be embedded in heaps[2] (which has 256B slot size) // On 32bit platforms there is no variable width allocation so it doesn't matter. STATIC_ASSERT(sizeof_rb_classext_t, sizeof(struct RClass_and_rb_classext_t) <= 4 * RVALUE_SIZE); #endif diff --git a/object.c b/object.c index 4dcd5d615f85a9..d3036c52f0a7c0 100644 --- a/object.c +++ b/object.c @@ -93,6 +93,7 @@ static ID id_instance_variables_to_inspect; size_t rb_obj_embedded_size(uint32_t fields_count) { + if (fields_count < 1) fields_count = 1; return offsetof(struct RObject, as.ary) + (sizeof(VALUE) * fields_count); } diff --git a/shape.c b/shape.c index 90036722f10026..93ccd3eb595fdd 100644 --- a/shape.c +++ b/shape.c @@ -477,14 +477,14 @@ static attr_index_t shape_grow_capa(attr_index_t current_capa) { const attr_index_t *capacities = rb_shape_tree.capacities; + size_t heaps_count = rb_shape_tree.heaps_count; // First try to use the next size that will be embeddable in a larger object slot. - attr_index_t capa; - while ((capa = *capacities)) { + for (size_t i = 0; i < heaps_count; i++) { + attr_index_t capa = capacities[i]; if (capa > current_capa) { return capa; } - capacities++; } return (attr_index_t)rb_malloc_grow_capa(current_capa, sizeof(VALUE)); @@ -1543,8 +1543,14 @@ Init_default_shapes(void) capacities[heaps_count] = 0; size_t index; for (index = 0; index < heaps_count; index++) { - capacities[index] = (heap_sizes[index] - sizeof(struct RBasic)) / sizeof(VALUE); + if (heap_sizes[index] > sizeof(struct RBasic)) { + capacities[index] = (heap_sizes[index] - sizeof(struct RBasic)) / sizeof(VALUE); + } + else { + capacities[index] = 0; + } } + rb_shape_tree.heaps_count = heaps_count; rb_shape_tree.capacities = capacities; #ifdef HAVE_MMAP diff --git a/shape.h b/shape.h index 96c78f2bc1a356..1296e62a1a4d7f 100644 --- a/shape.h +++ b/shape.h @@ -115,6 +115,7 @@ typedef struct { rb_shape_t *shape_list; rb_shape_t *root_shape; const attr_index_t *capacities; + size_t heaps_count; rb_atomic_t next_shape_id; redblack_node_t *shape_cache; diff --git a/test/-ext-/string/test_capacity.rb b/test/-ext-/string/test_capacity.rb index df000f7cdb8103..80a2d5db91cd45 100644 --- a/test/-ext-/string/test_capacity.rb +++ b/test/-ext-/string/test_capacity.rb @@ -5,13 +5,13 @@ class Test_StringCapacity < Test::Unit::TestCase def test_capacity_embedded - assert_equal GC::INTERNAL_CONSTANTS[:BASE_SLOT_SIZE] - embed_header_size - 1, capa('foo') + assert_equal GC::INTERNAL_CONSTANTS[:RVALUE_SIZE] - embed_header_size - 1, capa('foo') assert_equal max_embed_len, capa('1' * max_embed_len) assert_equal max_embed_len, capa('1' * (max_embed_len - 1)) end def test_capacity_shared - sym = ("a" * GC::INTERNAL_CONSTANTS[:BASE_SLOT_SIZE]).to_sym + sym = ("a" * GC::INTERNAL_CONSTANTS[:RVALUE_SIZE]).to_sym assert_equal 0, capa(sym.to_s) end diff --git a/test/-ext-/string/test_set_len.rb b/test/-ext-/string/test_set_len.rb index 1531d76167c35c..a18bbbc70cd237 100644 --- a/test/-ext-/string/test_set_len.rb +++ b/test/-ext-/string/test_set_len.rb @@ -5,7 +5,7 @@ class Test_StrSetLen < Test::Unit::TestCase def setup # Make string long enough so that it is not embedded - @range_end = ("0".ord + GC::INTERNAL_CONSTANTS[:BASE_SLOT_SIZE]).chr + @range_end = ("0".ord + GC::INTERNAL_CONSTANTS[:RVALUE_SIZE]).chr @s0 = [*"0"..@range_end].join("").freeze @s1 = Bug::String.new(@s0) end diff --git a/test/objspace/test_objspace.rb b/test/objspace/test_objspace.rb index d631f97d1bcad8..92e58219d69bdf 100644 --- a/test/objspace/test_objspace.rb +++ b/test/objspace/test_objspace.rb @@ -33,7 +33,7 @@ def test_memsize_of_root_shared_string b = a.dup c = nil ObjectSpace.each_object(String) {|x| break c = x if a == x and x.frozen?} - rv_size = GC::INTERNAL_CONSTANTS[:BASE_SLOT_SIZE] + rv_size = GC::INTERNAL_CONSTANTS[:RVALUE_SIZE] assert_equal([rv_size, rv_size, a.length + 1 + rv_size], [a, b, c].map {|x| ObjectSpace.memsize_of(x)}) end @@ -473,12 +473,12 @@ def test_dump_object assert_include(info, '"embedded":true') assert_include(info, '"ivars":0') - # Non-embed object + # Non-embed object (needs > 6 ivars to exceed pool 0 embed capacity) obj = klass.new - 5.times { |i| obj.instance_variable_set("@ivar#{i}", 0) } + 7.times { |i| obj.instance_variable_set("@ivar#{i}", 0) } info = ObjectSpace.dump(obj) assert_not_include(info, '"embedded":true') - assert_include(info, '"ivars":5') + assert_include(info, '"ivars":7') end def test_dump_control_char diff --git a/test/ruby/test_file_exhaustive.rb b/test/ruby/test_file_exhaustive.rb index be9e6bd44e702d..3f9a71253331e7 100644 --- a/test/ruby/test_file_exhaustive.rb +++ b/test/ruby/test_file_exhaustive.rb @@ -897,10 +897,10 @@ def test_expand_path_memsize bug9934 = '[ruby-core:63114] [Bug #9934]' require "objspace" path = File.expand_path("/foo") - assert_operator(ObjectSpace.memsize_of(path), :<=, path.bytesize + GC::INTERNAL_CONSTANTS[:BASE_SLOT_SIZE], bug9934) + assert_operator(ObjectSpace.memsize_of(path), :<=, path.bytesize + GC::INTERNAL_CONSTANTS[:RVALUE_SIZE], bug9934) path = File.expand_path("/a"*25) assert_operator(ObjectSpace.memsize_of(path), :<=, - (path.bytesize + 1) * 2 + GC::INTERNAL_CONSTANTS[:BASE_SLOT_SIZE], bug9934) + (path.bytesize + 1) * 2 + GC::INTERNAL_CONSTANTS[:RVALUE_SIZE], bug9934) end def test_expand_path_encoding diff --git a/test/ruby/test_gc_compact.rb b/test/ruby/test_gc_compact.rb index f3da8e4e138432..2964a67710657e 100644 --- a/test/ruby/test_gc_compact.rb +++ b/test/ruby/test_gc_compact.rb @@ -315,7 +315,7 @@ def test_moving_arrays_up_heaps GC.verify_compaction_references(expand_heap: true, toward: :empty) Fiber.new { - ary = "hello".chars + ary = "hello world".chars # > 6 elements to exceed pool 0 embed capacity $arys = ARY_COUNT.times.map do x = [] ary.each { |e| x << e } diff --git a/test/ruby/test_object.rb b/test/ruby/test_object.rb index f4dfe2251b884f..e64789d3c410f6 100644 --- a/test/ruby/test_object.rb +++ b/test/ruby/test_object.rb @@ -371,12 +371,17 @@ def initialize o1 = c.new o2 = c.new - o1.instance_variable_set(:@foo, 5) + # Add enough ivars to exceed pool 0 embed capacity (6 on 64-bit) + o1.instance_variable_set(:@d, 3) + o1.instance_variable_set(:@e, 4) + o1.instance_variable_set(:@f, 5) + o1.instance_variable_set(:@foo, 6) o1.instance_variable_set(:@a, 0) o1.instance_variable_set(:@b, 1) o1.instance_variable_set(:@c, 2) refute_includes ObjectSpace.dump(o1), '"embedded":true' o1.remove_instance_variable(:@foo) + o1.remove_instance_variable(:@f) assert_includes ObjectSpace.dump(o1), '"embedded":true' o2.instance_variable_set(:@a, 0) diff --git a/test/ruby/test_time.rb b/test/ruby/test_time.rb index 333edb80218a64..80b637d433ee56 100644 --- a/test/ruby/test_time.rb +++ b/test/ruby/test_time.rb @@ -1433,7 +1433,10 @@ def test_memsize RbConfig::SIZEOF["void*"] # Same size as VALUE end sizeof_vtm = RbConfig::SIZEOF["void*"] * 4 + 8 - expect = GC::INTERNAL_CONSTANTS[:BASE_SLOT_SIZE] + sizeof_timew + sizeof_vtm + data_size = GC::INTERNAL_CONSTANTS[:BASE_SLOT_SIZE] + sizeof_timew + sizeof_vtm + # Round up to the next slot size (pools are powers of 2) + expect = GC::INTERNAL_CONSTANTS[:BASE_SLOT_SIZE] + expect <<= 1 while expect < data_size assert_operator ObjectSpace.memsize_of(t), :<=, expect rescue LoadError => e omit "failed to load objspace: #{e.message}"