diff options
author | Matt Valentine-House <[email protected]> | 2024-10-03 13:53:49 +0100 |
---|---|---|
committer | Matt Valentine-House <[email protected]> | 2024-10-03 21:20:09 +0100 |
commit | 8e7df4b7c674cf408fa570b9593811167bbff04a () | |
tree | fe143affd6601f6fc1a94d5a25084696ea8f660e | |
parent | b58a3645229b6c82c1f199fd948ec1fa97c0cc10 (diff) |
Rename size_pool -> heap
Now that we've inlined the eden_heap into the size_pool, we should rename the size_pool to heap. So that Ruby contains multiple heaps, with different sized objects. The term heap as a collection of memory pages is more in memory management nomenclature, whereas size_pool was a name chosen out of necessity during the development of the Variable Width Allocation features of Ruby. The concept of size pools was introduced in order to facilitate different sized objects (other than the default 40 bytes). They wrapped the eden heap and the tomb heap, and some related state, and provided a reasonably simple way of duplicating all related concerns, to provide multiple pools that all shared the same structure but held different objects. Since then various changes have happend in Ruby's memory layout: * The concept of tomb heaps has been replaced by a global free pages list, with each page having it's slot size reconfigured at the point when it is resurrected * the eden heap has been inlined into the size pool itself, so that now the size pool directly controls the free_pages list, the sweeping page, the compaction cursor and the other state that was previously being managed by the eden heap. Now that there is no need for a heap wrapper, we should refer to the collection of pages containing Ruby objects as a heap again rather than a size pool
Notes: Merged: https://.com/ruby/ruby/pull/11771
-rw-r--r-- | benchmark/string_concat.yml | 4 | ||||
-rw-r--r-- | gc.c | 24 | ||||
-rw-r--r-- | gc/default.c | 790 | ||||
-rw-r--r-- | gc/gc.h | 2 | ||||
-rw-r--r-- | gc/gc_impl.h | 4 | ||||
-rw-r--r-- | internal/class.h | 2 | ||||
-rw-r--r-- | internal/gc.h | 4 | ||||
-rwxr-xr-x | misc/lldb_cruby.py | 2 | ||||
-rw-r--r-- | object.c | 4 | ||||
-rw-r--r-- | rjit_c.rb | 2 | ||||
-rw-r--r-- | shape.c | 14 | ||||
-rw-r--r-- | shape.h | 2 | ||||
-rw-r--r-- | test/ruby/test_gc.rb | 10 | ||||
-rw-r--r-- | test/ruby/test_gc_compact.rb | 14 | ||||
-rw-r--r-- | test/ruby/test_string.rb | 4 | ||||
-rw-r--r-- | yjit/src/cruby_bindings.inc.rs | 2 |
16 files changed, 442 insertions, 442 deletions
@@ -45,7 +45,7 @@ benchmark: "#{CHUNK}#{CHUNK}#{CHUNK}#{CHUNK}#{CHUNK}#{CHUNK}#{CHUNK}#{CHUNK}#{CHUNK}#{CHUNK}" \ "#{CHUNK}#{CHUNK}#{CHUNK}#{CHUNK}#{CHUNK}#{CHUNK}#{CHUNK}#{CHUNK}#{CHUNK}#{CHUNK}" \ "#{CHUNK}#{CHUNK}#{CHUNK}#{CHUNK}#{CHUNK}#{CHUNK}#{CHUNK}#{CHUNK}#{CHUNK}" - interpolation_same_size_pool: | buffer = "#{SHORT}#{SHORT}" - interpolation_switching_size_pools: | buffer = "#{SHORT}#{LONG}" @@ -310,13 +310,13 @@ rb_gc_set_shape(VALUE obj, uint32_t shape_id) } uint32_t -rb_gc_rebuild_shape(VALUE obj, size_t size_pool_id) { rb_shape_t *orig_shape = rb_shape_get_shape(obj); if (rb_shape_obj_too_complex(obj)) return (uint32_t)OBJ_TOO_COMPLEX_SHAPE_ID; - rb_shape_t *initial_shape = rb_shape_get_shape_by_id((shape_id_t)(size_pool_id + FIRST_T_OBJECT_SHAPE_ID)); rb_shape_t *new_shape = rb_shape_traverse_from_new_root(initial_shape, orig_shape); if (!new_shape) return 0; @@ -577,7 +577,7 @@ typedef struct gc_function_map { void (*ractor_cache_free)(void *objspace_ptr, void *cache); void (*set_params)(void *objspace_ptr); void (*init)(void); - size_t *(*size_pool_sizes)(void *objspace_ptr); // Shutdown void (*shutdown_free_objects)(void *objspace_ptr); // GC @@ -594,7 +594,7 @@ typedef struct gc_function_map { // Object allocation VALUE (*new_obj)(void *objspace_ptr, void *cache_ptr, VALUE klass, VALUE flags, VALUE v1, VALUE v2, VALUE v3, bool wb_protected, size_t alloc_size); size_t (*obj_slot_size)(VALUE obj); - size_t (*size_pool_id_for_size)(void *objspace_ptr, size_t size); bool (*size_allocatable_p)(size_t size); // Malloc void *(*malloc)(void *objspace_ptr, size_t size); @@ -708,7 +708,7 @@ ruby_external_gc_init(void) load_external_gc_func(ractor_cache_free); load_external_gc_func(set_params); load_external_gc_func(init); - load_external_gc_func(size_pool_sizes); // Shutdown load_external_gc_func(shutdown_free_objects); // GC @@ -725,7 +725,7 @@ ruby_external_gc_init(void) // Object allocation load_external_gc_func(new_obj); load_external_gc_func(obj_slot_size); - load_external_gc_func(size_pool_id_for_size); load_external_gc_func(size_allocatable_p); // Malloc load_external_gc_func(malloc); @@ -787,7 +787,7 @@ ruby_external_gc_init(void) # define rb_gc_impl_ractor_cache_free rb_gc_functions.ractor_cache_free # define rb_gc_impl_set_params rb_gc_functions.set_params # define rb_gc_impl_init rb_gc_functions.init -# define rb_gc_impl_size_pool_sizes rb_gc_functions.size_pool_sizes // Shutdown # define rb_gc_impl_shutdown_free_objects rb_gc_functions.shutdown_free_objects // GC @@ -804,7 +804,7 @@ ruby_external_gc_init(void) // Object allocation # define rb_gc_impl_new_obj rb_gc_functions.new_obj # define rb_gc_impl_obj_slot_size rb_gc_functions.obj_slot_size -# define rb_gc_impl_size_pool_id_for_size rb_gc_functions.size_pool_id_for_size # define rb_gc_impl_size_allocatable_p rb_gc_functions.size_allocatable_p // Malloc # define rb_gc_impl_malloc rb_gc_functions.malloc @@ -3000,9 +3000,9 @@ rb_gc_prepare_heap(void) } size_t -rb_gc_size_pool_id_for_size(size_t size) { - return rb_gc_impl_size_pool_id_for_size(rb_gc_get_objspace(), size); } bool @@ -3452,9 +3452,9 @@ rb_gc_initial_stress_set(VALUE flag) } size_t * -rb_gc_size_pool_sizes(void) { - return rb_gc_impl_size_pool_sizes(rb_gc_get_objspace()); } VALUE @@ -150,22 +150,22 @@ #define USE_TICK_T (PRINT_ENTER_EXIT_TICK || PRINT_ROOT_TICKS) -#ifndef SIZE_POOL_COUNT -# define SIZE_POOL_COUNT 5 #endif -typedef struct ractor_newobj_size_pool_cache { struct free_slot *freelist; struct heap_page *using_page; -} rb_ractor_newobj_size_pool_cache_t; typedef struct ractor_newobj_cache { size_t incremental_mark_step_allocated_slots; - rb_ractor_newobj_size_pool_cache_t size_pool_caches[SIZE_POOL_COUNT]; } rb_ractor_newobj_cache_t; typedef struct { - size_t size_pool_init_slots[SIZE_POOL_COUNT]; size_t heap_free_slots; double growth_factor; size_t growth_max_slots; @@ -405,7 +405,7 @@ typedef struct mark_stack { typedef int (*gc_compact_compare_func)(const void *l, const void *r, void *d); -typedef struct rb_size_pool_struct { short slot_size; /* Basic statistics */ @@ -429,7 +429,7 @@ typedef struct rb_size_pool_struct { size_t total_pages; /* total page count in a heap */ size_t total_slots; /* total slot count (about total_pages * HEAP_PAGE_OBJ_LIMIT) */ -} rb_size_pool_t; enum { gc_stress_no_major, @@ -477,7 +477,7 @@ typedef struct rb_objspace { rb_event_flag_t hook_events; unsigned long long next_object_id; - rb_size_pool_t size_pools[SIZE_POOL_COUNT]; size_t empty_pages_count; struct heap_page *empty_pages; @@ -724,7 +724,7 @@ struct heap_page { unsigned int has_uncollectible_wb_unprotected_objects : 1; } flags; - rb_size_pool_t *size_pool; struct heap_page *free_next; struct heap_page_body *body; @@ -769,7 +769,7 @@ heap_page_in_global_empty_pages_pool(rb_objspace_t *objspace, struct heap_page * if (page->total_slots == 0) { GC_ASSERT(page->start == 0); GC_ASSERT(page->slot_size == 0); - GC_ASSERT(page->size_pool == NULL); GC_ASSERT(page->free_slots == 0); asan_unpoisoning_memory_region(&page->freelist, sizeof(&page->freelist)) { GC_ASSERT(page->freelist == NULL); @@ -780,7 +780,7 @@ heap_page_in_global_empty_pages_pool(rb_objspace_t *objspace, struct heap_page * else { GC_ASSERT(page->start != 0); GC_ASSERT(page->slot_size != 0); - GC_ASSERT(page->size_pool != NULL); return false; } @@ -843,7 +843,7 @@ RVALUE_AGE_SET(VALUE obj, int age) #define heap_pages_himem objspace->heap_pages.range[1] #define heap_pages_freeable_pages objspace->heap_pages.freeable_pages #define heap_pages_deferred_final objspace->heap_pages.deferred_final -#define size_pools objspace->size_pools #define during_gc objspace->flags.during_gc #define finalizing objspace->atomic_flags.finalizing #define finalizer_table objspace->finalizer_table @@ -904,8 +904,8 @@ gc_mode_verify(enum gc_mode mode) static inline bool has_sweeping_pages(rb_objspace_t *objspace) { - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - if ((&size_pools[i])->sweeping_page) { return TRUE; } } @@ -916,8 +916,8 @@ static inline size_t heap_eden_total_pages(rb_objspace_t *objspace) { size_t count = 0; - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - count += (&size_pools[i])->total_pages; } return count; } @@ -926,9 +926,9 @@ static inline size_t total_allocated_objects(rb_objspace_t *objspace) { size_t count = 0; - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - rb_size_pool_t *size_pool = &size_pools[i]; - count += size_pool->total_allocated_objects; } return count; } @@ -937,9 +937,9 @@ static inline size_t total_freed_objects(rb_objspace_t *objspace) { size_t count = 0; - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - rb_size_pool_t *size_pool = &size_pools[i]; - count += size_pool->total_freed_objects; } return count; } @@ -948,9 +948,9 @@ static inline size_t total_final_slots_count(rb_objspace_t *objspace) { size_t count = 0; - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - rb_size_pool_t *size_pool = &size_pools[i]; - count += size_pool->final_slots_count; } return count; } @@ -1011,11 +1011,11 @@ static void gc_marking_enter(rb_objspace_t *objspace); static void gc_marking_exit(rb_objspace_t *objspace); static void gc_sweeping_enter(rb_objspace_t *objspace); static void gc_sweeping_exit(rb_objspace_t *objspace); -static bool gc_marks_continue(rb_objspace_t *objspace, rb_size_pool_t *size_pool); static void gc_sweep(rb_objspace_t *objspace); -static void gc_sweep_finish_size_pool(rb_objspace_t *objspace, rb_size_pool_t *size_pool); -static void gc_sweep_continue(rb_objspace_t *objspace, rb_size_pool_t *size_pool); static inline void gc_mark(rb_objspace_t *objspace, VALUE ptr); static inline void gc_pin(rb_objspace_t *objspace, VALUE ptr); @@ -1207,8 +1207,8 @@ RVALUE_UNCOLLECTIBLE(rb_objspace_t *objspace, VALUE obj) #define RVALUE_PAGE_MARKING(page, obj) MARKED_IN_BITMAP((page)->marking_bits, (obj)) static int rgengc_remember(rb_objspace_t *objspace, VALUE obj); -static void rgengc_mark_and_rememberset_clear(rb_objspace_t *objspace, rb_size_pool_t *size_pool); -static void rgengc_rememberset_mark(rb_objspace_t *objspace, rb_size_pool_t *size_pool); static int check_rvalue_consistency_force(rb_objspace_t *objspace, const VALUE obj, int terminate) @@ -1512,10 +1512,10 @@ rb_gc_impl_get_measure_total_time(void *objspace_ptr) } static size_t -minimum_slots_for_size_pool(rb_objspace_t *objspace, rb_size_pool_t *size_pool) { - size_t size_pool_idx = size_pool - size_pools; - return gc_params.size_pool_init_slots[size_pool_idx]; } static int @@ -1650,8 +1650,8 @@ heap_page_add_freeobj(rb_objspace_t *objspace, struct heap_page *page, VALUE obj } static void -size_pool_allocatable_slots_expand(rb_objspace_t *objspace, - rb_size_pool_t *size_pool, size_t free_slots, size_t total_slots) { double goal_ratio = gc_params.heap_free_slots_goal_ratio; size_t target_total_slots; @@ -1660,7 +1660,7 @@ size_pool_allocatable_slots_expand(rb_objspace_t *objspace, target_total_slots = (size_t)(total_slots * gc_params.growth_factor); } else if (total_slots == 0) { - target_total_slots = minimum_slots_for_size_pool(objspace, size_pool); } else { /* Find `f' where free_slots = f * total_slots * goal_ratio @@ -1696,14 +1696,14 @@ size_pool_allocatable_slots_expand(rb_objspace_t *objspace, } static inline void -heap_add_freepage(rb_size_pool_t *size_pool, struct heap_page *page) { asan_unlock_freelist(page); GC_ASSERT(page->free_slots != 0); GC_ASSERT(page->freelist != NULL); - page->free_next = size_pool->free_pages; - size_pool->free_pages = page; RUBY_DEBUG_LOG("page:%p freelist:%p", (void *)page, (void *)page->freelist); @@ -1711,25 +1711,25 @@ heap_add_freepage(rb_size_pool_t *size_pool, struct heap_page *page) } static inline void -heap_add_poolpage(rb_objspace_t *objspace, rb_size_pool_t *size_pool, struct heap_page *page) { asan_unlock_freelist(page); GC_ASSERT(page->free_slots != 0); GC_ASSERT(page->freelist != NULL); - page->free_next = size_pool->pooled_pages; - size_pool->pooled_pages = page; objspace->rincgc.pooled_slots += page->free_slots; asan_lock_freelist(page); } static void -heap_unlink_page(rb_objspace_t *objspace, rb_size_pool_t *size_pool, struct heap_page *page) { ccan_list_del(&page->page_node); - size_pool->total_pages--; - size_pool->total_slots -= page->total_slots; } static void @@ -1779,7 +1779,7 @@ heap_pages_free_unused_pages(rb_objspace_t *objspace) // Get number of pages estimated for the smallest size pool CEILDIV(objspace->heap_pages.allocatable_slots, HEAP_PAGE_OBJ_LIMIT) * // Estimate the average slot size multiple - (1 << (SIZE_POOL_COUNT / 2)); if (objspace->empty_pages != NULL && objspace->empty_pages_count > pages_to_keep_count) { GC_ASSERT(objspace->empty_pages_count > 0); @@ -1968,10 +1968,10 @@ heap_page_allocate(rb_objspace_t *objspace) } static void -size_pool_add_page(rb_objspace_t *objspace, rb_size_pool_t *size_pool, struct heap_page *page) { /* Adding to eden heap during incremental sweeping is forbidden */ - GC_ASSERT(!size_pool->sweeping_page); GC_ASSERT(heap_page_in_global_empty_pages_pool(objspace, page)); /* adjust obj_limit (object number available in this page) */ @@ -1986,50 +1986,50 @@ size_pool_add_page(rb_objspace_t *objspace, rb_size_pool_t *size_pool, struct he * In other words, ensure there are an even number of objects * per bit plane. */ if (NUM_IN_PAGE(start) == 1) { - start += size_pool->slot_size - BASE_SLOT_SIZE; } - GC_ASSERT(NUM_IN_PAGE(start) * BASE_SLOT_SIZE % size_pool->slot_size == 0); } - int slot_count = (int)((HEAP_PAGE_SIZE - (start - (uintptr_t)page->body))/size_pool->slot_size); page->start = start; page->total_slots = slot_count; - page->slot_size = size_pool->slot_size; - page->size_pool = size_pool; asan_unlock_freelist(page); page->freelist = NULL; asan_unpoison_memory_region(page->body, HEAP_PAGE_SIZE, false); - for (VALUE p = (VALUE)start; p < start + (slot_count * size_pool->slot_size); p += size_pool->slot_size) { heap_page_add_freeobj(objspace, page, p); } asan_lock_freelist(page); page->free_slots = slot_count; - size_pool->total_allocated_pages++; - ccan_list_add_tail(&size_pool->pages, &page->page_node); - size_pool->total_pages++; - size_pool->total_slots += page->total_slots; } static int -heap_page_allocate_and_initialize(rb_objspace_t *objspace, rb_size_pool_t *size_pool) { if (objspace->heap_pages.allocatable_slots > 0) { gc_report(1, objspace, "heap_page_allocate_and_initialize: rb_darray_size(objspace->heap_pages.sorted): %"PRIdSIZE", " "allocatable_slots: %"PRIdSIZE", heap->total_pages: %"PRIdSIZE"\n", - rb_darray_size(objspace->heap_pages.sorted), objspace->heap_pages.allocatable_slots, size_pool->total_pages); struct heap_page *page = heap_page_resurrect(objspace); if (page == NULL) { page = heap_page_allocate(objspace); } - size_pool_add_page(objspace, size_pool, page); - heap_add_freepage(size_pool, page); if (objspace->heap_pages.allocatable_slots > (size_t)page->total_slots) { objspace->heap_pages.allocatable_slots -= page->total_slots; @@ -2045,77 +2045,77 @@ heap_page_allocate_and_initialize(rb_objspace_t *objspace, rb_size_pool_t *size_ } static void -heap_page_allocate_and_initialize_force(rb_objspace_t *objspace, rb_size_pool_t *size_pool) { size_t prev_allocatable_slots = objspace->heap_pages.allocatable_slots; // Set allocatable slots to 1 to force a page to be created. objspace->heap_pages.allocatable_slots = 1; - heap_page_allocate_and_initialize(objspace, size_pool); - GC_ASSERT(size_pool->free_pages != NULL); objspace->heap_pages.allocatable_slots = prev_allocatable_slots; } static void -gc_continue(rb_objspace_t *objspace, rb_size_pool_t *size_pool) { unsigned int lock_lev; gc_enter(objspace, gc_enter_event_continue, &lock_lev); /* Continue marking if in incremental marking. */ if (is_incremental_marking(objspace)) { - if (gc_marks_continue(objspace, size_pool)) { gc_sweep(objspace); } } /* Continue sweeping if in lazy sweeping or the previous incremental * marking finished and did not yield a free page. */ - if (size_pool->free_pages == NULL && is_lazy_sweeping(objspace)) { - gc_sweep_continue(objspace, size_pool); } gc_exit(objspace, gc_enter_event_continue, &lock_lev); } static void -heap_prepare(rb_objspace_t *objspace, rb_size_pool_t *size_pool) { - GC_ASSERT(size_pool->free_pages == NULL); - if (size_pool->total_slots < gc_params.size_pool_init_slots[size_pool - size_pools] && - size_pool->sweeping_page == NULL) { - heap_page_allocate_and_initialize_force(objspace, size_pool); - GC_ASSERT(size_pool->free_pages != NULL); return; } /* Continue incremental marking or lazy sweeping, if in any of those steps. */ - gc_continue(objspace, size_pool); - if (size_pool->free_pages == NULL) { - heap_page_allocate_and_initialize(objspace, size_pool); } /* If we still don't have a free page and not allowed to create a new page, * we should start a new GC cycle. */ - if (size_pool->free_pages == NULL) { if (gc_start(objspace, GPR_FLAG_NEWOBJ) == FALSE) { rb_memerror(); } else { if (objspace->heap_pages.allocatable_slots == 0 && !gc_config_full_mark_val) { - size_pool_allocatable_slots_expand(objspace, size_pool, - size_pool->freed_slots + size_pool->empty_slots, - size_pool->total_slots); GC_ASSERT(objspace->heap_pages.allocatable_slots > 0); } /* Do steps of incremental marking or lazy sweeping if the GC run permits. */ - gc_continue(objspace, size_pool); /* If we're not incremental marking (e.g. a minor GC) or finished * sweeping and still don't have a free page, then - * gc_sweep_finish_size_pool should allow us to create a new page. */ - if (size_pool->free_pages == NULL && !heap_page_allocate_and_initialize(objspace, size_pool)) { if (gc_needs_major_flags == GPR_FLAG_NONE) { rb_bug("cannot create a new page after GC"); } @@ -2125,10 +2125,10 @@ heap_prepare(rb_objspace_t *objspace, rb_size_pool_t *size_pool) } else { /* Do steps of incremental marking or lazy sweeping. */ - gc_continue(objspace, size_pool); - if (size_pool->free_pages == NULL && - !heap_page_allocate_and_initialize(objspace, size_pool)) { rb_bug("cannot create a new page after major GC"); } } @@ -2137,7 +2137,7 @@ heap_prepare(rb_objspace_t *objspace, rb_size_pool_t *size_pool) } } - GC_ASSERT(size_pool->free_pages != NULL); } static inline VALUE @@ -2242,15 +2242,15 @@ rb_gc_impl_obj_slot_size(VALUE obj) } static inline size_t -size_pool_slot_size(unsigned char pool_id) { - GC_ASSERT(pool_id < SIZE_POOL_COUNT); size_t slot_size = (1 << pool_id) * BASE_SLOT_SIZE; #if RGENGC_CHECK_MODE rb_objspace_t *objspace = rb_gc_get_objspace(); - GC_ASSERT(size_pools[pool_id].slot_size == (short)slot_size); #endif slot_size -= RVALUE_OVERHEAD; @@ -2261,15 +2261,15 @@ size_pool_slot_size(unsigned char pool_id) bool rb_gc_impl_size_allocatable_p(size_t size) { - return size <= size_pool_slot_size(SIZE_POOL_COUNT - 1); } static inline VALUE ractor_cache_allocate_slot(rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, - size_t size_pool_idx) { - rb_ractor_newobj_size_pool_cache_t *size_pool_cache = &cache->size_pool_caches[size_pool_idx]; - struct free_slot *p = size_pool_cache->freelist; if (RB_UNLIKELY(is_incremental_marking(objspace))) { // Not allowed to allocate without running an incremental marking step @@ -2284,8 +2284,8 @@ ractor_cache_allocate_slot(rb_objspace_t *objspace, rb_ractor_newobj_cache_t *ca if (RB_LIKELY(p)) { VALUE obj = (VALUE)p; - MAYBE_UNUSED(const size_t) stride = size_pool_slot_size(size_pool_idx); - size_pool_cache->freelist = p->next; asan_unpoison_memory_region(p, stride, true); #if RGENGC_CHECK_MODE GC_ASSERT(rb_gc_impl_obj_slot_size(obj) == stride); @@ -2300,16 +2300,16 @@ ractor_cache_allocate_slot(rb_objspace_t *objspace, rb_ractor_newobj_cache_t *ca } static struct heap_page * -heap_next_free_page(rb_objspace_t *objspace, rb_size_pool_t *size_pool) { struct heap_page *page; - if (size_pool->free_pages == NULL) { - heap_prepare(objspace, size_pool); } - page = size_pool->free_pages; - size_pool->free_pages = page->free_next; GC_ASSERT(page->free_slots != 0); @@ -2319,78 +2319,78 @@ heap_next_free_page(rb_objspace_t *objspace, rb_size_pool_t *size_pool) } static inline void -ractor_cache_set_page(rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, size_t size_pool_idx, struct heap_page *page) { gc_report(3, objspace, "ractor_set_cache: Using page %p\n", (void *)page->body); - rb_ractor_newobj_size_pool_cache_t *size_pool_cache = &cache->size_pool_caches[size_pool_idx]; - GC_ASSERT(size_pool_cache->freelist == NULL); GC_ASSERT(page->free_slots != 0); GC_ASSERT(page->freelist != NULL); - size_pool_cache->using_page = page; - size_pool_cache->freelist = page->freelist; page->free_slots = 0; page->freelist = NULL; - asan_unpoison_object((VALUE)size_pool_cache->freelist, false); - GC_ASSERT(RB_TYPE_P((VALUE)size_pool_cache->freelist, T_NONE)); - asan_poison_object((VALUE)size_pool_cache->freelist); } static inline size_t -size_pool_idx_for_size(size_t size) { size += RVALUE_OVERHEAD; size_t slot_count = CEILDIV(size, BASE_SLOT_SIZE); - /* size_pool_idx is ceil(log2(slot_count)) */ - size_t size_pool_idx = 64 - nlz_int64(slot_count - 1); - if (size_pool_idx >= SIZE_POOL_COUNT) { - rb_bug("size_pool_idx_for_size: allocation size too large " - "(size=%"PRIuSIZE"u, size_pool_idx=%"PRIuSIZE"u)", size, size_pool_idx); } #if RGENGC_CHECK_MODE rb_objspace_t *objspace = rb_gc_get_objspace(); - GC_ASSERT(size <= (size_t)size_pools[size_pool_idx].slot_size); - if (size_pool_idx > 0) GC_ASSERT(size > (size_t)size_pools[size_pool_idx - 1].slot_size); #endif - return size_pool_idx; } size_t -rb_gc_impl_size_pool_id_for_size(void *objspace_ptr, size_t size) { - return size_pool_idx_for_size(size); } -static size_t size_pool_sizes[SIZE_POOL_COUNT + 1] = { 0 }; size_t * -rb_gc_impl_size_pool_sizes(void *objspace_ptr) { - if (size_pool_sizes[0] == 0) { - for (unsigned char i = 0; i < SIZE_POOL_COUNT; i++) { - size_pool_sizes[i] = size_pool_slot_size(i); } } - return size_pool_sizes; } -NOINLINE(static VALUE newobj_cache_miss(rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, size_t size_pool_idx, bool vm_locked)); static VALUE -newobj_cache_miss(rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, size_t size_pool_idx, bool vm_locked) { - rb_size_pool_t *size_pool = &size_pools[size_pool_idx]; VALUE obj = Qfalse; unsigned int lev = 0; @@ -2404,20 +2404,20 @@ newobj_cache_miss(rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, size { if (is_incremental_marking(objspace)) { - gc_continue(objspace, size_pool); cache->incremental_mark_step_allocated_slots = 0; // Retry allocation after resetting incremental_mark_step_allocated_slots - obj = ractor_cache_allocate_slot(objspace, cache, size_pool_idx); } if (obj == Qfalse) { // Get next free page (possibly running GC) - struct heap_page *page = heap_next_free_page(objspace, size_pool); - ractor_cache_set_page(objspace, cache, size_pool_idx, page); // Retry allocation after moving to new page - obj = ractor_cache_allocate_slot(objspace, cache, size_pool_idx); } } @@ -2432,27 +2432,27 @@ newobj_cache_miss(rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, size } static VALUE -newobj_alloc(rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, size_t size_pool_idx, bool vm_locked) { - VALUE obj = ractor_cache_allocate_slot(objspace, cache, size_pool_idx); if (RB_UNLIKELY(obj == Qfalse)) { - obj = newobj_cache_miss(objspace, cache, size_pool_idx, vm_locked); } - rb_size_pool_t *size_pool = &size_pools[size_pool_idx]; - size_pool->total_allocated_objects++; GC_ASSERT(rb_gc_multi_ractor_p() || - size_pool->total_slots >= - (size_pool->total_allocated_objects - size_pool->total_freed_objects - size_pool->final_slots_count)); return obj; } -ALWAYS_INLINE(static VALUE newobj_slowpath(VALUE klass, VALUE flags, rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, int wb_protected, size_t size_pool_idx)); static inline VALUE -newobj_slowpath(VALUE klass, VALUE flags, rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, int wb_protected, size_t size_pool_idx) { VALUE obj; unsigned int lev; @@ -2473,7 +2473,7 @@ newobj_slowpath(VALUE klass, VALUE flags, rb_objspace_t *objspace, rb_ractor_new } } - obj = newobj_alloc(objspace, cache, size_pool_idx, true); newobj_init(klass, flags, wb_protected, objspace, obj); } rb_gc_cr_unlock(lev); @@ -2482,20 +2482,20 @@ newobj_slowpath(VALUE klass, VALUE flags, rb_objspace_t *objspace, rb_ractor_new } NOINLINE(static VALUE newobj_slowpath_wb_protected(VALUE klass, VALUE flags, - rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, size_t size_pool_idx)); NOINLINE(static VALUE newobj_slowpath_wb_unprotected(VALUE klass, VALUE flags, - rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, size_t size_pool_idx)); static VALUE -newobj_slowpath_wb_protected(VALUE klass, VALUE flags, rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, size_t size_pool_idx) { - return newobj_slowpath(klass, flags, objspace, cache, TRUE, size_pool_idx); } static VALUE -newobj_slowpath_wb_unprotected(VALUE klass, VALUE flags, rb_objspace_t *objspace, rb_ractor_newobj_cache_t *cache, size_t size_pool_idx) { - return newobj_slowpath(klass, flags, objspace, cache, FALSE, size_pool_idx); } VALUE @@ -2514,21 +2514,21 @@ rb_gc_impl_new_obj(void *objspace_ptr, void *cache_ptr, VALUE klass, VALUE flags } } - size_t size_pool_idx = size_pool_idx_for_size(alloc_size); rb_ractor_newobj_cache_t *cache = (rb_ractor_newobj_cache_t *)cache_ptr; if (!RB_UNLIKELY(during_gc || ruby_gc_stressful) && wb_protected) { - obj = newobj_alloc(objspace, cache, size_pool_idx, false); newobj_init(klass, flags, wb_protected, objspace, obj); } else { RB_DEBUG_COUNTER_INC(obj_newobj_slowpath); obj = wb_protected ? - newobj_slowpath_wb_protected(klass, flags, objspace, cache, size_pool_idx) : - newobj_slowpath_wb_unprotected(klass, flags, objspace, cache, size_pool_idx); } return newobj_fill(obj, v1, v2, v3); @@ -2628,7 +2628,7 @@ rb_gc_impl_make_zombie(void *objspace_ptr, VALUE obj, void (*dfree)(void *), voi struct heap_page *page = GET_HEAP_PAGE(obj); page->final_slots++; - page->size_pool->final_slots_count++; } static void @@ -2659,8 +2659,8 @@ struct each_obj_data { each_page_callback *each_page_callback; void *data; - struct heap_page **pages[SIZE_POOL_COUNT]; - size_t pages_counts[SIZE_POOL_COUNT]; }; static VALUE @@ -2674,7 +2674,7 @@ objspace_each_objects_ensure(VALUE arg) objspace->flags.dont_incremental = FALSE; } - for (int i = 0; i < SIZE_POOL_COUNT; i++) { struct heap_page **pages = data->pages[i]; free(pages); } @@ -2688,10 +2688,10 @@ objspace_each_objects_try(VALUE arg) struct each_obj_data *data = (struct each_obj_data *)arg; rb_objspace_t *objspace = data->objspace; - /* Copy pages from all size_pools to their respective buffers. */ - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - rb_size_pool_t *size_pool = &size_pools[i]; - size_t size = size_pool->total_pages * sizeof(struct heap_page *); struct heap_page **pages = malloc(size); if (!pages) rb_memerror(); @@ -2703,21 +2703,21 @@ objspace_each_objects_try(VALUE arg) * an infinite loop. */ struct heap_page *page = 0; size_t pages_count = 0; - ccan_list_for_each(&size_pool->pages, page, page_node) { pages[pages_count] = page; pages_count++; } data->pages[i] = pages; data->pages_counts[i] = pages_count; - GC_ASSERT(pages_count == size_pool->total_pages); } - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - rb_size_pool_t *size_pool = &size_pools[i]; size_t pages_count = data->pages_counts[i]; struct heap_page **pages = data->pages[i]; - struct heap_page *page = ccan_list_top(&size_pool->pages, struct heap_page, page_node); for (size_t i = 0; i < pages_count; i++) { /* If we have reached the end of the linked list then there are no * more pages, so break. */ @@ -2728,10 +2728,10 @@ objspace_each_objects_try(VALUE arg) if (pages[i] != page) continue; uintptr_t pstart = (uintptr_t)page->start; - uintptr_t pend = pstart + (page->total_slots * size_pool->slot_size); if (data->each_obj_callback && - (*data->each_obj_callback)((void *)pstart, (void *)pend, size_pool->slot_size, data->data)) { break; } if (data->each_page_callback && @@ -2739,7 +2739,7 @@ objspace_each_objects_try(VALUE arg) break; } - page = ccan_list_next(&size_pool->pages, page, page_node); } } @@ -2930,14 +2930,14 @@ finalize_list(rb_objspace_t *objspace, VALUE zombie) obj_free_object_id(objspace, zombie); } - GC_ASSERT(page->size_pool->final_slots_count > 0); GC_ASSERT(page->final_slots > 0); - page->size_pool->final_slots_count--; page->final_slots--; page->free_slots++; heap_page_add_freeobj(objspace, page, zombie); - page->size_pool->total_freed_objects++; } rb_gc_vm_unlock(lev); @@ -2995,21 +2995,21 @@ gc_abort(void *objspace_ptr) } if (is_lazy_sweeping(objspace)) { - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - rb_size_pool_t *size_pool = &size_pools[i]; - size_pool->sweeping_page = NULL; struct heap_page *page = NULL; - ccan_list_for_each(&size_pool->pages, page, page_node) { page->flags.before_sweep = false; } } } - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - rb_size_pool_t *size_pool = &size_pools[i]; - rgengc_mark_and_rememberset_clear(objspace, size_pool); } gc_mode_set(objspace, gc_mode_none); @@ -3148,9 +3148,9 @@ static size_t objspace_available_slots(rb_objspace_t *objspace) { size_t total_slots = 0; - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - rb_size_pool_t *size_pool = &size_pools[i]; - total_slots += size_pool->total_slots; } return total_slots; } @@ -3214,7 +3214,7 @@ unlock_page_body(rb_objspace_t *objspace, struct heap_page_body *body) } static bool -try_move(rb_objspace_t *objspace, rb_size_pool_t *size_pool, struct heap_page *free_page, VALUE src) { GC_ASSERT(gc_is_moveable_obj(objspace, src)); @@ -3260,13 +3260,13 @@ try_move(rb_objspace_t *objspace, rb_size_pool_t *size_pool, struct heap_page *f } static void -gc_unprotect_pages(rb_objspace_t *objspace, rb_size_pool_t *size_pool) { - struct heap_page *cursor = size_pool->compact_cursor; while (cursor) { unlock_page_body(objspace, cursor->body); - cursor = ccan_list_next(&size_pool->pages, cursor, page_node); } } @@ -3460,9 +3460,9 @@ install_handlers(void) static void gc_compact_finish(rb_objspace_t *objspace) { - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - rb_size_pool_t *size_pool = &size_pools[i]; - gc_unprotect_pages(objspace, size_pool); } uninstall_handlers(); @@ -3470,11 +3470,11 @@ gc_compact_finish(rb_objspace_t *objspace) gc_update_references(objspace); objspace->profile.compact_count++; - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - rb_size_pool_t *size_pool = &size_pools[i]; - size_pool->compact_cursor = NULL; - size_pool->free_pages = NULL; - size_pool->compact_cursor_index = 0; } if (gc_prof_enabled(objspace)) { @@ -3492,7 +3492,7 @@ struct gc_sweep_context { }; static inline void -gc_sweep_plane(rb_objspace_t *objspace, rb_size_pool_t *size_pool, uintptr_t p, bits_t bitset, struct gc_sweep_context *ctx) { struct heap_page *sweep_page = ctx->page; short slot_size = sweep_page->slot_size; @@ -3572,10 +3572,10 @@ gc_sweep_plane(rb_objspace_t *objspace, rb_size_pool_t *size_pool, uintptr_t p, } static inline void -gc_sweep_page(rb_objspace_t *objspace, rb_size_pool_t *size_pool, struct gc_sweep_context *ctx) { struct heap_page *sweep_page = ctx->page; - GC_ASSERT(sweep_page->size_pool == size_pool); uintptr_t p; bits_t *bits, bitset; @@ -3610,19 +3610,19 @@ gc_sweep_page(rb_objspace_t *objspace, rb_size_pool_t *size_pool, struct gc_swee bitset = ~bits[0]; bitset >>= NUM_IN_PAGE(p); if (bitset) { - gc_sweep_plane(objspace, size_pool, p, bitset, ctx); } p += (BITS_BITLENGTH - NUM_IN_PAGE(p)) * BASE_SLOT_SIZE; for (int i = 1; i < bitmap_plane_count; i++) { bitset = ~bits[i]; if (bitset) { - gc_sweep_plane(objspace, size_pool, p, bitset, ctx); } p += BITS_BITLENGTH * BASE_SLOT_SIZE; } - if (!size_pool->compact_cursor) { gc_setup_mark_bits(sweep_page); } @@ -3639,7 +3639,7 @@ gc_sweep_page(rb_objspace_t *objspace, rb_size_pool_t *size_pool, struct gc_swee ctx->freed_slots, ctx->empty_slots, ctx->final_slots); sweep_page->free_slots += ctx->freed_slots + ctx->empty_slots; - sweep_page->size_pool->total_freed_objects += ctx->freed_slots; if (heap_pages_deferred_final && !finalizing) { gc_finalize_deferred_register(objspace); @@ -3715,15 +3715,15 @@ heap_page_freelist_append(struct heap_page *page, struct free_slot *freelist) } static void -gc_sweep_start_heap(rb_objspace_t *objspace, rb_size_pool_t *size_pool) { - size_pool->sweeping_page = ccan_list_top(&size_pool->pages, struct heap_page, page_node); - size_pool->free_pages = NULL; - size_pool->pooled_pages = NULL; if (!objspace->flags.immediate_sweep) { struct heap_page *page = NULL; - ccan_list_for_each(&size_pool->pages, page, page_node) { page->flags.before_sweep = TRUE; } } @@ -3745,8 +3745,8 @@ gc_ractor_newobj_cache_clear(void *c, void *data) newobj_cache->incremental_mark_step_allocated_slots = 0; - for (size_t size_pool_idx = 0; size_pool_idx < SIZE_POOL_COUNT; size_pool_idx++) { - rb_ractor_newobj_size_pool_cache_t *cache = &newobj_cache->size_pool_caches[size_pool_idx]; struct heap_page *page = cache->using_page; struct free_slot *freelist = cache->freelist; @@ -3775,15 +3775,15 @@ gc_sweep_start(rb_objspace_t *objspace) } #endif - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - rb_size_pool_t *size_pool = &size_pools[i]; - gc_sweep_start_heap(objspace, size_pool); - /* We should call gc_sweep_finish_size_pool for size pools with no pages. */ - if (size_pool->sweeping_page == NULL) { - GC_ASSERT(size_pool->total_pages == 0); - GC_ASSERT(size_pool->total_slots == 0); - gc_sweep_finish_size_pool(objspace, size_pool); } } @@ -3791,17 +3791,17 @@ gc_sweep_start(rb_objspace_t *objspace) } static void -gc_sweep_finish_size_pool(rb_objspace_t *objspace, rb_size_pool_t *size_pool) { - size_t total_slots = size_pool->total_slots; - size_t swept_slots = size_pool->freed_slots + size_pool->empty_slots; - size_t init_slots = gc_params.size_pool_init_slots[size_pool - size_pools]; size_t min_free_slots = (size_t)(MAX(total_slots, init_slots) * gc_params.heap_free_slots_min_ratio); if (swept_slots < min_free_slots && /* The heap is a growth heap if it freed more slots than had empty slots. */ - (size_pool->empty_slots == 0 || size_pool->freed_slots > size_pool->empty_slots)) { /* If we don't have enough slots and we have pages on the tomb heap, move * pages from the tomb heap to the eden heap. This may prevent page * creation thrashing (frequently allocating and deallocting pages) and @@ -3809,8 +3809,8 @@ gc_sweep_finish_size_pool(rb_objspace_t *objspace, rb_size_pool_t *size_pool) struct heap_page *resurrected_page; while (swept_slots < min_free_slots && (resurrected_page = heap_page_resurrect(objspace))) { - size_pool_add_page(objspace, size_pool, resurrected_page); - heap_add_freepage(size_pool, resurrected_page); swept_slots += resurrected_page->free_slots; } @@ -3820,11 +3820,11 @@ gc_sweep_finish_size_pool(rb_objspace_t *objspace, rb_size_pool_t *size_pool) * RVALUE_OLD_AGE minor GC since the last major GC. */ if (is_full_marking(objspace) || objspace->profile.count - objspace->rgengc.last_major_gc < RVALUE_OLD_AGE) { - size_pool_allocatable_slots_expand(objspace, size_pool, swept_slots, size_pool->total_slots); } else { gc_needs_major_flags |= GPR_FLAG_MAJOR_BY_NOFREE; - size_pool->force_major_gc_count++; } } } @@ -3838,22 +3838,22 @@ gc_sweep_finish(rb_objspace_t *objspace) gc_prof_set_heap_info(objspace); heap_pages_free_unused_pages(objspace); - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - rb_size_pool_t *size_pool = &size_pools[i]; - size_pool->freed_slots = 0; - size_pool->empty_slots = 0; if (!will_be_incremental_marking(objspace)) { - struct heap_page *end_page = size_pool->free_pages; if (end_page) { while (end_page->free_next) end_page = end_page->free_next; - end_page->free_next = size_pool->pooled_pages; } else { - size_pool->free_pages = size_pool->pooled_pages; } - size_pool->pooled_pages = NULL; objspace->rincgc.pooled_slots = 0; } } @@ -3867,9 +3867,9 @@ gc_sweep_finish(rb_objspace_t *objspace) } static int -gc_sweep_step(rb_objspace_t *objspace, rb_size_pool_t *size_pool) { - struct heap_page *sweep_page = size_pool->sweeping_page; int unlink_limit = GC_SWEEP_PAGES_FREEABLE_PER_STEP; int swept_slots = 0; int pooled_slots = 0; @@ -3889,10 +3889,10 @@ gc_sweep_step(rb_objspace_t *objspace, rb_size_pool_t *size_pool) .freed_slots = 0, .empty_slots = 0, }; - gc_sweep_page(objspace, size_pool, &ctx); int free_slots = ctx.freed_slots + ctx.empty_slots; - size_pool->sweeping_page = ccan_list_next(&size_pool->pages, sweep_page, page_node); if (free_slots == sweep_page->total_slots && heap_pages_freeable_pages > 0 && @@ -3900,12 +3900,12 @@ gc_sweep_step(rb_objspace_t *objspace, rb_size_pool_t *size_pool) heap_pages_freeable_pages--; unlink_limit--; /* There are no living objects, so move this page to the global empty pages. */ - heap_unlink_page(objspace, size_pool, sweep_page); sweep_page->start = 0; sweep_page->total_slots = 0; sweep_page->slot_size = 0; - sweep_page->size_pool = NULL; sweep_page->free_slots = 0; asan_unlock_freelist(sweep_page); @@ -3919,15 +3919,15 @@ gc_sweep_step(rb_objspace_t *objspace, rb_size_pool_t *size_pool) objspace->empty_pages = sweep_page; } else if (free_slots > 0) { - size_pool->freed_slots += ctx.freed_slots; - size_pool->empty_slots += ctx.empty_slots; if (pooled_slots < GC_INCREMENTAL_SWEEP_POOL_SLOT_COUNT) { - heap_add_poolpage(objspace, size_pool, sweep_page); pooled_slots += free_slots; } else { - heap_add_freepage(size_pool, sweep_page); swept_slots += free_slots; if (swept_slots > GC_INCREMENTAL_SWEEP_SLOT_COUNT) { break; @@ -3937,10 +3937,10 @@ gc_sweep_step(rb_objspace_t *objspace, rb_size_pool_t *size_pool) else { sweep_page->free_next = NULL; } - } while ((sweep_page = size_pool->sweeping_page)); - if (!size_pool->sweeping_page) { - gc_sweep_finish_size_pool(objspace, size_pool); if (!has_sweeping_pages(objspace)) { gc_sweep_finish(objspace); @@ -3951,35 +3951,35 @@ gc_sweep_step(rb_objspace_t *objspace, rb_size_pool_t *size_pool) gc_prof_sweep_timer_stop(objspace); #endif - return size_pool->free_pages != NULL; } static void gc_sweep_rest(rb_objspace_t *objspace) { - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - rb_size_pool_t *size_pool = &size_pools[i]; - while (size_pool->sweeping_page) { - gc_sweep_step(objspace, size_pool); } } } static void -gc_sweep_continue(rb_objspace_t *objspace, rb_size_pool_t *sweep_size_pool) { GC_ASSERT(dont_gc_val() == FALSE || objspace->profile.latest_gc_info & GPR_FLAG_METHOD); if (!GC_ENABLE_LAZY_SWEEP) return; gc_sweeping_enter(objspace); - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - rb_size_pool_t *size_pool = &size_pools[i]; - if (!gc_sweep_step(objspace, size_pool)) { - /* sweep_size_pool requires a free slot but sweeping did not yield any * and we cannot allocate a new page. */ - if (size_pool == sweep_size_pool && objspace->heap_pages.allocatable_slots == 0) { /* Not allowed to create a new page so finish sweeping. */ gc_sweep_rest(objspace); break; @@ -4094,14 +4094,14 @@ gc_compact_start(rb_objspace_t *objspace) struct heap_page *page = NULL; gc_mode_transition(objspace, gc_mode_compacting); - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - rb_size_pool_t *size_pool = &size_pools[i]; - ccan_list_for_each(&size_pool->pages, page, page_node) { page->flags.before_sweep = TRUE; } - size_pool->compact_cursor = ccan_list_tail(&size_pool->pages, struct heap_page, page_node); - size_pool->compact_cursor_index = 0; } if (gc_prof_enabled(objspace)) { @@ -4146,9 +4146,9 @@ gc_sweep(rb_objspace_t *objspace) else { /* Sweep every size pool. */ - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - rb_size_pool_t *size_pool = &size_pools[i]; - gc_sweep_step(objspace, size_pool); } } @@ -5153,8 +5153,8 @@ static int gc_verify_heap_pages(rb_objspace_t *objspace) { int remembered_old_objects = 0; - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - remembered_old_objects += gc_verify_heap_pages_(objspace, &((&size_pools[i])->pages)); } return remembered_old_objects; } @@ -5261,21 +5261,21 @@ gc_verify_internal_consistency(void *objspace_ptr) } static void -heap_move_pooled_pages_to_free_pages(rb_size_pool_t *size_pool) { - if (size_pool->pooled_pages) { - if (size_pool->free_pages) { - struct heap_page *free_pages_tail = size_pool->free_pages; while (free_pages_tail->free_next) { free_pages_tail = free_pages_tail->free_next; } - free_pages_tail->free_next = size_pool->pooled_pages; } else { - size_pool->free_pages = size_pool->pooled_pages; } - size_pool->pooled_pages = NULL; } } @@ -5321,11 +5321,11 @@ gc_marks_wb_unprotected_objects_plane(rb_objspace_t *objspace, uintptr_t p, bits } static void -gc_marks_wb_unprotected_objects(rb_objspace_t *objspace, rb_size_pool_t *size_pool) { struct heap_page *page = 0; - ccan_list_for_each(&size_pool->pages, page, page_node) { bits_t *mark_bits = page->mark_bits; bits_t *wbun_bits = page->wb_unprotected_bits; uintptr_t p = page->start; @@ -5398,8 +5398,8 @@ gc_marks_finish(rb_objspace_t *objspace) objspace->flags.during_incremental_marking = FALSE; /* check children of all marked wb-unprotected objects */ - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - gc_marks_wb_unprotected_objects(objspace, &size_pools[i]); } } @@ -5432,8 +5432,8 @@ gc_marks_finish(rb_objspace_t *objspace) /* Setup freeable slots. */ size_t total_init_slots = 0; - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - total_init_slots += gc_params.size_pool_init_slots[i] * r_mul; } if (max_free_slots < total_init_slots) { @@ -5493,14 +5493,14 @@ gc_marks_finish(rb_objspace_t *objspace) } static bool -gc_compact_heap_cursors_met_p(rb_size_pool_t *size_pool) { - return size_pool->sweeping_page == size_pool->compact_cursor; } -static rb_size_pool_t * -gc_compact_destination_pool(rb_objspace_t *objspace, rb_size_pool_t *src_pool, VALUE obj) { size_t obj_size = rb_gc_obj_optimal_size(obj); if (obj_size == 0) { @@ -5509,34 +5509,34 @@ gc_compact_destination_pool(rb_objspace_t *objspace, rb_size_pool_t *src_pool, V size_t idx = 0; if (rb_gc_impl_size_allocatable_p(obj_size)) { - idx = size_pool_idx_for_size(obj_size); } - return &size_pools[idx]; } static bool -gc_compact_move(rb_objspace_t *objspace, rb_size_pool_t *size_pool, VALUE src) { GC_ASSERT(BUILTIN_TYPE(src) != T_MOVED); GC_ASSERT(gc_is_moveable_obj(objspace, src)); - rb_size_pool_t *dest_pool = gc_compact_destination_pool(objspace, size_pool, src); uint32_t orig_shape = 0; uint32_t new_shape = 0; if (gc_compact_heap_cursors_met_p(dest_pool)) { - return dest_pool != size_pool; } if (RB_TYPE_P(src, T_OBJECT)) { orig_shape = rb_gc_get_shape(src); - if (dest_pool != size_pool) { - new_shape = rb_gc_rebuild_shape(src, dest_pool - size_pools); if (new_shape == 0) { - dest_pool = size_pool; } } } @@ -5562,7 +5562,7 @@ gc_compact_move(rb_objspace_t *objspace, rb_size_pool_t *size_pool, VALUE src) dest_pool->sweeping_page = ccan_list_next(&dest_pool->pages, dest_pool->sweeping_page, page_node); if (gc_compact_heap_cursors_met_p(dest_pool)) { - return dest_pool != size_pool; } } @@ -5578,7 +5578,7 @@ gc_compact_move(rb_objspace_t *objspace, rb_size_pool_t *size_pool, VALUE src) } static bool -gc_compact_plane(rb_objspace_t *objspace, rb_size_pool_t *size_pool, uintptr_t p, bits_t bitset, struct heap_page *page) { short slot_size = page->slot_size; short slot_bits = slot_size / BASE_SLOT_SIZE; @@ -5592,7 +5592,7 @@ gc_compact_plane(rb_objspace_t *objspace, rb_size_pool_t *size_pool, uintptr_t p objspace->rcompactor.considered_count_table[BUILTIN_TYPE(vp)]++; if (gc_is_moveable_obj(objspace, vp)) { - if (!gc_compact_move(objspace, size_pool, vp)) { //the cursors met. bubble up return false; } @@ -5607,9 +5607,9 @@ gc_compact_plane(rb_objspace_t *objspace, rb_size_pool_t *size_pool, uintptr_t p // Iterate up all the objects in page, moving them to where they want to go static bool -gc_compact_page(rb_objspace_t *objspace, rb_size_pool_t *size_pool, struct heap_page *page) { - GC_ASSERT(page == size_pool->compact_cursor); bits_t *mark_bits, *pin_bits; bits_t bitset; @@ -5622,7 +5622,7 @@ gc_compact_page(rb_objspace_t *objspace, rb_size_pool_t *size_pool, struct heap_ bitset = (mark_bits[0] & ~pin_bits[0]); bitset >>= NUM_IN_PAGE(p); if (bitset) { - if (!gc_compact_plane(objspace, size_pool, (uintptr_t)p, bitset, page)) return false; } p += (BITS_BITLENGTH - NUM_IN_PAGE(p)) * BASE_SLOT_SIZE; @@ -5630,7 +5630,7 @@ gc_compact_page(rb_objspace_t *objspace, rb_size_pool_t *size_pool, struct heap_ for (int j = 1; j < HEAP_PAGE_BITMAP_LIMIT; j++) { bitset = (mark_bits[j] & ~pin_bits[j]); if (bitset) { - if (!gc_compact_plane(objspace, size_pool, (uintptr_t)p, bitset, page)) return false; } p += BITS_BITLENGTH * BASE_SLOT_SIZE; @@ -5642,11 +5642,11 @@ gc_compact_page(rb_objspace_t *objspace, rb_size_pool_t *size_pool, struct heap_ static bool gc_compact_all_compacted_p(rb_objspace_t *objspace) { - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - rb_size_pool_t *size_pool = &size_pools[i]; - if (size_pool->total_pages > 0 && - !gc_compact_heap_cursors_met_p(size_pool)) { return false; } } @@ -5663,16 +5663,16 @@ gc_sweep_compact(rb_objspace_t *objspace) #endif while (!gc_compact_all_compacted_p(objspace)) { - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - rb_size_pool_t *size_pool = &size_pools[i]; - if (gc_compact_heap_cursors_met_p(size_pool)) { continue; } - struct heap_page *start_page = size_pool->compact_cursor; - if (!gc_compact_page(objspace, size_pool, start_page)) { lock_page_body(objspace, start_page->body); continue; @@ -5681,7 +5681,7 @@ gc_sweep_compact(rb_objspace_t *objspace) // If we get here, we've finished moving all objects on the compact_cursor page // So we can lock it and move the cursor on to the next one. lock_page_body(objspace, start_page->body); - size_pool->compact_cursor = ccan_list_prev(&size_pool->pages, size_pool->compact_cursor, page_node); } } @@ -5697,8 +5697,8 @@ gc_marks_rest(rb_objspace_t *objspace) { gc_report(1, objspace, "gc_marks_rest\n"); - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - (&size_pools[i])->pooled_pages = NULL; } if (is_incremental_marking(objspace)) { @@ -5727,14 +5727,14 @@ gc_marks_step(rb_objspace_t *objspace, size_t slots) } static bool -gc_marks_continue(rb_objspace_t *objspace, rb_size_pool_t *size_pool) { GC_ASSERT(dont_gc_val() == FALSE || objspace->profile.latest_gc_info & GPR_FLAG_METHOD); bool marking_finished = true; gc_marking_enter(objspace); - if (size_pool->free_pages) { gc_report(2, objspace, "gc_marks_continue: has pooled pages"); marking_finished = gc_marks_step(objspace, objspace->rincgc.step_slots); @@ -5742,7 +5742,7 @@ gc_marks_continue(rb_objspace_t *objspace, rb_size_pool_t *size_pool) else { gc_report(2, objspace, "gc_marks_continue: no more pooled pages (stack depth: %"PRIdSIZE").\n", mark_stack_size(&objspace->mark_stack)); - size_pool->force_incremental_marking_finish_count++; gc_marks_rest(objspace); } @@ -5776,15 +5776,15 @@ gc_marks_start(rb_objspace_t *objspace, int full_mark) objspace->rgengc.last_major_gc = objspace->profile.count; objspace->marked_slots = 0; - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - rb_size_pool_t *size_pool = &size_pools[i]; - rgengc_mark_and_rememberset_clear(objspace, size_pool); - heap_move_pooled_pages_to_free_pages(size_pool); if (objspace->flags.during_compacting) { struct heap_page *page = NULL; - ccan_list_for_each(&size_pool->pages, page, page_node) { page->pinned_slots = 0; } } @@ -5796,8 +5796,8 @@ gc_marks_start(rb_objspace_t *objspace, int full_mark) objspace->rgengc.old_objects + objspace->rgengc.uncollectible_wb_unprotected_objects; /* uncollectible objects are marked already */ objspace->profile.minor_gc_count++; - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - rgengc_rememberset_mark(objspace, &size_pools[i]); } } @@ -5939,7 +5939,7 @@ rgengc_rememberset_mark_plane(rb_objspace_t *objspace, uintptr_t p, bits_t bitse } static void -rgengc_rememberset_mark(rb_objspace_t *objspace, rb_size_pool_t *size_pool) { size_t j; struct heap_page *page = 0; @@ -5948,7 +5948,7 @@ rgengc_rememberset_mark(rb_objspace_t *objspace, rb_size_pool_t *size_pool) #endif gc_report(1, objspace, "rgengc_rememberset_mark: start\n"); - ccan_list_for_each(&size_pool->pages, page, page_node) { if (page->flags.has_remembered_objects | page->flags.has_uncollectible_wb_unprotected_objects) { uintptr_t p = page->start; bits_t bitset, bits[HEAP_PAGE_BITMAP_LIMIT]; @@ -5991,11 +5991,11 @@ rgengc_rememberset_mark(rb_objspace_t *objspace, rb_size_pool_t *size_pool) } static void -rgengc_mark_and_rememberset_clear(rb_objspace_t *objspace, rb_size_pool_t *size_pool) { struct heap_page *page = 0; - ccan_list_for_each(&size_pool->pages, page, page_node) { memset(&page->mark_bits[0], 0, HEAP_PAGE_BITMAP_SIZE); memset(&page->uncollectible_bits[0], 0, HEAP_PAGE_BITMAP_SIZE); memset(&page->marking_bits[0], 0, HEAP_PAGE_BITMAP_SIZE); @@ -6232,12 +6232,12 @@ rb_gc_impl_ractor_cache_free(void *objspace_ptr, void *cache) } static void -heap_ready_to_gc(rb_objspace_t *objspace, rb_size_pool_t *size_pool) { - if (!size_pool->free_pages) { - if (!heap_page_allocate_and_initialize(objspace, size_pool)) { objspace->heap_pages.allocatable_slots = 1; - heap_page_allocate_and_initialize(objspace, size_pool); } } } @@ -6246,9 +6246,9 @@ static int ready_to_gc(rb_objspace_t *objspace) { if (dont_gc_val() || during_gc || ruby_disable_gc) { - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - rb_size_pool_t *size_pool = &size_pools[i]; - heap_ready_to_gc(objspace, size_pool); } return FALSE; } @@ -7007,8 +7007,8 @@ gc_move(rb_objspace_t *objspace, VALUE src, VALUE dest, size_t src_slot_size, si RMOVED(src)->destination = dest; GC_ASSERT(BUILTIN_TYPE(dest) != T_NONE); - GET_HEAP_PAGE(src)->size_pool->total_freed_objects++; - GET_HEAP_PAGE(dest)->size_pool->total_allocated_objects++; return src; } @@ -7041,16 +7041,16 @@ compare_free_slots(const void *left, const void *right, void *dummy) static void gc_sort_heap_by_compare_func(rb_objspace_t *objspace, gc_compact_compare_func compare_func) { - for (int j = 0; j < SIZE_POOL_COUNT; j++) { - rb_size_pool_t *size_pool = &size_pools[j]; - size_t total_pages = size_pool->total_pages; size_t size = rb_size_mul_or_raise(total_pages, sizeof(struct heap_page *), rb_eRuntimeError); struct heap_page *page = 0, **page_list = malloc(size); size_t i = 0; - size_pool->free_pages = NULL; - ccan_list_for_each(&size_pool->pages, page, page_node) { page_list[i++] = page; GC_ASSERT(page); } @@ -7062,12 +7062,12 @@ gc_sort_heap_by_compare_func(rb_objspace_t *objspace, gc_compact_compare_func co ruby_qsort(page_list, total_pages, sizeof(struct heap_page *), compare_func, NULL); /* Reset the eden heap */ - ccan_list_head_init(&size_pool->pages); for (i = 0; i < total_pages; i++) { - ccan_list_add(&size_pool->pages, &page_list[i]->page_node); if (page_list[i]->free_slots != 0) { - heap_add_freepage(size_pool, page_list[i]); } } @@ -7128,16 +7128,16 @@ gc_update_references(rb_objspace_t *objspace) struct heap_page *page = NULL; - for (int i = 0; i < SIZE_POOL_COUNT; i++) { bool should_set_mark_bits = TRUE; - rb_size_pool_t *size_pool = &size_pools[i]; - ccan_list_for_each(&size_pool->pages, page, page_node) { uintptr_t start = (uintptr_t)page->start; - uintptr_t end = start + (page->total_slots * size_pool->slot_size); - gc_ref_update((void *)start, (void *)end, size_pool->slot_size, objspace, page); - if (page == size_pool->sweeping_page) { should_set_mark_bits = FALSE; } if (should_set_mark_bits) { @@ -7615,7 +7615,7 @@ setup_gc_stat_heap_symbols(void) } static size_t -stat_one_heap(rb_size_pool_t *size_pool, VALUE hash, VALUE key) { #define SET(name, attr) \ if (key == gc_stat_heap_symbols[gc_stat_heap_sym_##name]) \ @@ -7623,14 +7623,14 @@ stat_one_heap(rb_size_pool_t *size_pool, VALUE hash, VALUE key) else if (hash != Qnil) \ rb_hash_aset(hash, gc_stat_heap_symbols[gc_stat_heap_sym_##name], SIZET2NUM(attr)); - SET(slot_size, size_pool->slot_size); - SET(heap_eden_pages, size_pool->total_pages); - SET(heap_eden_slots, size_pool->total_slots); - SET(total_allocated_pages, size_pool->total_allocated_pages); - SET(force_major_gc_count, size_pool->force_major_gc_count); - SET(force_incremental_marking_finish_count, size_pool->force_incremental_marking_finish_count); - SET(total_allocated_objects, size_pool->total_allocated_objects); - SET(total_freed_objects, size_pool->total_freed_objects); #undef SET if (!NIL_P(key)) { /* matched key should return above */ @@ -7652,28 +7652,28 @@ rb_gc_impl_stat_heap(void *objspace_ptr, VALUE heap_name, VALUE hash_or_sym) rb_raise(rb_eTypeError, "non-hash given"); } - for (int i = 0; i < SIZE_POOL_COUNT; i++) { VALUE hash = rb_hash_aref(hash_or_sym, INT2FIX(i)); if (NIL_P(hash)) { hash = rb_hash_new(); rb_hash_aset(hash_or_sym, INT2FIX(i), hash); } - stat_one_heap(&size_pools[i], hash, Qnil); } } else if (FIXNUM_P(heap_name)) { - int size_pool_idx = FIX2INT(heap_name); - if (size_pool_idx < 0 || size_pool_idx >= SIZE_POOL_COUNT) { rb_raise(rb_eArgError, "size pool index out of range"); } if (SYMBOL_P(hash_or_sym)) { - return stat_one_heap(&size_pools[size_pool_idx], Qnil, hash_or_sym); } else if (RB_TYPE_P(hash_or_sym, T_HASH)) { - return stat_one_heap(&size_pools[size_pool_idx], hash_or_sym, Qnil); } else { rb_raise(rb_eTypeError, "non-hash or symbol given"); @@ -7896,11 +7896,11 @@ rb_gc_impl_set_params(void *objspace_ptr) /* ok */ } - for (int i = 0; i < SIZE_POOL_COUNT; i++) { char env_key[sizeof("RUBY_GC_HEAP_" "_INIT_SLOTS") + DECIMAL_SIZE_OF_BITS(sizeof(int) * CHAR_BIT)]; snprintf(env_key, sizeof(env_key), "RUBY_GC_HEAP_%d_INIT_SLOTS", i); - get_envparam_size(env_key, &gc_params.size_pool_init_slots[i], 0); } get_envparam_double("RUBY_GC_HEAP_GROWTH_FACTOR", &gc_params.growth_factor, 1.0, 0.0, FALSE); @@ -9085,7 +9085,7 @@ gc_compact(VALUE self) #if GC_CAN_COMPILE_COMPACTION struct desired_compaction_pages_i_data { rb_objspace_t *objspace; - size_t required_slots[SIZE_POOL_COUNT]; }; static int @@ -9094,15 +9094,15 @@ desired_compaction_pages_i(struct heap_page *page, void *data) struct desired_compaction_pages_i_data *tdata = data; rb_objspace_t *objspace = tdata->objspace; VALUE vstart = (VALUE)page->start; - VALUE vend = vstart + (VALUE)(page->total_slots * page->size_pool->slot_size); - for (VALUE v = vstart; v != vend; v += page->size_pool->slot_size) { asan_unpoisoning_object(v) { /* skip T_NONEs; they won't be moved */ if (BUILTIN_TYPE(v) != T_NONE) { - rb_size_pool_t *dest_pool = gc_compact_destination_pool(objspace, page->size_pool, v); - size_t dest_pool_idx = dest_pool - size_pools; tdata->required_slots[dest_pool_idx]++; } } @@ -9164,14 +9164,14 @@ gc_verify_compaction_references(int argc, VALUE* argv, VALUE self) /* Find out which pool has the most pages */ size_t max_existing_pages = 0; - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - rb_size_pool_t *size_pool = &size_pools[i]; - max_existing_pages = MAX(max_existing_pages, size_pool->total_pages); } /* Add pages to each size pool so that compaction is guaranteed to move every object */ - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - rb_size_pool_t *size_pool = &size_pools[i]; size_t pages_to_add = 0; /* @@ -9180,14 +9180,14 @@ gc_verify_compaction_references(int argc, VALUE* argv, VALUE self) * through all of the pools in `gc_sweep_compact` without hitting the "sweep & * compact cursors met" condition on some pools before fully compacting others */ - pages_to_add += max_existing_pages - size_pool->total_pages; /* * Step 2: Now add additional free pages to each size pool sufficient to hold all objects * that want to be in that size pool, whether moved into it or moved within it */ objspace->heap_pages.allocatable_slots = desired_compaction.required_slots[i]; while (objspace->heap_pages.allocatable_slots > 0) { - heap_page_allocate_and_initialize(objspace, size_pool); } /* * Step 3: Add two more pages so that the compact & sweep cursors will meet _after_ all objects @@ -9196,7 +9196,7 @@ gc_verify_compaction_references(int argc, VALUE* argv, VALUE self) pages_to_add += 2; for (; pages_to_add > 0; pages_to_add--) { - heap_page_allocate_and_initialize_force(objspace, size_pool); } } } @@ -9238,10 +9238,10 @@ rb_gc_impl_objspace_free(void *objspace_ptr) heap_pages_lomem = 0; heap_pages_himem = 0; - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - rb_size_pool_t *size_pool = &size_pools[i]; - size_pool->total_pages = 0; - size_pool->total_slots = 0; } st_free_table(objspace->id_to_obj_tbl); @@ -9309,12 +9309,12 @@ rb_gc_impl_objspace_init(void *objspace_ptr) rb_bug("Could not preregister postponed job for GC"); } - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - rb_size_pool_t *size_pool = &size_pools[i]; - size_pool->slot_size = (1 << i) * BASE_SLOT_SIZE; - ccan_list_head_init(&size_pool->pages); } rb_darray_make(&objspace->heap_pages.sorted, 0); @@ -9336,9 +9336,9 @@ rb_gc_impl_objspace_init(void *objspace_ptr) objspace->rgengc.oldmalloc_increase_limit = gc_params.oldmalloc_limit_min; #endif /* Set size pools allocatable pages. */ - for (int i = 0; i < SIZE_POOL_COUNT; i++) { - /* Set the default value of size_pool_init_slots. */ - gc_params.size_pool_init_slots[i] = GC_HEAP_INIT_SLOTS; } init_mark_stack(&objspace->mark_stack); @@ -9357,8 +9357,8 @@ rb_gc_impl_init(void) rb_hash_aset(gc_constants, ID2SYM(rb_intern("HEAP_PAGE_OBJ_LIMIT")), SIZET2NUM(HEAP_PAGE_OBJ_LIMIT)); rb_hash_aset(gc_constants, ID2SYM(rb_intern("HEAP_PAGE_BITMAP_SIZE")), SIZET2NUM(HEAP_PAGE_BITMAP_SIZE)); rb_hash_aset(gc_constants, ID2SYM(rb_intern("HEAP_PAGE_SIZE")), SIZET2NUM(HEAP_PAGE_SIZE)); - rb_hash_aset(gc_constants, ID2SYM(rb_intern("SIZE_POOL_COUNT")), LONG2FIX(SIZE_POOL_COUNT)); - rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVARGC_MAX_ALLOCATE_SIZE")), LONG2FIX(size_pool_slot_size(SIZE_POOL_COUNT - 1))); rb_hash_aset(gc_constants, ID2SYM(rb_intern("RVALUE_OLD_AGE")), LONG2FIX(RVALUE_OLD_AGE)); if (RB_BUG_INSTEAD_OF_RB_MEMERROR+0) { rb_hash_aset(gc_constants, ID2SYM(rb_intern("RB_BUG_INSTEAD_OF_RB_MEMERROR")), Qtrue); @@ -40,7 +40,7 @@ const char *rb_obj_info(VALUE obj); bool rb_gc_shutdown_call_finalizer_p(VALUE obj); uint32_t rb_gc_get_shape(VALUE obj); void rb_gc_set_shape(VALUE obj, uint32_t shape_id); -uint32_t rb_gc_rebuild_shape(VALUE obj, size_t size_pool_id); size_t rb_obj_memsize_of(VALUE obj); RUBY_SYMBOL_EXPORT_END @@ -32,7 +32,7 @@ GC_IMPL_FN void *rb_gc_impl_ractor_cache_alloc(void *objspace_ptr); GC_IMPL_FN void rb_gc_impl_ractor_cache_free(void *objspace_ptr, void *cache); GC_IMPL_FN void rb_gc_impl_set_params(void *objspace_ptr); GC_IMPL_FN void rb_gc_impl_init(void); -GC_IMPL_FN size_t *rb_gc_impl_size_pool_sizes(void *objspace_ptr); // Shutdown GC_IMPL_FN void rb_gc_impl_shutdown_free_objects(void *objspace_ptr); // GC @@ -49,7 +49,7 @@ GC_IMPL_FN VALUE rb_gc_impl_config_set(void *objspace_ptr, VALUE hash); // Object allocation GC_IMPL_FN VALUE rb_gc_impl_new_obj(void *objspace_ptr, void *cache_ptr, VALUE klass, VALUE flags, VALUE v1, VALUE v2, VALUE v3, bool wb_protected, size_t alloc_size); GC_IMPL_FN size_t rb_gc_impl_obj_slot_size(VALUE obj); -GC_IMPL_FN size_t rb_gc_impl_size_pool_id_for_size(void *objspace_ptr, size_t size); GC_IMPL_FN bool rb_gc_impl_size_allocatable_p(size_t size); // Malloc GC_IMPL_FN void *rb_gc_impl_malloc(void *objspace_ptr, size_t size); @@ -83,7 +83,7 @@ struct RClass { struct rb_id_table *m_tbl; }; -// Assert that classes can be embedded in size_pools[2] (which has 160B slot size) STATIC_ASSERT(sizeof_rb_classext_t, sizeof(struct RClass) + sizeof(rb_classext_t) <= 4 * RVALUE_SIZE); struct RClass_and_rb_classext_t { @@ -201,8 +201,8 @@ void *rb_gc_ractor_cache_alloc(void); void rb_gc_ractor_cache_free(void *cache); bool rb_gc_size_allocatable_p(size_t size); -size_t *rb_gc_size_pool_sizes(void); -size_t rb_gc_size_pool_id_for_size(size_t size); void rb_gc_mark_and_move(VALUE *ptr); @@ -547,7 +547,7 @@ class HeapPageIter: self.target = target self.start = page.GetChildMemberWithName('start').GetValueAsUnsigned(); self.num_slots = page.GetChildMemberWithName('total_slots').unsigned - self.slot_size = page.GetChildMemberWithName('size_pool').GetChildMemberWithName('slot_size').unsigned self.counter = 0 self.tRBasic = target.FindFirstType("struct RBasic") self.tRValue = target.FindFirstType("struct RVALUE") @@ -135,7 +135,7 @@ rb_class_allocate_instance(VALUE klass) RUBY_ASSERT(rb_shape_get_shape(obj)->type == SHAPE_ROOT); // Set the shape to the specific T_OBJECT shape. - ROBJECT_SET_SHAPE_ID(obj, (shape_id_t)(rb_gc_size_pool_id_for_size(size) + FIRST_T_OBJECT_SHAPE_ID)); #if RUBY_DEBUG RUBY_ASSERT(!rb_shape_obj_too_complex(obj)); @@ -358,7 +358,7 @@ rb_obj_copy_ivar(VALUE dest, VALUE obj) rb_shape_t * initial_shape = rb_shape_get_shape(dest); - if (initial_shape->size_pool_index != src_shape->size_pool_index) { RUBY_ASSERT(initial_shape->type == SHAPE_T_OBJECT); shape_to_set_on_dest = rb_shape_rebuild_shape(initial_shape, src_shape); @@ -1446,7 +1446,7 @@ module RubyVM::RJIT # :nodoc: all next_iv_index: [self.attr_index_t, Primitive.cexpr!("OFFSETOF((*((struct rb_shape *)NULL)), next_iv_index)")], capacity: [CType::Immediate.parse("uint32_t"), Primitive.cexpr!("OFFSETOF((*((struct rb_shape *)NULL)), capacity)")], type: [CType::Immediate.parse("uint8_t"), Primitive.cexpr!("OFFSETOF((*((struct rb_shape *)NULL)), type)")], - size_pool_index: [CType::Immediate.parse("uint8_t"), Primitive.cexpr!("OFFSETOF((*((struct rb_shape *)NULL)), size_pool_index)")], parent_id: [self.shape_id_t, Primitive.cexpr!("OFFSETOF((*((struct rb_shape *)NULL)), parent_id)")], ancestor_index: [CType::Pointer.new { self.redblack_node_t }, Primitive.cexpr!("OFFSETOF((*((struct rb_shape *)NULL)), ancestor_index)")], ) @@ -418,7 +418,7 @@ rb_shape_alloc(ID edge_name, rb_shape_t * parent, enum shape_type type) { rb_shape_t * shape = rb_shape_alloc_with_parent_id(edge_name, rb_shape_id(parent)); shape->type = (uint8_t)type; - shape->size_pool_index = parent->size_pool_index; shape->capacity = parent->capacity; shape->edges = 0; return shape; @@ -1059,7 +1059,7 @@ rb_shape_t_to_rb_cShape(rb_shape_t *shape) INT2NUM(shape->parent_id), rb_shape_edge_name(shape), INT2NUM(shape->next_iv_index), - INT2NUM(shape->size_pool_index), INT2NUM(shape->type), INT2NUM(shape->capacity)); rb_obj_freeze(obj); @@ -1266,7 +1266,7 @@ Init_default_shapes(void) rb_shape_t *root = rb_shape_alloc_with_parent_id(0, INVALID_SHAPE_ID); root->capacity = 0; root->type = SHAPE_ROOT; - root->size_pool_index = 0; GET_SHAPE_TREE()->root_shape = root; RUBY_ASSERT(rb_shape_id(GET_SHAPE_TREE()->root_shape) == ROOT_SHAPE_ID); @@ -1282,16 +1282,16 @@ Init_default_shapes(void) rb_shape_t *too_complex_shape = rb_shape_alloc_with_parent_id(0, ROOT_SHAPE_ID); too_complex_shape->type = SHAPE_OBJ_TOO_COMPLEX; - too_complex_shape->size_pool_index = 0; RUBY_ASSERT(OBJ_TOO_COMPLEX_SHAPE_ID == (GET_SHAPE_TREE()->next_shape_id - 1)); RUBY_ASSERT(rb_shape_id(too_complex_shape) == OBJ_TOO_COMPLEX_SHAPE_ID); // Make shapes for T_OBJECT - size_t *sizes = rb_gc_size_pool_sizes(); for (int i = 0; sizes[i] > 0; i++) { rb_shape_t *t_object_shape = rb_shape_alloc_with_parent_id(0, INVALID_SHAPE_ID); t_object_shape->type = SHAPE_T_OBJECT; - t_object_shape->size_pool_index = i; t_object_shape->capacity = (uint32_t)((sizes[i] - offsetof(struct RObject, as.ary)) / sizeof(VALUE)); t_object_shape->edges = rb_id_table_create(0); t_object_shape->ancestor_index = LEAF; @@ -1308,7 +1308,7 @@ Init_shape(void) "parent_id", "edge_name", "next_iv_index", - "size_pool_index", "type", "capacity", NULL); @@ -47,7 +47,7 @@ struct rb_shape { attr_index_t next_iv_index; uint32_t capacity; // Total capacity of the object with this shape uint8_t type; - uint8_t size_pool_index; shape_id_t parent_id; redblack_node_t * ancestor_index; }; @@ -226,7 +226,7 @@ class TestGc < Test::Unit::TestCase GC.stat_heap(0, stat_heap) GC.stat(stat) - GC::INTERNAL_CONSTANTS[:SIZE_POOL_COUNT].times do |i| EnvUtil.without_gc do GC.stat_heap(i, stat_heap) GC.stat(stat) @@ -248,7 +248,7 @@ class TestGc < Test::Unit::TestCase assert_equal stat_heap[:slot_size], GC.stat_heap(0)[:slot_size] assert_raise(ArgumentError) { GC.stat_heap(-1) } - assert_raise(ArgumentError) { GC.stat_heap(GC::INTERNAL_CONSTANTS[:SIZE_POOL_COUNT]) } end def test_stat_heap_all @@ -259,7 +259,7 @@ class TestGc < Test::Unit::TestCase GC.stat_heap(0, stat_heap) GC.stat_heap(nil, stat_heap_all) - GC::INTERNAL_CONSTANTS[:SIZE_POOL_COUNT].times do |i| GC.stat_heap(nil, stat_heap_all) GC.stat_heap(i, stat_heap) @@ -538,7 +538,7 @@ class TestGc < Test::Unit::TestCase gc_count = GC.stat(:count) # Fill up all of the size pools to the init slots - GC::INTERNAL_CONSTANTS[:SIZE_POOL_COUNT].times do |i| capa = (GC.stat_heap(i, :slot_size) - GC::INTERNAL_CONSTANTS[:RVALUE_OVERHEAD] - (2 * RbConfig::SIZEOF["void*"])) / RbConfig::SIZEOF["void*"] while GC.stat_heap(i, :heap_eden_slots) < GC_HEAP_INIT_SLOTS Array.new(capa) @@ -558,7 +558,7 @@ class TestGc < Test::Unit::TestCase gc_count = GC.stat(:count) # Fill up all of the size pools to the init slots - GC::INTERNAL_CONSTANTS[:SIZE_POOL_COUNT].times do |i| capa = (GC.stat_heap(i, :slot_size) - GC::INTERNAL_CONSTANTS[:RVALUE_OVERHEAD] - (2 * RbConfig::SIZEOF["void*"])) / RbConfig::SIZEOF["void*"] while GC.stat_heap(i, :heap_eden_slots) < SIZES[i] Array.new(capa) @@ -283,7 +283,7 @@ class TestGCCompact < Test::Unit::TestCase end; end - def test_moving_arrays_down_size_pools omit if GC::INTERNAL_CONSTANTS[:SIZE_POOL_COUNT] == 1 assert_separately(%w[-robjspace], "#{<<~"begin;"}\n#{<<~"end;"}", timeout: 10) @@ -305,7 +305,7 @@ class TestGCCompact < Test::Unit::TestCase end; end - def test_moving_arrays_up_size_pools omit if GC::INTERNAL_CONSTANTS[:SIZE_POOL_COUNT] == 1 assert_separately(%w[-robjspace], "#{<<~"begin;"}\n#{<<~"end;"}", timeout: 10) @@ -329,7 +329,7 @@ class TestGCCompact < Test::Unit::TestCase end; end - def test_moving_objects_between_size_pools omit if GC::INTERNAL_CONSTANTS[:SIZE_POOL_COUNT] == 1 assert_separately(%w[-robjspace], "#{<<~"begin;"}\n#{<<~"end;"}", timeout: 60) @@ -361,7 +361,7 @@ class TestGCCompact < Test::Unit::TestCase end; end - def test_moving_strings_up_size_pools omit if GC::INTERNAL_CONSTANTS[:SIZE_POOL_COUNT] == 1 assert_separately(%w[-robjspace], "#{<<~"begin;"}\n#{<<~"end;"}", timeout: 30) @@ -382,7 +382,7 @@ class TestGCCompact < Test::Unit::TestCase end; end - def test_moving_strings_down_size_pools omit if GC::INTERNAL_CONSTANTS[:SIZE_POOL_COUNT] == 1 assert_separately(%w[-robjspace], "#{<<~"begin;"}\n#{<<~"end;"}", timeout: 30) @@ -402,7 +402,7 @@ class TestGCCompact < Test::Unit::TestCase end; end - def test_moving_hashes_down_size_pools omit if GC::INTERNAL_CONSTANTS[:SIZE_POOL_COUNT] == 1 # AR and ST hashes are in the same size pool on 32 bit omit unless RbConfig::SIZEOF["uint64_t"] <= RbConfig::SIZEOF["void*"] @@ -425,7 +425,7 @@ class TestGCCompact < Test::Unit::TestCase end; end - def test_moving_objects_between_size_pools_keeps_shape_frozen_status # [Bug #19536] assert_separately([], "#{<<~"begin;"}\n#{<<~"end;"}") begin; @@ -662,8 +662,8 @@ CODE assert_equal(Encoding::UTF_8, "#{s}x".encoding) end - def test_string_interpolations_across_size_pools_get_embedded - omit if GC::INTERNAL_CONSTANTS[:SIZE_POOL_COUNT] == 1 require 'objspace' base_slot_size = GC::INTERNAL_CONSTANTS[:BASE_SLOT_SIZE] @@ -648,7 +648,7 @@ pub struct rb_shape { pub next_iv_index: attr_index_t, pub capacity: u32, pub type_: u8, - pub size_pool_index: u8, pub parent_id: shape_id_t, pub ancestor_index: *mut redblack_node_t, } |