Implement Objects on VWA

This commit implements Objects on Variable Width Allocation. This allows Objects with more ivars to be embedded (i.e. contents directly follow the object header) which improves performance through better cache locality.
author: Peter Zhu <[email protected]> 2022-07-11 10:09:39 -0400
committer: Peter Zhu <[email protected]> 2022-07-15 09:21:07 -0400
commit: 7424ea184f9d67c1c7f3ee97494ed3bd1aa60833 ()
tree: 822838e39d81cd2785c970cb45a86854823af6fe
parent: 7fda741f6e67b809b08423f0d4e903c078da2eed (diff)
14 files changed, 216 insertions, 74 deletions
@@ -53,7 +53,9 @@ const union {
 rb_econv_result_t econv_result;
 enum ruby_preserved_encindex encoding_index;
 enum ruby_robject_flags robject_flags;
 enum ruby_robject_consts robject_consts;
 enum ruby_rmodule_flags rmodule_flags;
 enum ruby_rstring_flags rstring_flags;
 #if !USE_RVARGC
@@ -2865,18 +2865,65 @@ rb_newobj(void)
 return newobj_of(0, T_NONE, 0, 0, 0, FALSE, sizeof(RVALUE));
 }
-VALUE
-rb_newobj_of(VALUE klass, VALUE flags)
 {
- if ((flags & RUBY_T_MASK) == T_OBJECT) {
- st_table *index_tbl = RCLASS_IV_INDEX_TBL(klass);
- VALUE obj = newobj_of(klass, (flags | ROBJECT_EMBED) & ~FL_WB_PROTECTED , Qundef, Qundef, Qundef, flags & FL_WB_PROTECTED, sizeof(RVALUE));
- if (index_tbl && index_tbl->num_entries > ROBJECT_EMBED_LEN_MAX) {
- rb_init_iv_list(obj);
 }
- return obj;
 }
 else {
 return newobj_of(klass, flags & ~FL_WB_PROTECTED, 0, 0, 0, flags & FL_WB_PROTECTED, sizeof(RVALUE));
@@ -2989,17 +3036,7 @@ rb_imemo_new_debug(enum imemo_type type, VALUE v1, VALUE v2, VALUE v3, VALUE v0,
 VALUE
 rb_class_allocate_instance(VALUE klass)
 {
- st_table *index_tbl = RCLASS_IV_INDEX_TBL(klass);
-
- VALUE flags = T_OBJECT | ROBJECT_EMBED;
-
- VALUE obj = newobj_of(klass, flags, Qundef, Qundef, Qundef, RGENGC_WB_PROTECTED_OBJECT, sizeof(RVALUE));
-
- if (index_tbl && index_tbl->num_entries > ROBJECT_EMBED_LEN_MAX) {
- rb_init_iv_list(obj);
- }
-
- return obj;
 }
 static inline void
@@ -8322,17 +8359,23 @@ gc_compact_destination_pool(rb_objspace_t *objspace, rb_size_pool_t *src_pool, V
 size_t obj_size;
 switch (BUILTIN_TYPE(src)) {
- case T_STRING:
- obj_size = rb_str_size_as_embedded(src);
- break;
 case T_ARRAY:
 obj_size = rb_ary_size_as_embedded(src);
 break;
 default:
 return src_pool;
 }
- if (rb_gc_size_allocatable_p(obj_size)) {
 return &size_pools[size_pool_idx_for_size(obj_size)];
 }
 else {
@@ -9896,12 +9939,37 @@ gc_ref_update_array(rb_objspace_t * objspace, VALUE v)
 }
 static void
-gc_ref_update_object(rb_objspace_t * objspace, VALUE v)
 {
 VALUE *ptr = ROBJECT_IVPTR(v);
- uint32_t i, len = ROBJECT_NUMIV(v);
- for (i = 0; i < len; i++) {
 UPDATE_IF_MOVED(objspace, ptr[i]);
 }
 }
@@ -22,7 +22,7 @@
 * In released versions of Ruby, this number should not be changed since teeny
 * versions of Ruby should guarantee ABI compatibility.
 */
-#define RUBY_ABI_VERSION 1
 /* Windows does not support weak symbols so ruby_abi_version will not exist
 * in the shared library. */
@@ -75,6 +75,7 @@ enum ruby_robject_flags {
 ROBJECT_EMBED = RUBY_FL_USER1
 };
 /**
 * This is an enum because GDB wants it (rather than a macro). People need not
 * bother.
@@ -83,6 +84,7 @@ enum ruby_robject_consts {
 /** Max possible number of instance variables that can be embedded. */
 ROBJECT_EMBED_LEN_MAX = RBIMPL_EMBED_LEN_MAX_OF(VALUE)
 };
 struct st_table;
@@ -95,6 +97,14 @@ struct RObject {
 /** Basic part, including flags and class. */
 struct RBasic basic;
 /** Object's specific fields. */
 union {
@@ -103,12 +113,13 @@ struct RObject {
 * this pattern.
 */
 struct {
-
 /**
 * Number of instance variables. This is per object; objects might
 * differ in this field even if they have the identical classes.
 */
 uint32_t numiv;
 /** Pointer to a C array that holds instance variables. */
 VALUE *ivptr;
@@ -124,14 +135,38 @@ struct RObject {
 struct st_table *iv_index_tbl;
 } heap;
- /**
- * Embedded instance variables. When an object is small enough, it
 * uses this area to store the instance variables.
 */
 VALUE ary[ROBJECT_EMBED_LEN_MAX];
 } as;
 };
 RBIMPL_ATTR_PURE_UNLESS_DEBUG()
 RBIMPL_ATTR_ARTIFICIAL()
 /**
@@ -146,12 +181,16 @@ ROBJECT_NUMIV(VALUE obj)
 {
 RBIMPL_ASSERT_TYPE(obj, RUBY_T_OBJECT);
 if (RB_FL_ANY_RAW(obj, ROBJECT_EMBED)) {
 return ROBJECT_EMBED_LEN_MAX;
 }
 else {
 return ROBJECT(obj)->as.heap.numiv;
 }
 }
 RBIMPL_ATTR_PURE_UNLESS_DEBUG()
@@ -353,12 +353,16 @@ mjit_compile_body(FILE *f, const rb_iseq_t *iseq, struct compile_status *status)
 // Generate merged ivar guards first if needed
 if (!status->compile_info->disable_ivar_cache && status->merge_ivar_guards_p) {
 fprintf(f, " if (UNLIKELY(!(RB_TYPE_P(GET_SELF(), T_OBJECT) && (rb_serial_t)%"PRI_SERIALT_PREFIX"u == RCLASS_SERIAL(RBASIC(GET_SELF())->klass) &&", status->ivar_serial);
 if (status->max_ivar_index >= ROBJECT_EMBED_LEN_MAX) {
 fprintf(f, "%"PRIuSIZE" < ROBJECT_NUMIV(GET_SELF())", status->max_ivar_index); // index < ROBJECT_NUMIV(obj) && !RB_FL_ANY_RAW(obj, ROBJECT_EMBED)
 }
 else {
 fprintf(f, "ROBJECT_EMBED_LEN_MAX == ROBJECT_NUMIV(GET_SELF())"); // index < ROBJECT_NUMIV(obj) && RB_FL_ANY_RAW(obj, ROBJECT_EMBED)
 }
 fprintf(f, "))) {\n");
 fprintf(f, " goto ivar_cancel;\n");
 fprintf(f, " }\n");
@@ -267,34 +267,13 @@ rb_obj_singleton_class(VALUE obj)
 MJIT_FUNC_EXPORTED void
 rb_obj_copy_ivar(VALUE dest, VALUE obj)
 {
- VALUE *dst_buf = 0;
- VALUE *src_buf = 0;
- uint32_t len = ROBJECT_EMBED_LEN_MAX;
- if (RBASIC(obj)->flags & ROBJECT_EMBED) {
- src_buf = ROBJECT(obj)->as.ary;
-
- // embedded -> embedded
- if (RBASIC(dest)->flags & ROBJECT_EMBED) {
- dst_buf = ROBJECT(dest)->as.ary;
- }
- // embedded -> extended
- else {
- dst_buf = ROBJECT(dest)->as.heap.ivptr;
- }
- }
- // extended -> extended
- else {
- RUBY_ASSERT(!(RBASIC(dest)->flags & ROBJECT_EMBED));
- uint32_t src_len = ROBJECT(obj)->as.heap.numiv;
- uint32_t dst_len = ROBJECT(dest)->as.heap.numiv;
-
- len = src_len < dst_len ? src_len : dst_len;
- dst_buf = ROBJECT(dest)->as.heap.ivptr;
- src_buf = ROBJECT(obj)->as.heap.ivptr;
- }
-
- MEMCPY(dst_buf, src_buf, VALUE, len);
 }
 static void
@@ -249,6 +249,30 @@ class TestGCCompact < Test::Unit::TestCase
 end;
 end
 def test_moving_strings_up_size_pools
 omit if !GC.using_rvargc?
 assert_separately([], "#{<<~"begin;"}\n#{<<~"end;"}", timeout: 10, signal: :SEGV)
@@ -969,23 +969,24 @@ class TestMJIT < Test::Unit::TestCase
 end
 def test_heap_promotion_of_ivar_in_the_middle_of_jit
 assert_eval_with_jit("#{<<~"begin;"}\n#{<<~"end;"}", stdout: "true\ntrue\n", success_count: 2, min_calls: 2)
 begin;
 class A
 def initialize
 @iv0 = nil
 @iv1 = []
- @iv2 = nil
 end
 def test(add)
 @iv0.nil?
- @iv2.nil?
 add_ivar if add
 @iv1.empty?
 end
 def add_ivar
 @iv3 = nil
 end
 end
@@ -30,15 +30,24 @@
 fprintf(f, " VM_ASSERT((rb_serial_t)%"PRI_SERIALT_PREFIX"u == RCLASS_SERIAL(RBASIC(obj)->klass));\n", ic_copy->entry->class_serial);
 fprintf(f, " VM_ASSERT(index < ROBJECT_NUMIV(obj));\n");
 % if insn.name == 'setinstancevariable'
 fprintf(f, " if (LIKELY(!RB_OBJ_FROZEN_RAW(obj) && %s)) {\n", status->max_ivar_index >= ROBJECT_EMBED_LEN_MAX ? "true" : "RB_FL_ANY_RAW(obj, ROBJECT_EMBED)");
 fprintf(f, " RB_OBJ_WRITE(obj, &ROBJECT(obj)->as.%s, stack[%d]);\n",
 status->max_ivar_index >= ROBJECT_EMBED_LEN_MAX ? "heap.ivptr[index]" : "ary[index]", b->stack_size - 1);
 fprintf(f, " }\n");
 % else
 fprintf(f, " VALUE val;\n");
 fprintf(f, " if (LIKELY(%s && (val = ROBJECT(obj)->as.%s) != Qundef)) {\n",
 status->max_ivar_index >= ROBJECT_EMBED_LEN_MAX ? "true" : "RB_FL_ANY_RAW(obj, ROBJECT_EMBED)",
 status->max_ivar_index >= ROBJECT_EMBED_LEN_MAX ? "heap.ivptr[index]" : "ary[index]");
 fprintf(f, " stack[%d] = val;\n", b->stack_size);
 fprintf(f, " }\n");
 %end
@@ -1467,6 +1467,8 @@ void
 rb_obj_transient_heap_evacuate(VALUE obj, int promote)
 {
 if (ROBJ_TRANSIENT_P(obj)) {
 uint32_t len = ROBJECT_NUMIV(obj);
 const VALUE *old_ptr = ROBJECT_IVPTR(obj);
 VALUE *new_ptr;
@@ -1493,7 +1495,7 @@ init_iv_list(VALUE obj, uint32_t len, uint32_t newsize, st_table *index_tbl)
 if (RBASIC(obj)->flags & ROBJECT_EMBED) {
 newptr = obj_ivar_heap_alloc(obj, newsize);
 MEMCPY(newptr, ptr, VALUE, len);
- RBASIC(obj)->flags &= ~ROBJECT_EMBED;
 ROBJECT(obj)->as.heap.ivptr = newptr;
 }
 else {
@@ -1503,7 +1505,11 @@ init_iv_list(VALUE obj, uint32_t len, uint32_t newsize, st_table *index_tbl)
 for (; len < newsize; len++) {
 newptr[len] = Qundef;
 }
 ROBJECT(obj)->as.heap.numiv = newsize;
 ROBJECT(obj)->as.heap.iv_index_tbl = index_tbl;
 }
@@ -62,6 +62,9 @@ fn main() {
 // Import YARV bytecode instruction constants
 .allowlist_type("ruby_vminsn_type")
 // From include/ruby/internal/intern/string.h
 .allowlist_function("rb_utf8_str_new")
 .allowlist_function("rb_str_buf_append")
@@ -135,7 +138,8 @@ fn main() {
 // From include/ruby/internal/core/robject.h
 .allowlist_type("ruby_robject_flags")
- .allowlist_type("ruby_robject_consts")
 // From include/ruby/internal/core/rarray.h
 .allowlist_type("ruby_rarray_flags")
@@ -1966,9 +1966,17 @@ fn gen_get_ivar(
 ctx.stack_pop(1);
 }
 // Compile time self is embedded and the ivar index lands within the object
 let test_result = unsafe { FL_TEST_RAW(comptime_receiver, VALUE(ROBJECT_EMBED.as_usize())) != VALUE(0) };
- if test_result && ivar_index < (ROBJECT_EMBED_LEN_MAX.as_usize()) {
 // See ROBJECT_IVPTR() from include/ruby/internal/core/robject.h
 // Guard that self is embedded
@@ -1988,7 +1996,7 @@ fn gen_get_ivar(
 );
 // Load the variable
- let offs = RUBY_OFFSET_ROBJECT_AS_ARY + (ivar_index * SIZEOF_VALUE) as i32;
 let ivar_opnd = mem_opnd(64, REG0, offs);
 mov(cb, REG1, ivar_opnd);
@@ -2019,17 +2027,16 @@ fn gen_get_ivar(
 side_exit,
 );
- // Check that the extended table is big enough
- if ivar_index > (ROBJECT_EMBED_LEN_MAX.as_usize()) {
 // Check that the slot is inside the extended table (num_slots > index)
- let num_slots = mem_opnd(32, REG0, RUBY_OFFSET_ROBJECT_AS_HEAP_NUMIV);
-
 cmp(cb, num_slots, uimm_opnd(ivar_index as u64));
 jle_ptr(cb, counted_exit!(ocb, side_exit, getivar_idx_out_of_range));
 }
 // Get a pointer to the extended table
- let tbl_opnd = mem_opnd(64, REG0, RUBY_OFFSET_ROBJECT_AS_HEAP_IVPTR);
 mov(cb, REG0, tbl_opnd);
 // Read the ivar from the extended table
@@ -739,10 +739,6 @@ mod manual_defs {
 pub const RUBY_OFFSET_RSTRUCT_AS_HEAP_PTR: i32 = 24; // struct RStruct, subfield "as.heap.ptr"
 pub const RUBY_OFFSET_RSTRUCT_AS_ARY: i32 = 16; // struct RStruct, subfield "as.ary"
- pub const RUBY_OFFSET_ROBJECT_AS_ARY: i32 = 16; // struct RObject, subfield "as.ary"
- pub const RUBY_OFFSET_ROBJECT_AS_HEAP_NUMIV: i32 = 16; // struct RObject, subfield "as.heap.numiv"
- pub const RUBY_OFFSET_ROBJECT_AS_HEAP_IVPTR: i32 = 24; // struct RObject, subfield "as.heap.ivptr"
-
 // Constants from rb_control_frame_t vm_core.h
 pub const RUBY_OFFSET_CFP_PC: i32 = 0;
 pub const RUBY_OFFSET_CFP_SP: i32 = 8;
@@ -1,5 +1,6 @@
 /* automatically generated by rust-bindgen 0.59.2 */
 pub const INTEGER_REDEFINED_OP_FLAG: u32 = 1;
 pub const FLOAT_REDEFINED_OP_FLAG: u32 = 2;
 pub const STRING_REDEFINED_OP_FLAG: u32 = 4;
@@ -110,8 +111,10 @@ extern "C" {
 }
 pub const ROBJECT_EMBED: ruby_robject_flags = 8192;
 pub type ruby_robject_flags = u32;
-pub const ROBJECT_EMBED_LEN_MAX: ruby_robject_consts = 3;
-pub type ruby_robject_consts = u32;
 extern "C" {
 pub static mut rb_mKernel: VALUE;
 }
author	Peter Zhu <[email protected]>	2022-07-11 10:09:39 -0400
committer	Peter Zhu <[email protected]>	2022-07-15 09:21:07 -0400
commit	7424ea184f9d67c1c7f3ee97494ed3bd1aa60833 ()
tree	822838e39d81cd2785c970cb45a86854823af6fe
parent	7fda741f6e67b809b08423f0d4e903c078da2eed (diff)