1 files changed, 339 insertions, 146 deletions
@@ -50,6 +50,11 @@ MJIT_STATIC VALUE
 ruby_vm_special_exception_copy(VALUE exc)
 {
 VALUE e = rb_obj_alloc(rb_class_real(RBASIC_CLASS(exc)));
 rb_obj_copy_ivar(e, exc);
 return e;
 }
@@ -1085,35 +1090,17 @@ vm_get_cvar_base(const rb_cref_t *cref, const rb_control_frame_t *cfp, int top_l
 return klass;
 }
-static bool
-iv_index_tbl_lookup(struct st_table *iv_index_tbl, ID id, struct rb_iv_index_tbl_entry **ent)
-{
- int found;
- st_data_t ent_data;
-
- if (iv_index_tbl == NULL) return false;
-
- RB_VM_LOCK_ENTER();
- {
- found = st_lookup(iv_index_tbl, (st_data_t)id, &ent_data);
- }
- RB_VM_LOCK_LEAVE();
- if (found) *ent = (struct rb_iv_index_tbl_entry *)ent_data;
-
- return found ? true : false;
-}
-
-ALWAYS_INLINE(static void fill_ivar_cache(const rb_iseq_t *iseq, IVC ic, const struct rb_callcache *cc, int is_attr, struct rb_iv_index_tbl_entry *ent));
-
 static inline void
-fill_ivar_cache(const rb_iseq_t *iseq, IVC ic, const struct rb_callcache *cc, int is_attr, struct rb_iv_index_tbl_entry *ent)
 {
- // fill cache
- if (!is_attr) {
- vm_ic_entry_set(ic, ent, iseq);
 }
 else {
- vm_cc_attr_index_set(cc, ent->index);
 }
 }
@@ -1123,68 +1110,120 @@ vm_getivar(VALUE obj, ID id, const rb_iseq_t *iseq, IVC ic, const struct rb_call
 {
 #if OPT_IC_FOR_IVAR
 VALUE val = Qundef;
 if (SPECIAL_CONST_P(obj)) {
- // frozen?
 }
- else if (LIKELY(is_attr ?
- RB_DEBUG_COUNTER_INC_UNLESS(ivar_get_ic_miss_unset, vm_cc_attr_index_p(cc)) :
- RB_DEBUG_COUNTER_INC_UNLESS(ivar_get_ic_miss_serial, vm_ic_entry_p(ic) && ic->entry->class_serial == RCLASS_SERIAL(RBASIC(obj)->klass)))) {
- uint32_t index = !is_attr ? vm_ic_entry_index(ic): (vm_cc_attr_index(cc));
- RB_DEBUG_COUNTER_INC(ivar_get_ic_hit);
-
- if (LIKELY(BUILTIN_TYPE(obj) == T_OBJECT) &&
- LIKELY(index < ROBJECT_NUMIV(obj))) {
- val = ROBJECT_IVPTR(obj)[index];
 VM_ASSERT(rb_ractor_shareable_p(obj) ? rb_ractor_shareable_p(val) : true);
- }
- else if (FL_TEST_RAW(obj, FL_EXIVAR)) {
- val = rb_ivar_generic_lookup_with_index(obj, id, index);
- }
- goto ret;
 }
- else {
- struct rb_iv_index_tbl_entry *ent;
- if (BUILTIN_TYPE(obj) == T_OBJECT) {
- struct st_table *iv_index_tbl = ROBJECT_IV_INDEX_TBL(obj);
- if (iv_index_tbl && iv_index_tbl_lookup(iv_index_tbl, id, &ent)) {
- fill_ivar_cache(iseq, ic, cc, is_attr, ent);
- // get value
- if (ent->index < ROBJECT_NUMIV(obj)) {
- val = ROBJECT_IVPTR(obj)[ent->index];
- VM_ASSERT(rb_ractor_shareable_p(obj) ? rb_ractor_shareable_p(val) : true);
- }
- }
 }
- else if (FL_TEST_RAW(obj, FL_EXIVAR)) {
- struct st_table *iv_index_tbl = RCLASS_IV_INDEX_TBL(rb_obj_class(obj));
- if (iv_index_tbl && iv_index_tbl_lookup(iv_index_tbl, id, &ent)) {
- fill_ivar_cache(iseq, ic, cc, is_attr, ent);
- val = rb_ivar_generic_lookup_with_index(obj, id, ent->index);
 }
 }
 else {
- // T_CLASS / T_MODULE
- goto general_path;
 }
- ret:
- if (LIKELY(val != Qundef)) {
- return val;
 }
 else {
- return Qnil;
 }
 }
- general_path:
 #endif /* OPT_IC_FOR_IVAR */
 RB_DEBUG_COUNTER_INC(ivar_get_ic_miss);
@@ -1196,6 +1235,20 @@ vm_getivar(VALUE obj, ID id, const rb_iseq_t *iseq, IVC ic, const struct rb_call
 }
 }
 ALWAYS_INLINE(static VALUE vm_setivar_slowpath(VALUE obj, ID id, VALUE val, const rb_iseq_t *iseq, IVC ic, const struct rb_callcache *cc, int is_attr));
 NOINLINE(static VALUE vm_setivar_slowpath_ivar(VALUE obj, ID id, VALUE val, const rb_iseq_t *iseq, IVC ic));
 NOINLINE(static VALUE vm_setivar_slowpath_attr(VALUE obj, ID id, VALUE val, const struct rb_callcache *cc));
@@ -1203,35 +1256,72 @@ NOINLINE(static VALUE vm_setivar_slowpath_attr(VALUE obj, ID id, VALUE val, cons
 static VALUE
 vm_setivar_slowpath(VALUE obj, ID id, VALUE val, const rb_iseq_t *iseq, IVC ic, const struct rb_callcache *cc, int is_attr)
 {
- rb_check_frozen_internal(obj);
-
 #if OPT_IC_FOR_IVAR
- if (RB_TYPE_P(obj, T_OBJECT)) {
- struct st_table *iv_index_tbl = ROBJECT_IV_INDEX_TBL(obj);
- struct rb_iv_index_tbl_entry *ent;
- if (iv_index_tbl_lookup(iv_index_tbl, id, &ent)) {
- if (!is_attr) {
- vm_ic_entry_set(ic, ent, iseq);
- }
- else if (ent->index >= INT_MAX) {
- rb_raise(rb_eArgError, "too many instance variables");
- }
- else {
- vm_cc_attr_index_set(cc, (int)(ent->index));
- }
- uint32_t index = ent->index;
- if (UNLIKELY(index >= ROBJECT_NUMIV(obj))) {
- rb_init_iv_list(obj);
 }
- VALUE *ptr = ROBJECT_IVPTR(obj);
- RB_OBJ_WRITE(obj, &ptr[index], val);
- RB_DEBUG_COUNTER_INC(ivar_set_ic_miss_iv_hit);
- return val;
- }
 }
 #endif
 RB_DEBUG_COUNTER_INC(ivar_set_ic_miss);
@@ -1250,39 +1340,94 @@ vm_setivar_slowpath_attr(VALUE obj, ID id, VALUE val, const struct rb_callcache
 return vm_setivar_slowpath(obj, id, val, NULL, NULL, cc, true);
 }
 static inline VALUE
-vm_setivar(VALUE obj, ID id, VALUE val, const rb_iseq_t *iseq, IVC ic, const struct rb_callcache *cc, int is_attr)
 {
 #if OPT_IC_FOR_IVAR
- if (LIKELY(RB_TYPE_P(obj, T_OBJECT)) &&
- LIKELY(!RB_OBJ_FROZEN_RAW(obj))) {
- VM_ASSERT(!rb_ractor_shareable_p(obj));
- if (LIKELY(
- (!is_attr && RB_DEBUG_COUNTER_INC_UNLESS(ivar_set_ic_miss_serial, vm_ic_entry_p(ic) && ic->entry->class_serial == RCLASS_SERIAL(RBASIC(obj)->klass))) ||
- ( is_attr && RB_DEBUG_COUNTER_INC_UNLESS(ivar_set_ic_miss_unset, vm_cc_attr_index_p(cc))))) {
- uint32_t index = !is_attr ? vm_ic_entry_index(ic) : vm_cc_attr_index(cc);
- if (UNLIKELY(index >= ROBJECT_NUMIV(obj))) {
- rb_init_iv_list(obj);
 }
- VALUE *ptr = ROBJECT_IVPTR(obj);
- RB_OBJ_WRITE(obj, &ptr[index], val);
- RB_DEBUG_COUNTER_INC(ivar_set_ic_hit);
- return val; /* inline cache hit */
- }
- }
- else {
 RB_DEBUG_COUNTER_INC(ivar_set_ic_miss_noobject);
 }
 #endif /* OPT_IC_FOR_IVAR */
- if (is_attr) {
- return vm_setivar_slowpath_attr(obj, id, val, cc);
- }
- else {
- return vm_setivar_slowpath_ivar(obj, id, val, iseq, ic);
- }
 }
 static VALUE
@@ -1377,7 +1522,22 @@ vm_getinstancevariable(const rb_iseq_t *iseq, VALUE obj, ID id, IVC ic)
 static inline void
 vm_setinstancevariable(const rb_iseq_t *iseq, VALUE obj, ID id, VALUE val, IVC ic)
 {
- vm_setivar(obj, id, val, iseq, ic, 0, 0);
 }
 void
@@ -1386,28 +1546,6 @@ rb_vm_setinstancevariable(const rb_iseq_t *iseq, VALUE obj, ID id, VALUE val, IV
 vm_setinstancevariable(iseq, obj, id, val, ic);
 }
-/* Set the instance variable +val+ on object +obj+ at the +index+.
- * This function only works with T_OBJECT objects, so make sure
- * +obj+ is of type T_OBJECT before using this function.
- */
-VALUE
-rb_vm_set_ivar_idx(VALUE obj, uint32_t index, VALUE val)
-{
- RUBY_ASSERT(RB_TYPE_P(obj, T_OBJECT));
-
- rb_check_frozen_internal(obj);
-
- VM_ASSERT(!rb_ractor_shareable_p(obj));
-
- if (UNLIKELY(index >= ROBJECT_NUMIV(obj))) {
- rb_init_iv_list(obj);
- }
- VALUE *ptr = ROBJECT_IVPTR(obj);
- RB_OBJ_WRITE(obj, &ptr[index], val);
-
- return val;
-}
-
 static VALUE
 vm_throw_continue(const rb_execution_context_t *ec, VALUE err)
 {
@@ -3100,17 +3238,45 @@ vm_call_ivar(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_call
 const struct rb_callcache *cc = calling->cc;
 RB_DEBUG_COUNTER_INC(ccf_ivar);
 cfp->sp -= 1;
- return vm_getivar(calling->recv, vm_cc_cme(cc)->def->body.attr.id, NULL, NULL, cc, TRUE);
 }
 static VALUE
-vm_call_attrset(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
 {
- const struct rb_callcache *cc = calling->cc;
 RB_DEBUG_COUNTER_INC(ccf_attrset);
 VALUE val = *(cfp->sp - 1);
 cfp->sp -= 2;
- return vm_setivar(calling->recv, vm_cc_cme(cc)->def->body.attr.id, val, NULL, NULL, cc, 1);
 }
 bool
@@ -3219,7 +3385,7 @@ vm_call_alias(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_cal
 {
 calling->cc = &VM_CC_ON_STACK(Qundef,
 vm_call_general,
- { 0 },
 aliased_callable_method_entry(vm_cc_cme(calling->cc)));
 return vm_call_method_each_type(ec, cfp, calling);
@@ -3389,7 +3555,7 @@ vm_call_method_missing_body(rb_execution_context_t *ec, rb_control_frame_t *reg_
 ec->method_missing_reason = reason;
 calling->ci = &VM_CI_ON_STACK(idMethodMissing, flag, argc, vm_ci_kwarg(orig_ci));
- calling->cc = &VM_CC_ON_STACK(Qundef, vm_call_general, { 0 },
 rb_callable_method_entry_without_refinements(CLASS_OF(calling->recv), idMethodMissing, NULL));
 return vm_call_method(ec, reg_cfp, calling);
 }
@@ -3415,7 +3581,7 @@ vm_call_zsuper(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_ca
 cme = refined_method_callable_without_refinement(cme);
 }
- calling->cc = &VM_CC_ON_STACK(Qundef, vm_call_general, { 0 }, cme);
 return vm_call_method_each_type(ec, cfp, calling);
 }
@@ -3522,7 +3688,7 @@ search_refined_method(rb_execution_context_t *ec, rb_control_frame_t *cfp, struc
 static VALUE
 vm_call_refined(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling)
 {
- struct rb_callcache *ref_cc = &VM_CC_ON_STACK(Qundef, vm_call_general, { 0 },
 search_refined_method(ec, cfp, calling));
 if (vm_cc_cme(ref_cc)) {
@@ -3702,18 +3868,45 @@ vm_call_method_each_type(rb_execution_context_t *ec, rb_control_frame_t *cfp, st
 CALLER_REMOVE_EMPTY_KW_SPLAT(cfp, calling, ci);
 rb_check_arity(calling->argc, 1, 1);
- vm_cc_attr_index_initialize(cc);
 const unsigned int aset_mask = (VM_CALL_ARGS_SPLAT | VM_CALL_KW_SPLAT | VM_CALL_KWARG);
- VM_CALL_METHOD_ATTR(v,
- vm_call_attrset(ec, cfp, calling),
- CC_SET_FASTPATH(cc, vm_call_attrset, !(vm_ci_flag(ci) & aset_mask)));
 return v;
 case VM_METHOD_TYPE_IVAR:
 CALLER_SETUP_ARG(cfp, calling, ci);
 CALLER_REMOVE_EMPTY_KW_SPLAT(cfp, calling, ci);
 rb_check_arity(calling->argc, 0, 0);
- vm_cc_attr_index_initialize(cc);
 const unsigned int ivar_mask = (VM_CALL_ARGS_SPLAT | VM_CALL_KW_SPLAT);
 VM_CALL_METHOD_ATTR(v,
 vm_call_ivar(ec, cfp, calling),