Skip to content

Commit e7833e8

Browse files
alpaca-tcalpaca-tc
committed
Optimize callcache invalidation for refinements
Fixes [Bug #21201] This change addresses a performance regression where defining methods inside `refine` blocks caused severe slowdowns. The issue was due to `rb_clear_all_refinement_method_cache()` triggering a full object space scan via `rb_objspace_each_objects` to find and invalidate affected callcaches, which is very inefficient. To fix this, I introduce `vm->cc_refinement_table` to track callcaches related to refinements. This allows us to invalidate only the necessary callcaches without scanning the entire heap, resulting in significant performance improvement.
1 parent 0e0008d commit e7833e8

File tree

9 files changed

+113
-19
lines changed

9 files changed

+113
-19
lines changed

‎gc.c

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2103,6 +2103,15 @@ rb_gc_obj_free_vm_weak_references(VALUE obj)
21032103
break;
21042104
case T_IMEMO:
21052105
switch (imemo_type(obj)) {
2106+
case imemo_callcache: {
2107+
const struct rb_callcache *cc = (const struct rb_callcache *)obj;
2108+
2109+
if (vm_cc_refinement_p(cc)) {
2110+
rb_vm_delete_cc_refinement(cc);
2111+
}
2112+
2113+
break;
2114+
}
21062115
case imemo_callinfo:
21072116
rb_vm_ci_free((const struct rb_callinfo *)obj);
21082117
break;
@@ -3937,6 +3946,23 @@ vm_weak_table_foreach_update_weak_key(st_data_t *key, st_data_t *value, st_data_
39373946
return ret;
39383947
}
39393948

3949+
static int
3950+
vm_weak_table_cc_refinement_foreach(st_data_t key, st_data_t data, int error)
3951+
{
3952+
struct global_vm_table_foreach_data *iter_data = (struct global_vm_table_foreach_data *)data;
3953+
3954+
return iter_data->callback((VALUE)key, iter_data->data);
3955+
}
3956+
3957+
static int
3958+
vm_weak_table_cc_refinement_foreach_update_update(st_data_t *key, st_data_t data, int existing)
3959+
{
3960+
struct global_vm_table_foreach_data *iter_data = (struct global_vm_table_foreach_data *)data;
3961+
3962+
return iter_data->update_callback((VALUE *)key, iter_data->data);
3963+
}
3964+
3965+
39403966
static int
39413967
vm_weak_table_str_sym_foreach(st_data_t key, st_data_t value, st_data_t data, int error)
39423968
{
@@ -4186,8 +4212,21 @@ rb_gc_vm_weak_table_foreach(vm_table_foreach_callback_func callback,
41864212
);
41874213
break;
41884214
}
4215+
case RB_GC_VM_CC_REFINEMENT_TABLE: {
4216+
if (vm->cc_refinement_table) {
4217+
set_foreach_with_replace(
4218+
vm->cc_refinement_table,
4219+
vm_weak_table_cc_refinement_foreach,
4220+
vm_weak_table_cc_refinement_foreach_update_update,
4221+
(st_data_t)&foreach_data
4222+
);
4223+
}
4224+
break;
4225+
}
41894226
case RB_GC_VM_WEAK_TABLE_COUNT:
41904227
rb_bug("Unreacheable");
4228+
default:
4229+
rb_bug("rb_gc_vm_weak_table_foreach: unknown table %d", table);
41914230
}
41924231
}
41934232

‎gc/gc.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ enum rb_gc_vm_weak_tables {
3131
RB_GC_VM_ID2REF_TABLE,
3232
RB_GC_VM_GENERIC_FIELDS_TABLE,
3333
RB_GC_VM_FROZEN_STRINGS_TABLE,
34+
RB_GC_VM_CC_REFINEMENT_TABLE,
3435
RB_GC_VM_WEAK_TABLE_COUNT
3536
};
3637

‎internal/set_table.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,8 @@ size_t rb_set_table_size(const struct set_table *tbl);
3737
set_table *rb_set_init_table_with_size(set_table *tab, const struct st_hash_type *, st_index_t);
3838
#define set_init_numtable rb_set_init_numtable
3939
set_table *rb_set_init_numtable(void);
40+
#define set_init_numtable_with_size rb_set_init_numtable_with_size
41+
set_table *rb_set_init_numtable_with_size(st_index_t size);
4042
#define set_delete rb_set_delete
4143
int rb_set_delete(set_table *, st_data_t *); /* returns 0:notfound 1:deleted */
4244
#define set_insert rb_set_insert

‎method.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -254,6 +254,9 @@ void rb_scope_visibility_set(rb_method_visibility_t);
254254

255255
VALUE rb_unnamed_parameters(int arity);
256256

257+
void rb_vm_insert_cc_refinement(const struct rb_callcache *cc);
258+
void rb_vm_delete_cc_refinement(const struct rb_callcache *cc);
259+
257260
void rb_clear_method_cache(VALUE klass_or_module, ID mid);
258261
void rb_clear_all_refinement_method_cache(void);
259262
void rb_invalidate_method_caches(struct rb_id_table *cm_tbl, struct rb_id_table *cc_tbl);

‎st.c

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2465,6 +2465,12 @@ set_init_numtable(void)
24652465
return set_init_table_with_size(NULL, &type_numhash, 0);
24662466
}
24672467

2468+
set_table *
2469+
set_init_numtable_with_size(st_index_t size)
2470+
{
2471+
return set_init_table_with_size(NULL, &type_numhash, size);
2472+
}
2473+
24682474
size_t
24692475
set_table_size(const struct set_table *tbl)
24702476
{

‎vm.c

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3194,6 +3194,10 @@ ruby_vm_destruct(rb_vm_t *vm)
31943194
st_free_table(vm->ci_table);
31953195
vm->ci_table = NULL;
31963196
}
3197+
if (vm->cc_refinement_table) {
3198+
rb_set_free_table(vm->cc_refinement_table);
3199+
vm->cc_refinement_table = NULL;
3200+
}
31973201
RB_ALTSTACK_FREE(vm->main_altstack);
31983202

31993203
struct global_object_list *next;
@@ -3294,6 +3298,7 @@ vm_memsize(const void *ptr)
32943298
vm_memsize_builtin_function_table(vm->builtin_function_table) +
32953299
rb_id_table_memsize(vm->negative_cme_table) +
32963300
rb_st_memsize(vm->overloaded_cme_table) +
3301+
rb_set_memsize(vm->cc_refinement_table) +
32973302
vm_memsize_constant_cache()
32983303
);
32993304

@@ -4503,6 +4508,7 @@ Init_vm_objects(void)
45034508
vm->mark_object_ary = pin_array_list_new(Qnil);
45044509
vm->loading_table = st_init_strtable();
45054510
vm->ci_table = st_init_table(&vm_ci_hashtype);
4511+
vm->cc_refinement_table = rb_set_init_numtable();
45064512
}
45074513

45084514
// Stub for builtin function when not building YJIT units

‎vm_callinfo.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -345,6 +345,7 @@ vm_cc_new(VALUE klass,
345345
break;
346346
case cc_type_refinement:
347347
*(VALUE *)&cc->flags |= VM_CALLCACHE_REFINEMENT;
348+
rb_vm_insert_cc_refinement(cc);
348349
break;
349350
}
350351

‎vm_core.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -803,6 +803,7 @@ typedef struct rb_vm_struct {
803803
struct rb_id_table *negative_cme_table;
804804
st_table *overloaded_cme_table; // cme -> overloaded_cme
805805
set_table *unused_block_warning_table;
806+
set_table *cc_refinement_table;
806807

807808
// This id table contains a mapping from ID to ICs. It does this with ID
808809
// keys and nested st_tables as values. The nested tables have ICs as keys

‎vm_method.c

Lines changed: 54 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -393,27 +393,29 @@ rb_invalidate_method_caches(struct rb_id_table *cm_tbl, struct rb_id_table *cc_t
393393
}
394394

395395
static int
396-
invalidate_all_refinement_cc(void *vstart, void *vend, size_t stride, void *data)
397-
{
398-
VALUE v = (VALUE)vstart;
399-
for (; v != (VALUE)vend; v += stride) {
400-
void *ptr = rb_asan_poisoned_object_p(v);
401-
rb_asan_unpoison_object(v, false);
402-
403-
if (RBASIC(v)->flags) { // liveness check
404-
if (imemo_type_p(v, imemo_callcache)) {
405-
const struct rb_callcache *cc = (const struct rb_callcache *)v;
406-
if (vm_cc_refinement_p(cc) && cc->klass) {
407-
vm_cc_invalidate(cc);
408-
}
409-
}
410-
}
396+
invalidate_cc_refinement(st_data_t key, st_data_t data)
397+
{
398+
VALUE v = (VALUE)key;
399+
void *ptr = rb_asan_poisoned_object_p(v);
400+
rb_asan_unpoison_object(v, false);
411401

412-
if (ptr) {
413-
rb_asan_poison_object(v);
402+
if (rb_gc_pointer_to_heap_p(v) &&
403+
!rb_objspace_garbage_object_p(v) &&
404+
RBASIC(v)->flags) { // liveness check
405+
const struct rb_callcache *cc = (const struct rb_callcache *)v;
406+
407+
VM_ASSERT(vm_cc_refinement_p(cc));
408+
409+
if (cc->klass) {
410+
vm_cc_invalidate(cc);
414411
}
415412
}
416-
return 0; // continue to iteration
413+
414+
if (ptr) {
415+
rb_asan_poison_object(v);
416+
}
417+
418+
return ST_CONTINUE;
417419
}
418420

419421
static st_index_t
@@ -525,10 +527,43 @@ rb_vm_ci_free(const struct rb_callinfo *ci)
525527
st_delete(vm->ci_table, &key, NULL);
526528
}
527529

530+
void
531+
rb_vm_insert_cc_refinement(const struct rb_callcache *cc)
532+
{
533+
st_data_t key = (st_data_t)cc;
534+
535+
rb_vm_t *vm = GET_VM();
536+
RB_VM_LOCK_ENTER();
537+
{
538+
rb_set_insert(vm->cc_refinement_table, key);
539+
}
540+
RB_VM_LOCK_LEAVE();
541+
}
542+
543+
void
544+
rb_vm_delete_cc_refinement(const struct rb_callcache *cc)
545+
{
546+
ASSERT_vm_locking();
547+
548+
rb_vm_t *vm = GET_VM();
549+
st_data_t key = (st_data_t)cc;
550+
551+
rb_set_delete(vm->cc_refinement_table, &key);
552+
}
553+
528554
void
529555
rb_clear_all_refinement_method_cache(void)
530556
{
531-
rb_objspace_each_objects(invalidate_all_refinement_cc, NULL);
557+
rb_vm_t *vm = GET_VM();
558+
559+
RB_VM_LOCK_ENTER();
560+
{
561+
rb_set_foreach(vm->cc_refinement_table, invalidate_cc_refinement, (st_data_t)NULL);
562+
rb_set_clear(vm->cc_refinement_table);
563+
rb_set_compact_table(vm->cc_refinement_table);
564+
}
565+
RB_VM_LOCK_LEAVE();
566+
532567
rb_yjit_invalidate_all_method_lookup_assumptions();
533568
}
534569

0 commit comments

Comments
 (0)