Skip to content

Commit 8afa0e6

Browse files
vtjnashvtjnash
committed
gc: improve mallocarrays locality
small_arraylist_t has much better memory locality and space utilization than a linked list with individually malloc'd elements
1 parent 35976df commit 8afa0e6

File tree

5 files changed

+32
-60
lines changed

5 files changed

+32
-60
lines changed

‎src/gc-common.c‎

Lines changed: 2 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -557,17 +557,8 @@ size_t jl_genericmemory_nbytes(jl_genericmemory_t *m) JL_NOTSAFEPOINT
557557
// tracking Memorys with malloc'd storage
558558
void jl_gc_track_malloced_genericmemory(jl_ptls_t ptls, jl_genericmemory_t *m, int isaligned){
559559
// This is **NOT** a GC safe point.
560-
mallocmemory_t *ma;
561-
if (ptls->gc_tls_common.heap.mafreelist == NULL) {
562-
ma = (mallocmemory_t*)malloc_s(sizeof(mallocmemory_t));
563-
}
564-
else {
565-
ma = ptls->gc_tls_common.heap.mafreelist;
566-
ptls->gc_tls_common.heap.mafreelist = ma->next;
567-
}
568-
ma->a = (jl_genericmemory_t*)((uintptr_t)m | !!isaligned);
569-
ma->next = ptls->gc_tls_common.heap.mallocarrays;
570-
ptls->gc_tls_common.heap.mallocarrays = ma;
560+
void *a = (void*)((uintptr_t)m | !!isaligned);
561+
small_arraylist_push(&ptls->gc_tls_common.heap.mallocarrays, a);
571562
}
572563

573564
// =========================================================================== //

‎src/gc-common.h‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -61,12 +61,6 @@ extern jl_gc_callback_list_t *gc_cblist_notify_gc_pressure;
6161
// malloc wrappers, aligned allocation
6262
// =========================================================================== //
6363

64-
// data structure for tracking malloc'd genericmemory.
65-
typedef struct _mallocmemory_t {
66-
jl_genericmemory_t *a; // lowest bit is tagged if this is aligned memory
67-
struct _mallocmemory_t *next;
68-
} mallocmemory_t;
69-
7064
#if defined(_OS_WINDOWS_)
7165
STATIC_INLINE void *jl_malloc_aligned(size_t sz, size_t align)
7266
{

‎src/gc-debug.c‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1025,12 +1025,11 @@ void gc_stats_big_obj(void)
10251025
v = v->next;
10261026
}
10271027

1028-
mallocmemory_t *ma = ptls2->gc_tls.heap.mallocarrays;
1029-
while (ma != NULL) {
1030-
uint8_t bits =jl_astaggedvalue(ma->a)->bits.gc;
1028+
void **lst = ptls2->gc_tls.heap.mallocarrays.items;
1029+
for (size_t i = 0, l = ptls2->gc_tls.heap.mallocarrays.len; i < l; i++) {
1030+
jl_genericmemory_t *m = (jl_genericmemory_t*)((uintptr_t)lst[i] & ~(uintptr_t)1);
1031+
uint8_t bits = jl_astaggedvalue(m)->bits.gc;
10311032
if (gc_marked(bits)) {
1032-
jl_genericmemory_t *m = (jl_genericmemory_t*)ma->a;
1033-
m = (jl_genericmemory_t*)((uintptr_t)m & ~(uintptr_t)1);
10341033
size_t sz = jl_genericmemory_nbytes(m);
10351034
if (gc_old(bits)) {
10361035
assert(bits == GC_OLD_MARKED);
@@ -1042,7 +1041,6 @@ void gc_stats_big_obj(void)
10421041
stat.nbytes_used += sz;
10431042
}
10441043
}
1045-
ma = ma->next;
10461044
}
10471045
}
10481046
jl_safe_printf("%lld kB (%lld%% old) in %lld large objects (%lld%% old)\n",

‎src/gc-stock.c‎

Lines changed: 24 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -388,24 +388,16 @@ static void sweep_weak_refs(void)
388388
jl_ptls_t ptls2 = gc_all_tls_states[i];
389389
if (ptls2 != NULL) {
390390
size_t n = 0;
391-
size_t ndel = 0;
391+
size_t i = 0;
392392
size_t l = ptls2->gc_tls_common.heap.weak_refs.len;
393393
void **lst = ptls2->gc_tls_common.heap.weak_refs.items;
394-
if (l == 0)
395-
continue;
396-
while (1) {
397-
jl_weakref_t *wr = (jl_weakref_t*)lst[n];
394+
// filter with preserving order
395+
for (i = 0; i < l; i++) {
396+
jl_weakref_t *wr = (jl_weakref_t*)lst[i];
398397
if (gc_marked(jl_astaggedvalue(wr)->bits.gc))
399-
n++;
400-
else
401-
ndel++;
402-
if (n >= l - ndel)
403-
break;
404-
void *tmp = lst[n];
405-
lst[n] = lst[n + ndel];
406-
lst[n + ndel] = tmp;
398+
lst[n++] = wr;
407399
}
408-
ptls2->gc_tls_common.heap.weak_refs.len -= ndel;
400+
ptls2->gc_tls_common.heap.weak_refs.len = n;
409401
}
410402
}
411403
}
@@ -629,10 +621,9 @@ void jl_gc_reset_alloc_count(void) JL_NOTSAFEPOINT
629621
reset_thread_gc_counts();
630622
}
631623

632-
static void jl_gc_free_memory(jl_value_t *v, int isaligned) JL_NOTSAFEPOINT
624+
static void jl_gc_free_memory(jl_genericmemory_t *m, int isaligned) JL_NOTSAFEPOINT
633625
{
634-
assert(jl_is_genericmemory(v));
635-
jl_genericmemory_t *m = (jl_genericmemory_t*)v;
626+
assert(jl_is_genericmemory(m));
636627
assert(jl_genericmemory_how(m) == 1 || jl_genericmemory_how(m) == 2);
637628
char *d = (char*)m->ptr;
638629
size_t freed_bytes = memory_block_usable_size(d, isaligned);
@@ -654,25 +645,25 @@ static void sweep_malloced_memory(void) JL_NOTSAFEPOINT
654645
for (int t_i = 0; t_i < gc_n_threads; t_i++) {
655646
jl_ptls_t ptls2 = gc_all_tls_states[t_i];
656647
if (ptls2 != NULL) {
657-
mallocmemory_t *ma = ptls2->gc_tls_common.heap.mallocarrays;
658-
mallocmemory_t **pma = &ptls2->gc_tls_common.heap.mallocarrays;
659-
while (ma != NULL) {
660-
mallocmemory_t *nxt = ma->next;
661-
jl_value_t *a = (jl_value_t*)((uintptr_t)ma->a & ~1);
662-
int bits = jl_astaggedvalue(a)->bits.gc;
663-
if (gc_marked(bits)) {
664-
pma = &ma->next;
648+
size_t n = 0;
649+
size_t l = ptls2->gc_tls_common.heap.mallocarrays.len;
650+
void **lst = ptls2->gc_tls_common.heap.mallocarrays.items;
651+
// filter without preserving order
652+
while (n < l) {
653+
jl_genericmemory_t *m = (jl_genericmemory_t*)((uintptr_t)lst[n] & ~1);
654+
if (gc_marked(jl_astaggedvalue(m)->bits.gc)) {
655+
n++;
665656
}
666657
else {
667-
*pma = nxt;
668-
int isaligned = (uintptr_t)ma->a & 1;
669-
jl_gc_free_memory(a, isaligned);
670-
ma->next = ptls2->gc_tls_common.heap.mafreelist;
671-
ptls2->gc_tls_common.heap.mafreelist = ma;
658+
int isaligned = (uintptr_t)lst[n] & 1;
659+
jl_gc_free_memory(m, isaligned);
660+
l--;
661+
if (l == 0)
662+
break;
663+
lst[n] = lst[l];
672664
}
673-
gc_time_count_mallocd_memory(bits);
674-
ma = nxt;
675665
}
666+
ptls2->gc_tls_common.heap.mallocarrays.len = l;
676667
}
677668
}
678669
gc_time_mallocd_memory_end();
@@ -3439,8 +3430,7 @@ void jl_init_thread_heap(jl_ptls_t ptls)
34393430
small_arraylist_new(&common_heap->live_tasks, 0);
34403431
for (int i = 0; i < JL_N_STACK_POOLS; i++)
34413432
small_arraylist_new(&common_heap->free_stacks[i], 0);
3442-
common_heap->mallocarrays = NULL;
3443-
common_heap->mafreelist = NULL;
3433+
small_arraylist_new(&common_heap->mallocarrays, 0);
34443434
heap->young_generation_of_bigvals = (bigval_t*)calloc_s(sizeof(bigval_t)); // sentinel
34453435
assert(gc_bigval_sentinel_tag != 0); // make sure the sentinel is initialized
34463436
heap->young_generation_of_bigvals->header = gc_bigval_sentinel_tag;

‎src/gc-tls-common.h‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,9 +21,8 @@ typedef struct {
2121
// that are holding onto a stack from the pool
2222
small_arraylist_t live_tasks;
2323

24-
// variables for tracking malloc'd arrays
25-
struct _mallocmemory_t *mallocarrays;
26-
struct _mallocmemory_t *mafreelist;
24+
// variable for tracking malloc'd arrays
25+
small_arraylist_t mallocarrays;
2726

2827
#define JL_N_STACK_POOLS 16
2928
small_arraylist_t free_stacks[JL_N_STACK_POOLS];

0 commit comments

Comments
 (0)