Port of 03a0247

gc: improve mallocarrays locality (#56801)

Author: gbaraldi

src/gc-debug.c

@@ -1100,13 +1100,14 @@ void gc_stats_big_obj(void)
             v = v->next;
         }
 
-        mallocarray_t *ma = ptls2->heap.mallocarrays;
-        while (ma != NULL) {
-            if (gc_marked(jl_astaggedvalue(ma->a)->bits.gc)) {
+        void **lst = ptls2->gc_tls.heap.mallocarrays.items;
+        for (size_t i = 0, l = ptls2->gc_tls.heap.mallocarrays.len; i < l; i++) {
+            jl_genericmemory_t *m = (jl_genericmemory_t*)((uintptr_t)lst[i] & ~(uintptr_t)1);
+            uint8_t bits = jl_astaggedvalue(m)->bits.gc;
+            if (gc_marked(bits)) {
                 nused++;
-                nbytes += jl_genericmemory_nbytes((jl_genericmemory_t*)ma->a);
+                nbytes += jl_genericmemory_nbytes(m);
             }
-            ma = ma->next;
         }
     }
 

src/gc.c

@@ -1121,17 +1121,8 @@ static void sweep_big(jl_ptls_t ptls, int sweep_full) JL_NOTSAFEPOINT
 
 void jl_gc_track_malloced_genericmemory(jl_ptls_t ptls, jl_genericmemory_t *m, int isaligned){
     // This is **NOT** a GC safe point.
-    mallocarray_t *ma;
-    if (ptls->heap.mafreelist == NULL) {
-        ma = (mallocarray_t*)malloc_s(sizeof(mallocarray_t));
-    }
-    else {
-        ma = ptls->heap.mafreelist;
-        ptls->heap.mafreelist = ma->next;
-    }
-    ma->a = (jl_value_t*)((uintptr_t)m | !!isaligned);
-    ma->next = ptls->heap.mallocarrays;
-    ptls->heap.mallocarrays = ma;
+    void *a = (void*)((uintptr_t)m | !!isaligned);
+    small_arraylist_push(&ptls->heap.mallocarrays, a);
 }
 
 
@@ -1245,24 +1236,23 @@ static void sweep_malloced_memory(void) JL_NOTSAFEPOINT
     for (int t_i = 0; t_i < gc_n_threads; t_i++) {
         jl_ptls_t ptls2 = gc_all_tls_states[t_i];
         if (ptls2 != NULL) {
-            mallocarray_t *ma = ptls2->heap.mallocarrays;
-            mallocarray_t **pma = &ptls2->heap.mallocarrays;
-            while (ma != NULL) {
-                mallocarray_t *nxt = ma->next;
-                jl_value_t *a = (jl_value_t*)((uintptr_t)ma->a & ~1);
-                int bits = jl_astaggedvalue(a)->bits.gc;
-                if (gc_marked(bits)) {
-                    pma = &ma->next;
+            size_t n = 0;
+            size_t l = ptls2->heap.mallocarrays.len;
+            void **lst = ptls2->heap.mallocarrays.items;
+            // filter without preserving order
+            while (n < l) {
+                jl_genericmemory_t *m = (jl_genericmemory_t*)((uintptr_t)lst[n] & ~1);
+                if (gc_marked(jl_astaggedvalue(m)->bits.gc)) {
+                    n++;
                 }
                 else {
-                    *pma = nxt;
-                    int isaligned = (uintptr_t)ma->a & 1;
-                    jl_gc_free_memory(a, isaligned);
-                    free(ma);
+                    int isaligned = (uintptr_t)lst[n] & 1;
+                    jl_gc_free_memory(m, isaligned);
+                    l--;
+                    lst[n] = lst[l];
                 }
-                gc_time_count_mallocd_memory(bits);
-                ma = nxt;
             }
+            ptls2->heap.mallocarrays.len = l;
         }
     }
     gc_time_mallocd_memory_end();
@@ -3968,8 +3958,7 @@ void jl_init_thread_heap(jl_ptls_t ptls)
     small_arraylist_new(&heap->live_tasks, 0);
     for (int i = 0; i < JL_N_STACK_POOLS; i++)
         small_arraylist_new(&heap->free_stacks[i], 0);
-    heap->mallocarrays = NULL;
-    heap->mafreelist = NULL;
+    small_arraylist_new(&heap->mallocarrays, 0);
     heap->big_objects = NULL;
     heap->remset = &heap->_remset[0];
     heap->last_remset = &heap->_remset[1];

src/gc.h

@@ -143,11 +143,6 @@ JL_EXTENSION typedef struct _bigval_t {
 
 // data structure for tracking malloc'd arrays and genericmemory.
 
-typedef struct _mallocarray_t {
-    jl_value_t *a;
-    struct _mallocarray_t *next;
-} mallocarray_t;
-
 // pool page metadata
 typedef struct _jl_gc_pagemeta_t {
     // next metadata structure in per-thread list

src/julia_threads.h

@@ -130,8 +130,7 @@ typedef struct {
     small_arraylist_t live_tasks;
 
     // variables for tracking malloc'd arrays
-    struct _mallocarray_t *mallocarrays;
-    struct _mallocarray_t *mafreelist;
+    small_arraylist_t mallocarrays;
 
     // variables for tracking big objects
     struct _bigval_t *big_objects;

src/mtarraylist.c

@@ -14,8 +14,8 @@ extern "C" {
 // but there can be any number of observers
 
 typedef struct {
-    _Atomic(uint32_t) len;
-    uint32_t max;
+    _Atomic(size_t) len;
+    size_t max;
     _Atomic(_Atomic(void*)*) items;
     _Atomic(void*) _space[SMALL_AL_N_INLINE];
 } small_mtarraylist_t;

src/support/arraylist.h

@@ -5,15 +5,15 @@
 
 #define AL_N_INLINE 29
 
-#define SMALL_AL_N_INLINE 6
+#define SMALL_AL_N_INLINE 5
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
 #include "analyzer_annotations.h"
 
-typedef struct {
+typedef struct { // 32 words
     size_t len;
     size_t max;
     void **items;
@@ -27,9 +27,9 @@ void arraylist_push(arraylist_t *a, void *elt) JL_NOTSAFEPOINT;
 void *arraylist_pop(arraylist_t *a) JL_NOTSAFEPOINT;
 JL_DLLEXPORT void arraylist_grow(arraylist_t *a, size_t n) JL_NOTSAFEPOINT;
 
-typedef struct {
-    uint32_t len;
-    uint32_t max;
+typedef struct { // 8 words
+    size_t len;
+    size_t max;
     void **items;
     void *_space[SMALL_AL_N_INLINE];
 } small_arraylist_t;