Implement fixed heap size for Julia (#38)

This PR introduces fixed heap size for stock Julia. With the build time option `WITH_GC_FIXED_HEAP=1` and using `--fixed-heap-size=...`, it will bypass all the existing GC triggering heuristics, and only do GC when the heap size reaches the defined heap size, and will only do a full heap GC if the free memory after a GC is less than 20% of the heap size.

This PR also introduces a global counter for mallocd bytes. This will slow down the performance of malloc. For MMTK Julia, we also use such a counter (see mmtk/mmtk-julia#141). I plan to do another PR to fix this for both MMTK Julia and stock Julia.

Author: qinsoon

Make.inc

@@ -86,6 +86,9 @@ HAVE_SSP := 0
 WITH_GC_VERIFY := 0
 WITH_GC_DEBUG_ENV := 0
 
+# Overwrite Julia's GC heuristics and only trigger a GC if the heap is full (fixed_heap_size needs to be set in this build)
+WITH_GC_FIXED_HEAP ?= 0
+
 # MMTk GC
 WITH_MMTK ?= 0
 
@@ -738,6 +741,11 @@ JCXXFLAGS += -DGC_DEBUG_ENV
 JCFLAGS += -DGC_DEBUG_ENV
 endif
 
+ifeq ($(WITH_GC_FIXED_HEAP), 1)
+JCXXFLAGS += -DGC_FIXED_HEAP
+JCFLAGS += -DGC_FIXED_HEAP
+endif
+
 ifeq ($(WITH_MMTK), 1)
 ifeq (${MMTK_JULIA_DIR},)
 $(error MMTK_JULIA_DIR must be set to use MMTk)

base/options.jl

@@ -56,6 +56,7 @@ struct JLOptions
     strip_ir::Int8
     permalloc_pkgimg::Int8
     heap_size_hint::UInt64
+    fixed_heap_size::UInt64
 end
 
 # This runs early in the sysimage != is not defined yet

src/gc.c

@@ -25,6 +25,13 @@ _Atomic(int) gc_master_tid;
 uv_mutex_t gc_threads_lock;
 uv_cond_t gc_threads_cond;
 
+#ifdef GC_FIXED_HEAP
+// Globally allocated bytes by malloc - used for fixed heap size
+_Atomic(uint64_t) malloc_bytes;
+// Globally allocated pool pages - used for fixed heap size
+extern uint64_t jl_current_pg_count(void);
+#endif
+
 // Linked list of callback functions
 
 typedef void (*jl_gc_cb_func_t)(void);
@@ -393,6 +400,9 @@ extern int64_t live_bytes;
 static int64_t perm_scanned_bytes; // old bytes scanned while marking
 int prev_sweep_full = 1;
 int current_sweep_full = 0;
+#ifdef GC_FIXED_HEAP
+int next_sweep_full = 0; // force next sweep to be a full sweep - used by fixed heap size
+#endif
 
 // Full collection heuristics
 static int64_t promoted_bytes = 0;
@@ -574,6 +584,18 @@ void gc_setmark_buf(jl_ptls_t ptls, void *o, uint8_t mark_mode, size_t minsz) JL
 
 inline void maybe_collect(jl_ptls_t ptls)
 {
+#ifdef GC_FIXED_HEAP
+    if (jl_options.fixed_heap_size) {
+        uint64_t current_heap_size = ((uint64_t)jl_current_pg_count()) << (uint64_t)14;
+        current_heap_size += jl_atomic_load_relaxed(&malloc_bytes);
+        if (current_heap_size >= jl_options.fixed_heap_size) {
+            jl_gc_collect(JL_GC_AUTO);
+        } else {
+            jl_gc_safepoint_(ptls);
+        }
+        return;
+    }
+#endif
     if (jl_atomic_load_relaxed(&ptls->gc_num.allocd) >= 0 || jl_gc_debug_check_other()) {
         jl_gc_collect(JL_GC_AUTO);
     }
@@ -2708,6 +2730,16 @@ static int _jl_gc_collect(jl_ptls_t ptls, jl_gc_collection_t collection)
         sweep_full = 1;
         recollect = 1;
     }
+#ifdef GC_FIXED_HEAP
+    if (jl_options.fixed_heap_size) {
+        // For fixed heap size, do not trigger full sweep for any other heuristics
+        sweep_full = 0;
+    }
+    if (next_sweep_full) {
+        next_sweep_full = 0;
+        sweep_full = 1;
+    }
+#endif
     if (sweep_full) {
         // these are the difference between the number of gc-perm bytes scanned
         // on the first collection after sweep_full, and the current scan
@@ -2824,6 +2856,15 @@ static int _jl_gc_collect(jl_ptls_t ptls, jl_gc_collection_t collection)
         }
     }
 
+#ifdef GC_FIXED_HEAP
+    if (jl_options.fixed_heap_size) {
+        uint64_t current_heap_size = ((uint64_t)jl_current_pg_count()) << ((uint64_t)14);
+        if (current_heap_size > (jl_options.fixed_heap_size * 4 / 5)) {
+            next_sweep_full = 1;
+        }
+    }
+#endif
+
     gc_time_summary(sweep_full, t_start, gc_end_time, gc_num.freed,
                     live_bytes, gc_num.interval, pause,
                     gc_num.time_to_safepoint,
@@ -3029,6 +3070,16 @@ void jl_gc_init(void)
 #endif
     if (jl_options.heap_size_hint)
         jl_gc_set_max_memory(jl_options.heap_size_hint);
+
+#ifdef GC_FIXED_HEAP
+    if (jl_options.fixed_heap_size) {
+        // This guarantees that we will not trigger a GC before reaching heap limit
+        gc_num.interval = jl_options.fixed_heap_size;
+    } else {
+        jl_printf(JL_STDERR, "Warning: The option fixed-heap-size is not set for a build with WITH_GC_FIXED_HEAP\n");
+    }
+#endif
+
     t_start = jl_hrtime();
 }
 
@@ -3045,6 +3096,9 @@ JL_DLLEXPORT void *jl_gc_counted_malloc(size_t sz)
             jl_atomic_load_relaxed(&ptls->gc_num.allocd) + sz);
         jl_atomic_store_relaxed(&ptls->gc_num.malloc,
             jl_atomic_load_relaxed(&ptls->gc_num.malloc) + 1);
+#ifdef GC_FIXED_HEAP
+        jl_atomic_fetch_add_relaxed(&malloc_bytes, sz);
+#endif
     }
     return malloc(sz);
 }
@@ -3060,6 +3114,9 @@ JL_DLLEXPORT void *jl_gc_counted_calloc(size_t nm, size_t sz)
             jl_atomic_load_relaxed(&ptls->gc_num.allocd) + nm*sz);
         jl_atomic_store_relaxed(&ptls->gc_num.malloc,
             jl_atomic_load_relaxed(&ptls->gc_num.malloc) + 1);
+#ifdef GC_FIXED_HEAP
+        jl_atomic_fetch_add_relaxed(&malloc_bytes, nm * sz);
+#endif
     }
     return calloc(nm, sz);
 }
@@ -3075,6 +3132,9 @@ JL_DLLEXPORT void jl_gc_counted_free_with_size(void *p, size_t sz)
             jl_atomic_load_relaxed(&ptls->gc_num.freed) + sz);
         jl_atomic_store_relaxed(&ptls->gc_num.freecall,
             jl_atomic_load_relaxed(&ptls->gc_num.freecall) + 1);
+#ifdef GC_FIXED_HEAP
+        jl_atomic_fetch_add_relaxed(&malloc_bytes, -sz);
+#endif
     }
 }
 
@@ -3085,12 +3145,19 @@ JL_DLLEXPORT void *jl_gc_counted_realloc_with_old_size(void *p, size_t old, size
     if (pgcstack != NULL && ct->world_age) {
         jl_ptls_t ptls = ct->ptls;
         maybe_collect(ptls);
-        if (sz < old)
+        if (sz < old) {
             jl_atomic_store_relaxed(&ptls->gc_num.freed,
                 jl_atomic_load_relaxed(&ptls->gc_num.freed) + (old - sz));
-        else
+#ifdef GC_FIXED_HEAP
+            jl_atomic_fetch_add_relaxed(&malloc_bytes, old - sz);
+#endif
+        } else {
             jl_atomic_store_relaxed(&ptls->gc_num.allocd,
                 jl_atomic_load_relaxed(&ptls->gc_num.allocd) + (sz - old));
+#ifdef GC_FIXED_HEAP
+            jl_atomic_fetch_add_relaxed(&malloc_bytes, sz - old);
+#endif
+        }
         jl_atomic_store_relaxed(&ptls->gc_num.realloc,
             jl_atomic_load_relaxed(&ptls->gc_num.realloc) + 1);
     }

src/jloptions.c

@@ -255,6 +255,7 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp)
            opt_strip_metadata,
            opt_strip_ir,
            opt_heap_size_hint,
+           opt_fixed_heap_size,
            opt_permalloc_pkgimg,
            opt_gc_threads,
     };
@@ -318,6 +319,7 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp)
         { "strip-ir",        no_argument,       0, opt_strip_ir },
         { "permalloc-pkgimg",required_argument, 0, opt_permalloc_pkgimg },
         { "heap-size-hint",  required_argument, 0, opt_heap_size_hint },
+        { "fixed-heap-size", required_argument, 0, opt_fixed_heap_size },
         { 0, 0, 0, 0 }
     };
 
@@ -823,6 +825,39 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp)
                 jl_errorf("julia: invalid argument to --heap-size-hint without memory size specified");
 
             break;
+        case opt_fixed_heap_size:
+            if (optarg != NULL) {
+                size_t endof = strlen(optarg);
+                long double value = 0.0;
+                if (sscanf(optarg, "%Lf", &value) == 1 && value > 1e-7) {
+                    char unit = optarg[endof - 1];
+                    uint64_t multiplier = 1ull;
+                    switch (unit) {
+                        case 'k':
+                        case 'K':
+                            multiplier <<= 10;
+                            break;
+                        case 'm':
+                        case 'M':
+                            multiplier <<= 20;
+                            break;
+                        case 'g':
+                        case 'G':
+                            multiplier <<= 30;
+                            break;
+                        case 't':
+                        case 'T':
+                            multiplier <<= 40;
+                            break;
+                        default:
+                            break;
+                    }
+                    jl_options.fixed_heap_size = (uint64_t)(value * multiplier);
+                }
+            }
+            if (jl_options.fixed_heap_size == 0)
+                jl_errorf("julia: invalid argument to --fixed-heap-size without memory size specified");
+            break;
         case opt_permalloc_pkgimg:
             if (!strcmp(optarg,"yes"))
                 jl_options.permalloc_pkgimg = 1;

src/jloptions.h

@@ -60,6 +60,7 @@ typedef struct {
     int8_t strip_ir;
     int8_t permalloc_pkgimg;
     uint64_t heap_size_hint;
+    uint64_t fixed_heap_size;
 } jl_options_t;
 
 #endif