[mmtk-julia] Allow building Julia with variations of the binding (#57176)

This PR enables building the mmtk-julia binding with different
configurations other than non-moving Immix. While it was already
possible to do so when building from source, this should enable using
the binaries generated via the latest version from BinaryBuilder
(JuliaPackaging/Yggdrasil#10357).
I have also decoupled MMTk from the code, such that it should be *an
option* for a third-party heap. This should be set using
`WITH_THIRD_PARTY_GC=mmtk` (in `Make.user` or as an environment
variable).

The different configurations of MMTk can be achieved by setting the
variables below (set in `Make.user`, for example):

| Variable      |       |        |
|---------------|--------------|---------------|
| `MMTK_PLAN`     | Immix        | StickyImmix   |
| `MMTK_MOVING`   | 0            | 1             |
| `MMTK_BUILD`    | release      | debug         |

Note that the actual code to support building with moving and sticky
immix will be added in future PRs.
I've also added some preliminary documentation in `doc/gc.md` about
building Julia with MMTk, including a link to the binding and an small
FAQ on what to do if you break the build of Julia+MMTk.

Author: udesou

Make.inc

@@ -80,9 +80,6 @@ HAVE_SSP := 0
 WITH_GC_VERIFY := 0
 WITH_GC_DEBUG_ENV := 0
 
-# Use stock if MMTK_PLAN hasn't been defined
-MMTK_PLAN ?= None
-
 # Enable DTrace support
 WITH_DTRACE := 0
 
@@ -841,29 +838,44 @@ JCXXFLAGS += -DGC_DEBUG_ENV
 JCFLAGS += -DGC_DEBUG_ENV
 endif
 
-ifneq (${MMTK_PLAN},None)
-JCXXFLAGS += -DMMTK_GC
-JCFLAGS += -DMMTK_GC
-# Do a release build on the binding by default
-MMTK_BUILD ?= release
+# By default we use the stock GC
+WITH_THIRD_PARTY_GC ?= 0
+
+# Convert to lowercase
+USE_THIRD_PARTY_GC := $(shell echo $(WITH_THIRD_PARTY_GC) | tr A-Z a-z)
+
+# NB: When adding a new GC, make sure to add -DWITH_THIRD_PARTY_HEAP=<#NEW_GC>
+# to the variables JCFLAGS and JCXXFLAGS according to the mapping (MMTK=1, #NEW_GC=2)
+
+ifeq (${USE_THIRD_PARTY_GC},mmtk)
+JCXXFLAGS += -DWITH_THIRD_PARTY_HEAP=1
+JCFLAGS += -DWITH_THIRD_PARTY_HEAP=1
+
+# Must specify a supported MMTk Plan: Immix or StickyImmix
 ifeq (${MMTK_PLAN},Immix)
 JCXXFLAGS += -DMMTK_PLAN_IMMIX
 JCFLAGS += -DMMTK_PLAN_IMMIX
+else ifeq (${MMTK_PLAN},StickyImmix)
+JCXXFLAGS += -DMMTK_PLAN_STICKYIMMIX
+JCFLAGS += -DMMTK_PLAN_STICKYIMMIX
 else
-$(error "Unsupported MMTk plan: $(MMTK_PLAN)")
+$(error "Unsupported MMTk plan: $(MMTK_PLAN). Supported plan(s): Immix or StickyImmix.")
 endif
 
+# Do a release build on the binding by default
+MMTK_BUILD ?= release
+
 # Location of mmtk-julia binding
 # (needed for api/*.h and .so file)
 MMTK_JULIA_DIR ?= $(BUILDROOT)/usr/lib/mmtk_julia
-
 MMTK_DIR = ${MMTK_JULIA_DIR}/mmtk
 MMTK_API_INC = ${MMTK_DIR}/api
-
 MMTK_LIB := -lmmtk_julia
-else
-MMTK_JULIA_INC :=
-MMTK_LIB :=
+
+# Must specify a supported third-party GC or use the stock GC.
+# Supported third-party GCs: mmtk
+else ifneq (${USE_THIRD_PARTY_GC},0)
+$(error "Unsupported third-party GC: $(WITH_THIRD_PARTY_GC). Supported option(s): mmtk")
 endif
 
 ifeq ($(WITH_DTRACE), 1)
@@ -966,19 +978,17 @@ ARCH := $(BUILD_OS)
 endif
 endif
 
-# MMTk is only available on x86_64 Linux for now
-ifneq (${MMTK_PLAN},None)
 
+ifeq (${USE_THIRD_PARTY_GC},mmtk)
+# MMTk is only available on x86_64 Linux for now
 ifeq ($(OS),Linux)
 MMTK_LIB_NAME := libmmtk_julia.so
 else
 $(error "Unsupported OS for MMTk")
 endif
-
 ifneq ($(ARCH),x86_64)
 $(error "Unsupported build architecture for MMTk")
 endif
-
 endif
 
 # Detect common pre-SSE2 JULIA_CPU_TARGET values known not to work (#7185)
@@ -1407,7 +1417,12 @@ CSL_NEXT_GLIBCXX_VERSION=GLIBCXX_3\.4\.34|GLIBCXX_3\.5\.|GLIBCXX_4\.
 # Note: we explicitly _do not_ define `CSL` here, since it requires some more
 # advanced techniques to decide whether it should be installed from a BB source
 # or not.  See `deps/csl.mk` for more detail.
-BB_PROJECTS := BLASTRAMPOLINE OPENBLAS LLVM LIBSUITESPARSE OPENLIBM GMP OPENSSL LIBSSH2 NGHTTP2 MPFR CURL LIBGIT2 PCRE LIBUV LIBUNWIND DSFMT OBJCONV ZLIB P7ZIP LLD LIBTRACYCLIENT BOLT MMTK_JULIA
+BB_PROJECTS := BLASTRAMPOLINE OPENBLAS LLVM LIBSUITESPARSE OPENLIBM GMP OPENSSL LIBSSH2 NGHTTP2 MPFR CURL LIBGIT2 PCRE LIBUV LIBUNWIND DSFMT OBJCONV ZLIB P7ZIP LLD LIBTRACYCLIENT BOLT
+
+ifeq (${USE_THIRD_PARTY_GC},mmtk)
+BB_PROJECTS += MMTK_JULIA
+endif
+
 define SET_BB_DEFAULT
 # First, check to see if BB is disabled on a global setting
 ifeq ($$(USE_BINARYBUILDER),0)

Makefile

@@ -281,7 +281,7 @@ endif
 endif
 endif
 
-ifneq (${MMTK_PLAN},None)
+ifeq (${USE_THIRD_PARTY_GC},mmtk)
 # Make sure we use the right version of $MMTK_PLAN, $MMTK_MOVING and $MMTK_BUILD
 # if we use the BinaryBuilder version of mmtk-julia
 ifeq ($(USE_BINARYBUILDER_MMTK_JULIA),1)
@@ -291,10 +291,10 @@ else ifeq (${MMTK_PLAN},StickyImmix)
 LIB_PATH_PLAN = sticky
 endif
 
-ifeq ($(MMTK_MOVING), 0)
-LIB_PATH_MOVING := non_moving
-else
+ifeq ($(MMTK_MOVING), 1)
 LIB_PATH_MOVING := moving
+else
+LIB_PATH_MOVING := non_moving
 endif
 
 JL_PRIVATE_LIBS-0 += $(LIB_PATH_PLAN)/$(LIB_PATH_MOVING)/$(MMTK_BUILD)/libmmtk_julia

deps/Makefile

@@ -195,7 +195,7 @@ DEP_LIBS += libwhich
 endif
 endif
 
-ifneq (${MMTK_PLAN},None)
+ifeq (${USE_THIRD_PARTY_GC},mmtk)
 DEP_LIBS += mmtk_julia
 endif
 

deps/mmtk_julia.mk

@@ -1,9 +1,9 @@
 ## MMTK ##
 
 # Both MMTK_MOVING and MMTK_PLAN should be specified in the Make.user file.
-# At this point, since we only support non-moving this is always set to 0
-# FIXME: change it to `?:` when introducing moving plans
-MMTK_MOVING := 0
+# FIXME: By default we do a non-moving build. We should change the default to 1
+# once we support moving plans.
+MMTK_MOVING ?= 0
 MMTK_VARS := MMTK_PLAN=$(MMTK_PLAN) MMTK_MOVING=$(MMTK_MOVING)
 
 ifneq ($(USE_BINARYBUILDER_MMTK_JULIA),1)

doc/make.jl

@@ -229,6 +229,7 @@ DevDocs = [
         "devdocs/aot.md",
         "devdocs/gc-sa.md",
         "devdocs/gc.md",
+        "devdocs/gc-mmtk.md",
         "devdocs/jit.md",
         "devdocs/builtins.md",
         "devdocs/precompile_hang.md",

doc/src/devdocs/gc-mmtk.md

@@ -0,0 +1,40 @@
+# Julia + MMTk
+
+There has been quite a lot of effort to refactor the GC code inside Julia to support external GCs. The first step to enable using different GC algorithms for Julia was the design and implementation of a [GC interface](https://docs.google.com/document/d/1v0jtSrIpdEDNOxj5S9g1jPqSpuAkNWhr_T8ToFC9RLI/edit?usp=sharing). To drive that interface, we added support for building Julia with [MMTk](https://www.mmtk.io) (Memory Management Toolkit). Using Julia + MMTk enables testing different GC implementations, allowing developers to choose a specific implementation when building Julia from source. The connection between Julia and MMTk is done via a *binding*, which links the language runtime with MMTk core. The mmtk-julia binding is written in Rust and can be found in [this repository](https://github.com/mmtk/mmtk-julia).
+
+> [!NOTE]
+> Using a different GC requires building Julia from source. It is not possible to switch implementations at runtime. To see what version of the GC is currently being used, run `versioninfo()` from the Julia REPL and it should show the version under `GC: ...`.
+
+## Building Julia with MMTk
+
+There are 3 different ways of building Julia with MMTk: building from source using a fixed release of the binding, checking out a custom version in the mmtk-julia [repository](https://github.com/mmtk/mmtk-julia) or using a precompiled binary from Julia's BinaryBuilder. The easiest way is to use the BinaryBuilder binary. First, to enable MMTk as a third-party GC, set the variable `WITH_THIRD_PARTY_GC` to `mmtk`. Then, for example, to use the Immix as the GC, simply set the variable `MMTK_PLAN=Immix` and build Julia as usual.
+
+There are different configurations supported by the following variables, which can be set in a `Make.user` file or as an environment variable. Note that at this time, setting `MMTK_PLAN=StickyImmix` (to use a generational version of Immix) or `MMTK_MOVING=1` (to enable object movement) will likely cause segmentation faults or other build failures, since we have not added support for these configurations yet. Setting `MMTK_BUILD=debug` will force a debug build of the binding, which will print some logging information that can be used to find errors that are specific to MMTk.
+
+| Variable      |       |        |
+|---------------|--------------|---------------|
+| `MMTK_PLAN`     | Immix        | StickyImmix   |
+| `MMTK_MOVING`   | 0            | 1             |
+| `MMTK_BUILD`    | release      | debug         |
+
+Note that when setting only `MMTK_PLAN`, then the default is to do a non-moving, release build.
+
+### Building mmtk-julia from source
+
+It is also possible to build the binding from source. To do so, set the variable `USE_BINARYBUILDER_MMTK_JULIA=0` and the latest release version of the binding will be downloaded and built as part of building Julia. Note that this requires an installation of the rust toolchain.
+
+It is also possible to build a custom version of binding by checking it out from the [git repository](https://github.com/mmtk/mmtk-julia) and setting a variable named `MMTK_JULIA_DIR` as the path that contains the binding.
+
+For more information on building Julia with MMTk, please refer to the [README](https://github.com/mmtk/mmtk-julia/blob/master/README.md) file in the binding repo.
+
+### I've got a build error when building Julia with MMTk, what should I do?
+
+If you try to build Julia with MMTk and get an error it is likely due to a change to Julia that has not been yet propagated to the binding or to the code in Julia that is specific to MMTk. Some changes include:
+
+(1) **Changing the memory layout of objects in Julia**. The binding relies on automatically generated Rust FFI bindings from Julia code. These files are generated using a crate named [`rust-bindgen`](https://github.com/rust-lang/rust-bindgen). To regenerate those files, check out the latest version of the `mmtk-julia` binding, set the variable `JULIA_PATH` to the path of the Julia version you are trying to build and run `make regen-bindgen-ffi` from the directory containing the binding. This should delete the current version of the FFI bindings and generate a new version based on the Julia code from `JULIA_PATH`.
+
+(2) **Changing the root objects passed to the GC**. Julia passes a set of objects to the GC as roots in the function [gc_mark_roots](https://github.com/JuliaLang/julia/blob/fbe865657942da7d73cc02f76064f9ba9cdef56c/src/gc-stock.c#L2846). At the moment, this set needs to be consistent between both the Stock GC and MMTk (in the function [`jl_gc_scan_vm_specific_roots`](https://github.com/JuliaLang/julia/blob/fbe865657942da7d73cc02f76064f9ba9cdef56c/src/gc-mmtk.c#L496)).
+
+(3) **Changing how objects are scanned**. MMTk uses the same strategy to find references in Julia objects as the stock GC (see [gc_mark_outrefs](https://github.com/JuliaLang/julia/blob/fbe865657942da7d73cc02f76064f9ba9cdef56c/src/gc-stock.c#L2227C19-L2227C34)). Changing the logic from this function should be reflected in the Rust code in the binding that [scan Julia objects](https://github.com/mmtk/mmtk-julia/blob/c9e046baf3a0d52fe75d6c8b28f6afd69b045d95/mmtk/src/julia_scanning.rs#L68).
+
+If your case is not included in one of the alternatives above, please create an issue in the Julia repository tagging it with the `GC: MMTK` label.

src/Makefile

@@ -29,7 +29,7 @@ ifeq ($(USECLANG),1)
 FLAGS += -Wno-return-type-c-linkage -Wno-atomic-alignment
 endif
 
-ifneq (${MMTK_PLAN},None)
+ifeq (${USE_THIRD_PARTY_GC},mmtk)
 FLAGS += -I$(MMTK_API_INC)
 endif
 
@@ -46,7 +46,7 @@ endif
 
 # GC source code. It depends on which GC implementation to use.
 GC_SRCS := gc-common gc-stacks gc-alloc-profiler gc-heap-snapshot
-ifneq (${MMTK_PLAN},None)
+ifeq (${USE_THIRD_PARTY_GC},mmtk)
 GC_SRCS += gc-mmtk
 else
 GC_SRCS += gc-stock gc-debug gc-pages gc-page-profiler
@@ -67,7 +67,7 @@ CG_LLVMLINK :=
 ifeq ($(JULIACODEGEN),LLVM)
 # Currently these files are used by both GCs. But we should make the list specific to stock, and MMTk should have its own implementation.
 GC_CODEGEN_SRCS := llvm-final-gc-lowering llvm-late-gc-lowering llvm-gc-invariant-verifier
-ifneq (${MMTK_PLAN},None)
+ifeq (${USE_THIRD_PARTY_GC},mmtk)
 FLAGS += -I$(MMTK_API_INC)
 GC_CODEGEN_SRCS += llvm-late-gc-lowering-mmtk
 else
@@ -122,7 +122,7 @@ UV_HEADERS += uv.h
 UV_HEADERS += uv/*.h
 endif
 PUBLIC_HEADERS := $(BUILDDIR)/julia_version.h $(wildcard $(SRCDIR)/support/*.h) $(addprefix $(SRCDIR)/,work-stealing-queue.h gc-interface.h gc-tls-common.h julia.h julia_assert.h julia_threads.h julia_fasttls.h julia_locks.h julia_atomics.h jloptions.h)
-ifneq (${MMTK_PLAN},None)
+ifeq (${USE_THIRD_PARTY_GC},mmtk)
 	PUBLIC_HEADERS += $(addprefix $(SRCDIR)/,gc-tls-mmtk.h gc-wb-mmtk.h)
 else
 	PUBLIC_HEADERS += $(addprefix $(SRCDIR)/,gc-tls-stock.h gc-wb-stock.h)
@@ -191,8 +191,14 @@ LIBJULIA_PATH_REL := libjulia
 endif
 
 COMMON_LIBPATHS := -L$(build_libdir) -L$(build_shlibdir)
-RT_LIBS := $(WHOLE_ARCHIVE) $(LIBUV) $(WHOLE_ARCHIVE) $(LIBUTF8PROC) $(NO_WHOLE_ARCHIVE) $(LIBUNWIND) $(RT_LLVMLINK) $(OSLIBS) $(LIBTRACYCLIENT) $(LIBITTAPI) $(MMTK_LIB)
-CG_LIBS := $(LIBUNWIND) $(CG_LLVMLINK) $(OSLIBS) $(LIBTRACYCLIENT) $(LIBITTAPI) $(MMTK_LIB)
+RT_LIBS := $(WHOLE_ARCHIVE) $(LIBUV) $(WHOLE_ARCHIVE) $(LIBUTF8PROC) $(NO_WHOLE_ARCHIVE) $(LIBUNWIND) $(RT_LLVMLINK) $(OSLIBS) $(LIBTRACYCLIENT) $(LIBITTAPI)
+CG_LIBS := $(LIBUNWIND) $(CG_LLVMLINK) $(OSLIBS) $(LIBTRACYCLIENT) $(LIBITTAPI)
+
+ifeq (${USE_THIRD_PARTY_GC},mmtk)
+RT_LIBS += $(MMTK_LIB)
+CG_LIBS += $(MMTK_LIB)
+endif
+
 RT_DEBUG_LIBS := $(COMMON_LIBPATHS) $(WHOLE_ARCHIVE) $(BUILDDIR)/flisp/libflisp-debug.a $(WHOLE_ARCHIVE) $(BUILDDIR)/support/libsupport-debug.a -ljulia-debug $(RT_LIBS)
 CG_DEBUG_LIBS := $(COMMON_LIBPATHS) $(CG_LIBS) -ljulia-debug -ljulia-internal-debug
 RT_RELEASE_LIBS := $(COMMON_LIBPATHS) $(WHOLE_ARCHIVE) $(BUILDDIR)/flisp/libflisp.a $(WHOLE_ARCHIVE) $(BUILDDIR)/support/libsupport.a -ljulia $(RT_LIBS)

src/julia.h

@@ -1251,11 +1251,17 @@ JL_DLLEXPORT JL_CONST_FUNC jl_gcframe_t **(jl_get_pgcstack)(void) JL_GLOBALLY_RO
 STATIC_INLINE jl_value_t *jl_genericmemory_owner(jl_genericmemory_t *m JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT;
 
 // write barriers
-#ifndef MMTK_GC
+
+#ifndef WITH_THIRD_PARTY_HEAP
 #include "gc-wb-stock.h"
 #else
+// Pick the appropriate third-party implementation
+#ifdef WITH_THIRD_PARTY_HEAP
+#if WITH_THIRD_PARTY_HEAP == 1 // MMTk
 #include "gc-wb-mmtk.h"
 #endif
+#endif
+#endif
 
 /*
   how - allocation style

src/julia_threads.h

@@ -4,11 +4,16 @@
 #ifndef JL_THREADS_H
 #define JL_THREADS_H
 
-#ifndef MMTK_GC
+#ifndef WITH_THIRD_PARTY_HEAP
 #include "gc-tls-stock.h"
 #else
+// Pick the appropriate third-party implementation
+#ifdef WITH_THIRD_PARTY_HEAP
+#if WITH_THIRD_PARTY_HEAP == 1 // MMTk
 #include "gc-tls-mmtk.h"
 #endif
+#endif
+#endif
 #include "gc-tls-common.h"
 #include "julia_atomics.h"
 #ifndef _OS_WINDOWS_