Refactor how compiler-rt is managed in CI and locally (#591)

Building the PIC version of libc.a is currently broken with LLVM 19+,
and it's due to how this repository handles the compiler-rt library.
This PR refactors the handling to automatically download an appropriate
copy and handle it internally or otherwise use a `BUILTINS_LIB` from the
environment. This means that it's now always available for `libc.so`
which is needed to fix the build issue with LLVM 19+

Author: alexcrichton

.github/actions/setup/action.yml

@@ -53,7 +53,7 @@ runs:
         set -ex
         v=${{ inputs.clang_version }}
         sudo apt-get update
-        sudo apt-get install -y clang-$v clang-tools-$v
+        sudo apt-get install -y clang-$v clang-tools-$v lld-$v
         echo "CC=clang-$v" >> $GITHUB_ENV
         echo "AR=llvm-ar-$v" >> $GITHUB_ENV
         echo "NM=llvm-nm-$v" >> $GITHUB_ENV

.github/workflows/main.yml

@@ -38,13 +38,25 @@ jobs:
             clang_version: 16.0.0
             upload: windows-clang-16
 
-          # Historical versions of LLVM (11.0.0 currently)
+          # Other versions of LLVM
           - name: Build with LLVM 11
             os: ubuntu-22.04
             clang_version: 11
             upload: linux-x86_64-clang-11
             env:
               BUILD_LIBSETJMP: no
+          - name: Build with LLVM 19
+            os: ubuntu-24.04
+            clang_version: 19
+            upload: linux-x86_64-clang-19
+            env:
+              MAKE_TARGETS: "default libc_so"
+          - name: Build with LLVM 18
+            os: ubuntu-24.04
+            clang_version: 18
+            upload: linux-x86_64-clang-18
+            env:
+              MAKE_TARGETS: "default libc_so"
 
           # Test various combinations of targets triples.
           #
@@ -145,21 +157,7 @@ jobs:
 
     - name: Test
       if: matrix.test
-      # For Clang linking to work correctly, we need to place Clang's runtime
-      # library for `wasm32-wasi` in the right location (i.e., the `mkdir` and
-      # `cp` below).
-      run: |
-        cd test
-        mkdir resource-dir
-        export WASI_DIR=./resource-dir/lib/wasi/
-        export WASIP1_DIR=./resource-dir/lib/wasip1/
-        export WASIP2_DIR=./resource-dir/lib/wasip2/
-        mkdir -p $WASI_DIR $WASIP1_DIR $WASIP2_DIR
-        cp build/download/libclang_rt.builtins-wasm32.a $WASI_DIR
-        cp build/download/libclang_rt.builtins-wasm32.a $WASIP1_DIR
-        cp build/download/libclang_rt.builtins-wasm32.a $WASIP2_DIR
-        export LDFLAGS="-resource-dir $(pwd)/resource-dir"
-        make test
+      run: make -C test test
 
     - uses: actions/[email protected]
       if: matrix.upload

Makefile

@@ -556,11 +556,48 @@ PIC_OBJS = \
 	$(LIBC_BOTTOM_HALF_CRT_OBJS) \
 	$(FTS_SO_OBJS)
 
+# Figure out what to do about compiler-rt.
+#
+# The compiler-rt library is not built here in the wasi-libc repository, but it
+# is required to link artifacts. Notably `libc.so` and test and such all require
+# it to exist. Currently the ways this is handled are:
+#
+# * If `BUILTINS_LIB` is defined at build time then that's assumed to be a path
+#   to the libcompiler-rt.a. That's then ingested into the build here and copied
+#   around to special locations to get the `*.so` rules below to work (see docs
+#   there).
+#
+# * If `BUILTINS_LIB` is not defined then a known-good copy is downloaded from
+#   wasi-sdk CI and used instead.
+#
+# In the future this may also want some form of configuration to support
+# assuming the system compiler has a compiler-rt, e.g. if $(SYSTEM_BUILTINS_LIB)
+# exists that should be used instead.
 SYSTEM_BUILTINS_LIB := $(shell ${CC} ${CFLAGS} --print-libgcc-file-name)
 SYSTEM_RESOURCE_DIR := $(shell ${CC} ${CFLAGS} -print-resource-dir)
 BUILTINS_LIB_REL := $(subst $(SYSTEM_RESOURCE_DIR),,$(SYSTEM_BUILTINS_LIB))
-RESOURCE_DIR := $(OBJDIR)/resource-dir
-BUILTINS_LIB ?= $(RESOURCE_DIR)/$(BUILTINS_LIB_REL)
+TMP_RESOURCE_DIR := $(OBJDIR)/resource-dir
+BUILTINS_LIB_PATH := $(TMP_RESOURCE_DIR)/$(BUILTINS_LIB_REL)
+BUILTINS_LIB_DIR := $(dir $(BUILTINS_LIB_PATH))
+
+ifneq ($(BUILTINS_LIB),)
+$(BUILTINS_LIB_PATH): $(BUILTINS_LIB)
+	mkdir -p $(BUILTINS_LIB_DIR)
+	cp $(BUILTINS_LIB) $(BUILTINS_LIB_PATH)
+else
+
+BUILTINS_URL := https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-25/libclang_rt.builtins-wasm32-wasi-25.0.tar.gz
+
+$(BUILTINS_LIB_PATH):
+	mkdir -p $(BUILTINS_LIB_DIR)
+	curl -sSfL $(BUILTINS_URL) | \
+		tar xzf - -C $(BUILTINS_LIB_DIR) --strip-components 1
+	if [ ! -f $(BUILTINS_LIB_PATH) ]; then \
+	  mv $(BUILTINS_LIB_DIR)/*.a $(BUILTINS_LIB_PATH); \
+	fi
+endif
+
+builtins: $(BUILTINS_LIB_PATH)
 
 # TODO: Specify SDK version, e.g. libc.so.wasi-sdk-21, as SO_NAME once `wasm-ld`
 # supports it.
@@ -570,22 +607,34 @@ BUILTINS_LIB ?= $(RESOURCE_DIR)/$(BUILTINS_LIB_REL)
 # to CC.  This is a workaround for a Windows command line size limitation.  See
 # the `%.a` rule below for details.
 
-# Note: libc.so is special because it shouldn't link to libc.so.
+# Note: libc.so is special because it shouldn't link to libc.so, and the
+# -nodefaultlibs flag here disables the default `-lc` logic that clang
+# has. Note though that this also disables linking of compiler-rt
+# libraries so that is explicitly passed in via `$(BUILTINS_LIB_PATH)`
+#
 # Note: --allow-undefined-file=linker-provided-symbols.txt is
 # a workaround for https://github.com/llvm/llvm-project/issues/103592
-$(SYSROOT_LIB)/libc.so: $(OBJDIR)/libc.so.a $(BUILTINS_LIB)
-	$(CC) $(EXTRA_CFLAGS) --target=${TARGET_TRIPLE} -nodefaultlibs \
+$(SYSROOT_LIB)/libc.so: $(OBJDIR)/libc.so.a $(BUILTINS_LIB_PATH)
+	$(CC) --target=${TARGET_TRIPLE} -nodefaultlibs \
 	-shared --sysroot=$(SYSROOT) \
-	-o $@ -Wl,--whole-archive $< -Wl,--no-whole-archive $(BUILTINS_LIB) \
+	-o $@ -Wl,--whole-archive $< -Wl,--no-whole-archive \
 	-Wl,--allow-undefined-file=linker-provided-symbols.txt \
-	-resource-dir $(RESOURCE_DIR)
-
+	$(BUILTINS_LIB_PATH) \
+	$(EXTRA_CFLAGS) $(LDFLAGS)
+
+# Note that unlike `libc.so` above this rule does not pass `-nodefaultlibs`
+# which means that libc will be linked by default. Additionally clang will try
+# to find, locate, and link compiler-rt. To get compiler-rt to work a
+# `-resource-dir` argument is passed to ensure that our custom
+# `TMP_RESOURCE_DIR` built here locally is used instead of the system directory
+# which may or may not already have compiler-rt.
 $(SYSROOT_LIB)/%.so: $(OBJDIR)/%.so.a $(SYSROOT_LIB)/libc.so
-	$(CC) $(EXTRA_CFLAGS) --target=${TARGET_TRIPLE} \
+	$(CC) --target=${TARGET_TRIPLE} \
 	-shared --sysroot=$(SYSROOT) \
 	-o $@ -Wl,--whole-archive $< -Wl,--no-whole-archive \
 	-Wl,--allow-undefined-file=linker-provided-symbols.txt \
-	-resource-dir $(RESOURCE_DIR)
+	-resource-dir $(TMP_RESOURCE_DIR) \
+	$(EXTRA_CFLAGS) $(LDFLAGS)
 
 $(OBJDIR)/libc.so.a: $(LIBC_SO_OBJS) $(MUSL_PRINTSCAN_LONG_DOUBLE_SO_OBJS)
 

README.md

@@ -40,11 +40,25 @@ To use the sysroot, use the `--sysroot=` option:
 
 to run the compiler using the newly built sysroot.
 
+## Other default libraries
+
 Note that Clang packages typically don't include cross-compiled builds of
 compiler-rt, libcxx, or libcxxabi, for `libclang_rt.builtins-wasm32.a`,
-`libc++.a`, or `libc++abi.a`, respectively, so they may not be usable without
-extra setup. This is one of the things [wasi-sdk] simplifies, as it includes
-cross-compiled builds of compiler-rt, `libc++.a`, and `libc++abi.a`.
+`libc++.a`, or `libc++abi.a`, respectively. This means that by default just
+passing `--sysroot` may not be enough as Clang will also look for some of these
+libraries, notably `libclang_rt.builtins-wasm32.a`, by default. This is one of
+the things [wasi-sdk] simplifies, as it includes cross-compiled builds of
+compiler-rt, `libc++.a`, and `libc++abi.a`.
+
+The build of wasi-libc itself may depend on compiler-rt, or
+`libclang_rt.builtins-wasm32.a`. This manifests in error messages such as
+`undefined symbol: __muloti4` for example. Specifically the PIC build of
+wasi-libc will require that Clang can locate `libclang_rt.builtins-wasm32.a`,
+meaning that `libc.so` requires this library to be present. To handle this the
+default behavior of the build system is to download the latest version of
+compiler-rt from [wasi-sdk] and use that for the build of `libc.so`. You can
+also set `BUILTINS_LIB` as a path to `libclang_rt.builtins-wasm32.a` as well to
+use that instead of downloading a version.
 
 ## Building in pthread support
 

test/Makefile

@@ -35,8 +35,6 @@ $(SYSROOT):
 
 LIBC_TEST_URL ?= https://github.com/bytecodealliance/libc-test
 LIBC_TEST = $(DOWNDIR)/libc-test
-LIBRT_URL ?= https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-25/libclang_rt.builtins-wasm32-wasi-25.0.tar.gz
-LIBRT = $(DOWNDIR)/libclang_rt.builtins-wasm32.a
 ARCH := $(shell uname -m)
 WASMTIME_URL ?= https://github.com/bytecodealliance/wasmtime/releases/download/v29.0.1/wasmtime-v29.0.1-$(ARCH)-linux.tar.xz
 WASMTIME = $(abspath $(DOWNDIR)/$(shell basename $(WASMTIME_URL) .tar.xz)/wasmtime)
@@ -51,12 +49,6 @@ $(DOWNDIR):
 $(LIBC_TEST): | $(DOWNDIR)
 	git clone --depth 1 $(LIBC_TEST_URL) $@
 
-# TODO: add install target to copy builtins library directly into a Clang
-# installation.
-$(LIBRT): | $(DOWNDIR)
-	wget --no-clobber --directory-prefix=$(DOWNDIR) $(LIBRT_URL)
-	tar --extract --file=$(DOWNDIR)/$(shell basename $(LIBRT_URL)) --strip-components=1 --directory=$(DOWNDIR)/
-
 $(WASMTIME): | $(DOWNDIR)
 	wget --no-clobber --directory-prefix=$(DOWNDIR) $(WASMTIME_URL)
 	tar --extract --file=$(DOWNDIR)/$(shell basename $(WASMTIME_URL)) --directory=$(DOWNDIR)/
@@ -69,7 +61,7 @@ $(ADAPTER): | $(DOWNDIR)
 	wget --no-clobber --directory-prefix=$(DOWNDIR) $(ADAPTER_URL)
 
 # Target to download all necessary dependencies.
-TO_DOWNLOAD = $(LIBC_TEST) $(LIBRT) $(WASMTIME)
+TO_DOWNLOAD = $(LIBC_TEST) $(WASMTIME)
 ifeq ($(TARGET_TRIPLE), wasm32-wasip2)
 TO_DOWNLOAD += $(ADAPTER) $(WASM_TOOLS)
 endif
@@ -133,6 +125,16 @@ ifneq ($(findstring -threads,$(TARGET_TRIPLE)),)
 CFLAGS += -pthread
 endif
 
+# Handle compiler-rt which is required for tests. This is done by requesting
+# that the parent directory, the main wasi-libc directory, fetch its compiler-rt
+# which will create a `resource-dir` argument which we can then add to LDFLAGS
+# which gets fed down below into the actual linking of wasms.
+LDFLAGS += -resource-dir ../build/$(TARGET_TRIPLE)/resource-dir
+BUILTINS_STAMP := $(OBJDIR)/builtins.stamp
+$(BUILTINS_STAMP):
+	make -C .. builtins
+	touch $@
+
 # Build up all the `*.wasm.o` object files; these are the same regardless of
 # whether we're building core modules or components.
 $(WASM_OBJS): $(INFRA_HEADERS)
@@ -142,7 +144,7 @@ $(OBJDIR)/%.wasm.o: $(SRCDIR)/%.c $(DOWNLOADED) $(SYSROOT)
 
 # Build up all the `*.wasm` files.
 obj_to_c = $(patsubst $(OBJDIR)/%.wasm.o,$(SRCDIR)/%.c,$1)
-$(OBJDIR)/%.core.wasm: $(OBJDIR)/%.wasm.o $(INFRA_WASM_OBJS)
+$(OBJDIR)/%.core.wasm: $(OBJDIR)/%.wasm.o $(INFRA_WASM_OBJS) | $(BUILTINS_STAMP)
 	@mkdir -p $(@D)
 	$(CC) $(CFLAGS) $(LDFLAGS) $(shell scripts/add-flags.py LDFLAGS $(call obj_to_c,$<)) $^ -o $@