Merge bitcoin/bitcoin#30201: depends: remove FORCE_USE_SYSTEM_CLANG

7cbfd7a refactor: rename (macho) ld64 to lld (fanquake)
d851451 ci: update deps for macOS cross build (fanquake)
9ebdd5e depends: update install docs for macOS cross compilation (fanquake)
fb74fd6 depends: remove no-longer used llvm_* vars from macOS build (fanquake)
9043f12 depends: no-longer pass -B to clang in macOS cross-compile (fanquake)
f9994b0 depends: remove native LLVM package (fanquake)
e9a44fa depends: remove FORCE_USE_SYSTEM_CLANG (fanquake)
9946618 guix: use clang-toolchain-18 for macOS build (fanquake)

Pull request description:

  Remove `FORCE_USE_SYSTEM_CLANG` in favour of always using the system Clang and lld for macOS cross-compilation; rather than downloading precompiled blobs.

  For example, anyone using Ubuntu 24.04 should be able to `apt install clang llvm lld .. etc`, and then cross-compile for macOS using:
  ```bash
  # clang --version
  Ubuntu clang version 18.1.3 (1)
  make -C depends HOST=arm64-apple-darwin FORCE_USE_SYSTEM_CLANG=1
  ./autogen.sh
  CONFIG_SITE=/path/to/depends/arm64-apple-darwin/share/config.site ./configure
  make
  # file src/qt/bitcoin-qt
  src/qt/bitcoin-qt: Mach-O 64-bit arm64 executable, flags:<NOUNDEFS|DYLDLINK|TWOLEVEL|WEAK_DEFINES|BINDS_TO_WEAK|PIE|HAS_TLV_DESCRIPTORS>
  ```

  Note that the minimum supported version of Clang we will support for macOS cross-compilation will likely be more recent than our other minimum supported version of Clang, due to compiler/linker option usage.

ACKs for top commit:
  Sjors:
    tACK 7cbfd7a
  theuni:
    ACK 7cbfd7a
  TheCharlatan:
    Nice, ACK 7cbfd7a

Tree-SHA512: 1499e29b3b238c5c85278c38e8fb6bb5e7883db3443f62b6bf397c5d761bedbc054962be645a9defce15266f0a969bb4b3ccd28b6e4dd874472857b928f185d1

Author: fanquake

ci/test/00_setup_env_mac_cross.sh

@@ -9,9 +9,9 @@ export LC_ALL=C.UTF-8
 export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks}
 
 export CONTAINER_NAME=ci_macos_cross
-export CI_IMAGE_NAME_TAG="docker.io/ubuntu:22.04"
+export CI_IMAGE_NAME_TAG="docker.io/ubuntu:24.04"
 export HOST=x86_64-apple-darwin
-export PACKAGES="zip"
+export PACKAGES="clang lld llvm zip"
 export XCODE_VERSION=15.0
 export XCODE_BUILD_ID=15A240d
 export RUN_UNIT_TESTS=false

contrib/devtools/symbol-check.py

@@ -239,8 +239,8 @@ def check_MACHO_sdk(binary) -> bool:
         return True
     return False
 
-def check_MACHO_ld64(binary) -> bool:
-    if binary.build_version.tools[0].version == [17, 0, 6]:
+def check_MACHO_lld(binary) -> bool:
+    if binary.build_version.tools[0].version == [18, 1, 6]:
         return True
     return False
 
@@ -282,7 +282,7 @@ def check_ELF_ABI(binary) -> bool:
     ('DYNAMIC_LIBRARIES', check_MACHO_libraries),
     ('MIN_OS', check_MACHO_min_os),
     ('SDK', check_MACHO_sdk),
-    ('LD64', check_MACHO_ld64),
+    ('LLD', check_MACHO_lld),
 ],
 lief.EXE_FORMATS.PE: [
     ('DYNAMIC_LIBRARIES', check_PE_libraries),

contrib/guix/libexec/build.sh

@@ -178,8 +178,7 @@ make -C depends --jobs="$JOBS" HOST="$HOST" \
                                    x86_64_linux_AR=x86_64-linux-gnu-gcc-ar \
                                    x86_64_linux_RANLIB=x86_64-linux-gnu-gcc-ranlib \
                                    x86_64_linux_NM=x86_64-linux-gnu-gcc-nm \
-                                   x86_64_linux_STRIP=x86_64-linux-gnu-strip \
-                                   FORCE_USE_SYSTEM_CLANG=1
+                                   x86_64_linux_STRIP=x86_64-linux-gnu-strip
 
 
 ###########################

contrib/guix/manifest.scm

@@ -532,9 +532,9 @@ inspecting signatures in Mach-O binaries.")
           ((string-contains target "darwin")
            (list ;; Native GCC 11 toolchain
                  gcc-toolchain-11
-                 clang-toolchain-17
-                 lld-17
-                 (make-lld-wrapper lld-17 #:lld-as-ld? #t)
+                 clang-toolchain-18
+                 lld-18
+                 (make-lld-wrapper lld-18 #:lld-as-ld? #t)
                  python-signapple
                  zip))
           (else '())))))

depends/Makefile

@@ -181,8 +181,6 @@ all_packages = $(packages) $(native_packages)
 
 meta_depends = Makefile config.guess config.sub funcs.mk builders/default.mk hosts/default.mk hosts/$(host_os).mk builders/$(build_os).mk
 
-$(host_arch)_$(host_os)_native_toolchain?=$($(host_os)_native_toolchain)
-
 include funcs.mk
 
 final_build_id_long+=$(shell $(build_SHA256SUM) config.site.in)

depends/README.md

@@ -49,11 +49,11 @@ The paths are automatically configured and no other options are needed.
 
 #### For macOS cross compilation
 
-    sudo apt-get install g++ zip
+    apt install clang lld llvm g++ zip
 
-Note: You must obtain the macOS SDK before proceeding with a cross-compile.
-Under the depends directory, create a subdirectory named `SDKs`.
-Then, place the extracted SDK under this new directory.
+Clang 18 or later is required. You must also obtain the macOS SDK before
+proceeding with a cross-compile. Under the depends directory, create a
+subdirectory named `SDKs`. Then, place the extracted SDK under this new directory.
 For more information, see [SDK Extraction](../contrib/macdeploy/README.md#sdk-extraction).
 
 #### For Win64 cross compilation
@@ -119,9 +119,6 @@ The following can be set when running make: `make FOO=bar`
 - `DEBUG`: Disable some optimizations and enable more runtime checking
 - `HOST_ID_SALT`: Optional salt to use when generating host package ids
 - `BUILD_ID_SALT`: Optional salt to use when generating build package ids
-- `FORCE_USE_SYSTEM_CLANG`: (EXPERTS ONLY) When cross-compiling for macOS, use Clang found in the
-  system's `$PATH` rather than the default prebuilt release of Clang
-  from llvm.org. Clang 8 or later is required
 - `LOG`: Use file-based logging for individual packages. During a package build its log file
   resides in the `depends` directory, and the log file is printed out automatically in case
   of build error. After successful build log files are moved along with package archives

depends/builders/darwin.mk

@@ -18,7 +18,6 @@ darwin_STRIP:=$(shell xcrun -f strip)
 darwin_OBJDUMP:=$(shell xcrun -f objdump)
 darwin_NM:=$(shell xcrun -f nm)
 darwin_DSYMUTIL:=$(shell xcrun -f dsymutil)
-darwin_native_toolchain=
 
 x86_64_darwin_CFLAGS += -arch x86_64
 x86_64_darwin_CXXFLAGS += -arch x86_64

depends/funcs.mk

@@ -46,7 +46,7 @@ endef
 
 define int_get_build_id
 $(eval $(1)_dependencies += $($(1)_$(host_arch)_$(host_os)_dependencies) $($(1)_$(host_os)_dependencies))
-$(eval $(1)_all_dependencies:=$(call int_get_all_dependencies,$(1),$($($(1)_type)_native_toolchain) $($(1)_dependencies)))
+$(eval $(1)_all_dependencies:=$(call int_get_all_dependencies,$(1),$($(1)_dependencies)))
 $(foreach dep,$($(1)_all_dependencies),$(eval $(1)_build_id_deps+=$(dep)-$($(dep)_version)-$($(dep)_recipe_hash)))
 $(eval $(1)_build_id_long:=$(1)-$($(1)_version)-$($(1)_recipe_hash)-$(release_type) $($(1)_build_id_deps) $($($(1)_type)_id))
 $(eval $(1)_build_id:=$(shell echo -n "$($(1)_build_id_long)" | $(build_SHA256SUM) | cut -c-$(HASH_LENGTH)))
@@ -297,6 +297,3 @@ $(foreach package,$(all_packages),$(eval $(call int_config_attach_build_config,$
 
 #create build targets
 $(foreach package,$(all_packages),$(eval $(call int_add_cmds,$(package))))
-
-#special exception: if a toolchain package exists, all non-native packages depend on it
-$(foreach package,$(packages),$(eval $($(package)_extracted): |$($($(host_arch)_$(host_os)_native_toolchain)_cached) ))

depends/hosts/darwin.mk

@@ -2,39 +2,10 @@ OSX_MIN_VERSION=11.0
 OSX_SDK_VERSION=14.0
 XCODE_VERSION=15.0
 XCODE_BUILD_ID=15A240d
-LD64_VERSION=711
+LLD_VERSION=711
 
 OSX_SDK=$(SDK_PATH)/Xcode-$(XCODE_VERSION)-$(XCODE_BUILD_ID)-extracted-SDK-with-libcxx-headers
 
-ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
-# FORCE_USE_SYSTEM_CLANG is empty, so we use our depends-managed, pinned LLVM
-# from llvm.org
-
-darwin_native_toolchain=native_llvm
-
-clang_prog=$(build_prefix)/bin/clang
-clangxx_prog=$(clang_prog)++
-llvm_config_prog=$(build_prefix)/bin/llvm-config
-
-llvm_TOOLS=AR NM OBJDUMP RANLIB STRIP
-
-# Make-only lowercase function
-lc = $(subst A,a,$(subst B,b,$(subst C,c,$(subst D,d,$(subst E,e,$(subst F,f,$(subst G,g,$(subst H,h,$(subst I,i,$(subst J,j,$(subst K,k,$(subst L,l,$(subst M,m,$(subst N,n,$(subst O,o,$(subst P,p,$(subst Q,q,$(subst R,r,$(subst S,s,$(subst T,t,$(subst U,u,$(subst V,v,$(subst W,w,$(subst X,x,$(subst Y,y,$(subst Z,z,$1))))))))))))))))))))))))))
-
-# For well-known tools provided by LLVM, make sure that their well-known
-# variable is set to the full path of the tool, just like how AC_PATH_{TOO,PROG}
-# would.
-$(foreach TOOL,$(llvm_TOOLS),$(eval darwin_$(TOOL) = $$(build_prefix)/bin/llvm-$(call lc,$(TOOL))))
-
-# Clang expects dsymutil to be called dsymutil
-darwin_DSYMUTIL=$(build_prefix)/bin/dsymutil
-
-else
-# FORCE_USE_SYSTEM_CLANG is non-empty, so we use the clang from the user's
-# system
-
-darwin_native_toolchain=
-
 # We can't just use $(shell command -v clang) because GNU Make handles builtins
 # in a special way and doesn't know that `command` is a POSIX-standard builtin
 # prior to 1af314465e5dfe3e8baa839a32a72e83c04f26ef, first released in v4.2.90.
@@ -44,17 +15,13 @@ darwin_native_toolchain=
 # Source: https://lists.gnu.org/archive/html/bug-make/2017-11/msg00017.html
 clang_prog=$(shell $(SHELL) $(.SHELLFLAGS) "command -v clang")
 clangxx_prog=$(shell $(SHELL) $(.SHELLFLAGS) "command -v clang++")
-llvm_config_prog=$(shell $(SHELL) $(.SHELLFLAGS) "command -v llvm-config")
-
-llvm_lib_dir=$(shell $(llvm_config_prog) --libdir)
 
 darwin_AR=$(shell $(SHELL) $(.SHELLFLAGS) "command -v llvm-ar")
 darwin_DSYMUTIL=$(shell $(SHELL) $(.SHELLFLAGS) "command -v dsymutil")
 darwin_NM=$(shell $(SHELL) $(.SHELLFLAGS) "command -v llvm-nm")
 darwin_OBJDUMP=$(shell $(SHELL) $(.SHELLFLAGS) "command -v llvm-objdump")
 darwin_RANLIB=$(shell $(SHELL) $(.SHELLFLAGS) "command -v llvm-ranlib")
 darwin_STRIP=$(shell $(SHELL) $(.SHELLFLAGS) "command -v llvm-strip")
-endif
 
 # Flag explanations:
 #
@@ -63,11 +30,6 @@ endif
 #         Ensures that modern linker features are enabled. See here for more
 #         details: https://github.com/bitcoin/bitcoin/pull/19407.
 #
-#     -B$(build_prefix)/bin
-#
-#         Explicitly point to our binaries so that they are
-#         ensured to be found and preferred over other possibilities.
-#
 #     -isysroot$(OSX_SDK) -nostdlibinc
 #
 #         Disable default include paths built into the compiler as well as
@@ -92,15 +54,13 @@ darwin_CC=env -u C_INCLUDE_PATH -u CPLUS_INCLUDE_PATH \
               -u OBJC_INCLUDE_PATH -u OBJCPLUS_INCLUDE_PATH -u CPATH \
               -u LIBRARY_PATH \
               $(clang_prog) --target=$(host) \
-              -B$(build_prefix)/bin \
               -isysroot$(OSX_SDK) -nostdlibinc \
               -iwithsysroot/usr/include -iframeworkwithsysroot/System/Library/Frameworks
 
 darwin_CXX=env -u C_INCLUDE_PATH -u CPLUS_INCLUDE_PATH \
                -u OBJC_INCLUDE_PATH -u OBJCPLUS_INCLUDE_PATH -u CPATH \
                -u LIBRARY_PATH \
                $(clangxx_prog) --target=$(host) \
-               -B$(build_prefix)/bin \
                -isysroot$(OSX_SDK) -nostdlibinc \
                -iwithsysroot/usr/include/c++/v1 \
                -iwithsysroot/usr/include -iframeworkwithsysroot/System/Library/Frameworks
@@ -110,8 +70,8 @@ darwin_CXXFLAGS=-pipe -std=$(CXX_STANDARD) -mmacosx-version-min=$(OSX_MIN_VERSIO
 darwin_LDFLAGS=-Wl,-platform_version,macos,$(OSX_MIN_VERSION),$(OSX_SDK_VERSION)
 
 ifneq ($(build_os),darwin)
-darwin_CFLAGS += -mlinker-version=$(LD64_VERSION)
-darwin_CXXFLAGS += -mlinker-version=$(LD64_VERSION)
+darwin_CFLAGS += -mlinker-version=$(LLD_VERSION)
+darwin_CXXFLAGS += -mlinker-version=$(LLD_VERSION)
 darwin_LDFLAGS += -Wl,-no_adhoc_codesign -fuse-ld=lld
 endif