backport: merge bitcoin#24241, #24534, #24948, #28622, #28880, #29185, #29170, #29233, #29298, #29598, #29732, #29890, #29739, #30074, #30198, #29072 (toolchain backports: part 2)

.gitlab-ci.yml

@@ -42,8 +42,8 @@ builder-image:
   image: $CI_REGISTRY_IMAGE:builder-$CI_COMMIT_REF_SLUG
   variables:
     SDK_URL: https://bitcoincore.org/depends-sources/sdks
-    XCODE_VERSION: "12.2"
-    XCODE_BUILD_ID: 12B45b
+    XCODE_VERSION: "15.0"
+    XCODE_BUILD_ID: 15A240d
   before_script:
     - echo HOST=$HOST
     - |

Makefile.am

@@ -130,7 +130,7 @@ $(OSX_DMG): deploydir
 	$(XORRISOFS) -D -l -V "$(OSX_VOLNAME)" -no-pad -r -dir-mode 0755 -o $@ $(APP_DIST_DIR) -- $(if $(SOURCE_DATE_EPOCH),-volume_date all_file_dates =$(SOURCE_DATE_EPOCH))
 
 $(APP_DIST_DIR)/$(OSX_APP)/Contents/MacOS/Dash-Qt: $(OSX_APP_BUILT) $(OSX_PACKAGING)
-	INSTALL_NAME_TOOL=$(INSTALL_NAME_TOOL) OTOOL=$(OTOOL) STRIP=$(STRIP) $(PYTHON) $(OSX_DEPLOY_SCRIPT) $(OSX_APP) $(OSX_VOLNAME) -translations-dir=$(QT_TRANSLATION_DIR)
+	OBJDUMP=$(OBJDUMP) $(PYTHON) $(OSX_DEPLOY_SCRIPT) $(OSX_APP) $(OSX_VOLNAME) -translations-dir=$(QT_TRANSLATION_DIR)
 
 deploydir: $(APP_DIST_DIR)/$(OSX_APP)/Contents/MacOS/Dash-Qt
 endif !BUILD_DARWIN

ci/test/00_setup_env_mac.sh

@@ -9,12 +9,9 @@ export LC_ALL=C.UTF-8
 export CONTAINER_NAME=ci_macos_cross
 export HOST=x86_64-apple-darwin
 export PACKAGES="cmake libz-dev python3-setuptools xorriso"
-export XCODE_VERSION=12.2
-export XCODE_BUILD_ID=12B45b
+export XCODE_VERSION=15.0
+export XCODE_BUILD_ID=15A240d
 export RUN_UNIT_TESTS=false
 export RUN_FUNCTIONAL_TESTS=false
 export GOAL="all deploy"
-
-# False-positive warning is fixed with clang 17, remove this when that version
-# can be used.
-export BITCOIN_CONFIG="--with-gui --enable-reduce-exports --disable-miner --with-boost-process LDFLAGS=-Wno-error=unused-command-line-argument"
+export BITCOIN_CONFIG="--with-gui --enable-reduce-exports --disable-miner --with-boost-process"

configure.ac

@@ -122,6 +122,7 @@ AC_PATH_PROG(CCACHE,ccache)
 AC_PATH_PROG(XGETTEXT,xgettext)
 AC_PATH_PROG(HEXDUMP,hexdump)
 AC_PATH_TOOL(OBJCOPY, objcopy)
+AC_PATH_TOOL([OBJDUMP], [objdump])
 AC_PATH_TOOL(DSYMUTIL, dsymutil)
 AC_PATH_PROG(DOXYGEN, doxygen)
 AM_CONDITIONAL([HAVE_DOXYGEN], [test -n "$DOXYGEN"])
@@ -345,11 +346,6 @@ AC_ARG_WITH([boost-process],
     [boost_process=$withval],
     [boost_process=no])
 
-AC_ARG_ENABLE([lto],
-    [AS_HELP_STRING([--enable-lto],[build using LTO (default is no)])],
-    [enable_lto=$enableval],
-    [enable_lto=no])
-
 AC_LANG_PUSH([C++])
 
 dnl Check for a flag to turn compiler warnings into errors. This is helpful for checks which may
@@ -408,11 +404,6 @@ else
     fi
 fi
 
-if test "x$enable_lto" = "xyes"; then
-  AX_CHECK_COMPILE_FLAG([-flto], [LTO_CXXFLAGS="$LTO_CXXFLAGS -flto"], [AC_MSG_ERROR([compile failed with -flto])], [$CXXFLAG_WERROR])
-  AX_CHECK_LINK_FLAG([-flto], [LTO_LDFLAGS="$LTO_LDFLAGS -flto"], [AC_MSG_ERROR([link failed with -flto])], [$CXXFLAG_WERROR])
-fi
-
 if test "x$enable_stacktraces" != xno; then
   AC_CHECK_HEADERS([execinfo.h], [], [enable_stacktraces=no])
 fi
@@ -780,6 +771,9 @@ case $host in
      TARGET_OS=darwin
      if  test x$cross_compiling != xyes; then
        BUILD_OS=darwin
+
+       AX_CHECK_LINK_FLAG([[-Wl,-headerpad_max_install_names]], [LDFLAGS="$LDFLAGS -Wl,-headerpad_max_install_names"],, [[$LDFLAG_WERROR]])
+
        AC_CHECK_PROG([BREW],brew, brew)
        if test x$BREW = xbrew; then
          dnl These Homebrew packages may be keg-only, meaning that they won't be found
@@ -843,8 +837,6 @@ case $host in
            ;;
          *)
            AC_PATH_TOOL([DSYMUTIL], [dsymutil], dsymutil)
-           AC_PATH_TOOL([INSTALL_NAME_TOOL], [install_name_tool], install_name_tool)
-           AC_PATH_TOOL([OTOOL], [otool], otool)
            AC_PATH_PROGS([XORRISOFS], [xorrisofs], xorrisofs)
 
            dnl libtool will try to strip the static lib, which is a problem for
@@ -856,7 +848,6 @@ case $host in
        esac
      fi
 
-     AX_CHECK_LINK_FLAG([[-Wl,-headerpad_max_install_names]], [LDFLAGS="$LDFLAGS -Wl,-headerpad_max_install_names"],, [[$LDFLAG_WERROR]])
      CPPFLAGS="$CPPFLAGS -DMAC_OSX -DOBJC_OLD_DISPATCH_PROTOTYPES=0"
      OBJCXXFLAGS="$CXXFLAGS"
      ;;
@@ -1529,6 +1520,7 @@ if test x$use_reduce_exports = xyes; then
     AX_CHECK_COMPILE_FLAG([-fvisibility=hidden],[CXXFLAGS="$CXXFLAGS -fvisibility=hidden"],
     [AC_MSG_ERROR([Cannot set hidden symbol visibility. Use --disable-reduce-exports.])],[[$CXXFLAG_WERROR]])
     AX_CHECK_LINK_FLAG([[-Wl,--exclude-libs,ALL]],[RELDFLAGS="-Wl,--exclude-libs,ALL"],,[[$LDFLAG_WERROR]])
+    AX_CHECK_LINK_FLAG([-Wl,-no_exported_symbols], [LIBTOOL_APP_LDFLAGS="$LIBTOOL_APP_LDFLAGS -Wl,-no_exported_symbols"], [], [$LDFLAG_WERROR])
 fi
 
 if test x$use_tests = xyes; then
@@ -1887,8 +1879,6 @@ AC_SUBST(GPROF_LDFLAGS)
 AC_SUBST(HARDENED_CXXFLAGS)
 AC_SUBST(HARDENED_CPPFLAGS)
 AC_SUBST(HARDENED_LDFLAGS)
-AC_SUBST(LTO_CXXFLAGS)
-AC_SUBST(LTO_LDFLAGS)
 AC_SUBST(PIC_FLAGS)
 AC_SUBST(PIE_FLAGS)
 AC_SUBST(SANITIZER_CXXFLAGS)
@@ -1999,7 +1989,6 @@ echo "  crash hooks enabled = $enable_crashhooks"
 echo "  miner enabled       = $enable_miner"
 echo "  gprof enabled       = $enable_gprof"
 echo "  werror              = $enable_werror"
-echo "  LTO                 = $enable_lto"
 echo
 echo "  target os           = $host_os"
 echo "  build os            = $build_os"
@@ -2008,7 +1997,7 @@ echo "  CC                  = $CC"
 echo "  CFLAGS              = $PTHREAD_CFLAGS $CFLAGS"
 echo "  CPPFLAGS            = $DEBUG_CPPFLAGS $HARDENED_CPPFLAGS $CPPFLAGS"
 echo "  CXX                 = $CXX"
-echo "  CXXFLAGS            = $LTO_CXXFLAGS $DEBUG_CXXFLAGS $HARDENED_CXXFLAGS $WARN_CXXFLAGS $NOWARN_CXXFLAGS $ERROR_CXXFLAGS $GPROF_CXXFLAGS $CXXFLAGS"
-echo "  LDFLAGS             = $LTO_LDFLAGS $PTHREAD_LIBS $HARDENED_LDFLAGS $GPROF_LDFLAGS $LDFLAGS"
+echo "  CXXFLAGS            = $DEBUG_CXXFLAGS $HARDENED_CXXFLAGS $WARN_CXXFLAGS $NOWARN_CXXFLAGS $ERROR_CXXFLAGS $GPROF_CXXFLAGS $CXXFLAGS"
+echo "  LDFLAGS             = $PTHREAD_LIBS $HARDENED_LDFLAGS $GPROF_LDFLAGS $LDFLAGS"
 echo "  ARFLAGS             = $ARFLAGS"
 echo

contrib/containers/guix/scripts/guix-start

@@ -9,8 +9,8 @@ if [[ ! -d "${WORKSPACE_PATH}" || ! "${WORKSPACE_PATH}" = /* || ! -f "${WORKSPAC
     exit 1
 fi
 
-XCODE_VERSION="12.2"
-XCODE_RELEASE="12B45b"
+XCODE_VERSION="15.0"
+XCODE_RELEASE="15A240d"
 XCODE_ARCHIVE="Xcode-${XCODE_VERSION}-${XCODE_RELEASE}-extracted-SDK-with-libcxx-headers"
 XCODE_SOURCE="${XCODE_SOURCE:-https://bitcoincore.org/depends-sources/sdks}"
 

contrib/devtools/security-check.py

@@ -193,6 +193,16 @@ def check_MACHO_control_flow(binary) -> bool:
         return True
     return False
 
+def check_MACHO_branch_protection(binary) -> bool:
+    '''
+    Check for branch protection instrumentation
+    '''
+    content = binary.get_content_from_virtual_address(binary.entrypoint, 4, lief.Binary.VA_TYPES.AUTO)
+
+    if content.tolist() == [95, 36, 3, 213]: # bti
+        return True
+    return False
+
 BASE_ELF = [
     ('PIE', check_PIE),
     ('NX', check_NX),
@@ -232,7 +242,7 @@ def check_MACHO_control_flow(binary) -> bool:
         lief.ARCHITECTURES.X86: BASE_MACHO + [('PIE', check_PIE),
                                               ('NX', check_NX),
                                               ('CONTROL_FLOW', check_MACHO_control_flow)],
-        lief.ARCHITECTURES.ARM64: BASE_MACHO,
+        lief.ARCHITECTURES.ARM64: BASE_MACHO + [('BRANCH_PROTECTION', check_MACHO_branch_protection)],
     }
 }
 

contrib/devtools/symbol-check.py

@@ -247,7 +247,7 @@ def check_MACHO_min_os(binary) -> bool:
     return False
 
 def check_MACHO_sdk(binary) -> bool:
-    if binary.build_version.sdk == [11, 0, 0]:
+    if binary.build_version.sdk == [14, 0, 0]:
         return True
     return False
 

contrib/devtools/test-security-check.py

@@ -28,22 +28,24 @@ def clean_files(source, executable):
     os.remove(source)
     os.remove(executable)
 
-def call_security_check(cc, source, executable, options):
+def env_flags() -> List[str]:
     # This should behave the same as AC_TRY_LINK, so arrange well-known flags
     # in the same order as autoconf would.
     #
     # See the definitions for ac_link in autoconf's lib/autoconf/c.m4 file for
     # reference.
-    env_flags: List[str] = []
+    flags: List[str] = []
     for var in ['CFLAGS', 'CPPFLAGS', 'LDFLAGS']:
-        env_flags += filter(None, os.environ.get(var, '').split(' '))
+        flags += filter(None, os.environ.get(var, '').split(' '))
+    return flags
 
-    subprocess.run([*cc,source,'-o',executable] + env_flags + options, check=True)
+def call_security_check(cc, source, executable, options):
+    subprocess.run([*cc,source,'-o',executable] + env_flags() + options, check=True)
     p = subprocess.run(['./contrib/devtools/security-check.py',executable], stdout=subprocess.PIPE, universal_newlines=True)
     return (p.returncode, p.stdout.rstrip())
 
 def get_arch(cc, source, executable):
-    subprocess.run([*cc, source, '-o', executable], check=True)
+    subprocess.run([*cc, source, '-o', executable] + env_flags(), check=True)
     binary = lief.parse(executable)
     arch = binary.abstract.header.architecture
     os.remove(executable)
@@ -138,12 +140,12 @@ def test_MACHO(self):
         else:
             # arm64 darwin doesn't support non-PIE binaries, control flow or executable stacks
             self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-flat_namespace','-fno-stack-protector', '-Wl,-no_fixup_chains']),
-                (1, executable+': failed NOUNDEFS Canary FIXUP_CHAINS'))
-            self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-flat_namespace','-fno-stack-protector', '-Wl,-fixup_chains']),
+                (1, executable+': failed NOUNDEFS Canary FIXUP_CHAINS BRANCH_PROTECTION'))
+            self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-flat_namespace','-fno-stack-protector', '-Wl,-fixup_chains', '-mbranch-protection=bti']),
                 (1, executable+': failed NOUNDEFS Canary'))
-            self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-flat_namespace','-fstack-protector-all', '-Wl,-fixup_chains']),
+            self.assertEqual(call_security_check(cc, source, executable, ['-Wl,-flat_namespace','-fstack-protector-all', '-Wl,-fixup_chains', '-mbranch-protection=bti']),
                 (1, executable+': failed NOUNDEFS'))
-            self.assertEqual(call_security_check(cc, source, executable, ['-fstack-protector-all', '-Wl,-fixup_chains']),
+            self.assertEqual(call_security_check(cc, source, executable, ['-fstack-protector-all', '-Wl,-fixup_chains', '-mbranch-protection=bti']),
                 (0, ''))
 
 

contrib/devtools/test-symbol-check.py

@@ -28,10 +28,6 @@ def call_symbol_check(cc: List[str], source, executable, options):
     os.remove(executable)
     return (p.returncode, p.stdout.rstrip())
 
-def get_machine(cc: List[str]):
-    p = subprocess.run([*cc,'-dumpmachine'], stdout=subprocess.PIPE, universal_newlines=True)
-    return p.stdout.rstrip()
-
 class TestSymbolChecks(unittest.TestCase):
     def test_ELF(self):
         source = 'test1.c'

contrib/guix/Dockerfile

@@ -59,5 +59,5 @@ RUN mkdir base_cache sources SDKs
 WORKDIR /dash
 
 RUN mkdir -p depends/SDKs && \
-    curl -L https://bitcoincore.org/depends-sources/sdks/Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers.tar.gz | tar -xz -C depends/SDKs
+    curl -L https://bitcoincore.org/depends-sources/sdks/Xcode-15.0-15A240d-extracted-SDK-with-libcxx-headers.tar.gz | tar -xz -C depends/SDKs
 

contrib/guix/manifest.scm

@@ -609,7 +609,7 @@ inspecting signatures in Mach-O binaries.")
            (list ;; Native GCC 11 toolchain
                  gcc-toolchain-11
                  binutils
-                 clang-toolchain-15
+                 clang-toolchain-17
                  python-signapple
                  xorriso))
           (else '())))))

contrib/macdeploy/README.md

@@ -12,48 +12,50 @@ When complete, it will have produced `Dash-Qt.dmg`.
 
 ### Step 1: Obtaining `Xcode.app`
 
-Our current macOS SDK
-(`Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers.tar.gz`) can be
-extracted from
-[Xcode_12.2.xip](https://download.developer.apple.com/Developer_Tools/Xcode_12.2/Xcode_12.2.xip).
+A free Apple Developer Account is required to proceed.
+
+Our macOS SDK can be extracted from
+[Xcode_15.xip](https://download.developer.apple.com/Developer_Tools/Xcode_15/Xcode_15.xip).
+
 Alternatively, after logging in to your account go to 'Downloads', then 'More'
-and look for [`Xcode_12.2`](https://download.developer.apple.com/Developer_Tools/Xcode_12.2/Xcode_12.2.xip).
+and search for [`Xcode 15`](https://developer.apple.com/download/all/?q=Xcode%2015).
+
 An Apple ID and cookies enabled for the hostname are needed to download this.
-The `sha256sum` of the archive should be `28d352f8c14a43d9b8a082ac6338dc173cb153f964c6e8fb6ba389e5be528bd0`.
 
-After Xcode version 7.x, Apple started shipping the `Xcode.app` in a `.xip`
-archive. This makes the SDK less-trivial to extract on non-macOS machines. One
-approach (tested on Debian Buster) is outlined below:
+The `sha256sum` of the downloaded XIP archive should be `4daaed2ef2253c9661779fa40bfff50655dc7ec45801aba5a39653e7bcdde48e`.
+
+To extract the `.xip` on Linux:
 
 ```bash
 # Install/clone tools needed for extracting Xcode.app
 apt install cpio
 git clone https://github.com/bitcoin-core/apple-sdk-tools.git
 
-# Unpack Xcode_12.2.xip and place the resulting Xcode.app in your current
+# Unpack the .xip and place the resulting Xcode.app in your current
 # working directory
-python3 apple-sdk-tools/extract_xcode.py -f Xcode_12.2.xip | cpio -d -i
+python3 apple-sdk-tools/extract_xcode.py -f Xcode_15.xip | cpio -d -i
 ```
 
-On macOS the process is more straightforward:
+On macOS:
 
 ```bash
-xip -x Xcode_12.2.xip
+xip -x Xcode_15.xip
 ```
 
-### Step 2: Generating `Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers.tar.gz` from `Xcode.app`
+### Step 2: Generating the SDK tarball from `Xcode.app`
 
-To generate `Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers.tar.gz`, run
-the script [`gen-sdk`](./gen-sdk) with the path to `Xcode.app` (extracted in the
-previous stage) as the first argument.
+To generate the SDK, run the script [`gen-sdk`](./gen-sdk) with the
+path to `Xcode.app` (extracted in the previous stage) as the first argument.
 
 ```bash
-# Generate a Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers.tar.gz from
-# the supplied Xcode.app
 ./contrib/macdeploy/gen-sdk '/path/to/Xcode.app'
 ```
 
+The generated archive should be: `Xcode-15.0-15A240d-extracted-SDK-with-libcxx-headers.tar.gz`.
+The `sha256sum` should be `c0c2e7bb92c1fee0c4e9f3a485e4530786732d6c6dd9e9f418c282aa6892f55d`.
+
 ## Deterministic macOS DMG Notes
+
 Working macOS DMGs are created in Linux by combining a recent `clang`, the Apple
 `binutils` (`ld`, `ar`, etc) and DMG authoring tools.
 
@@ -64,9 +66,8 @@ building for macOS.
 
 Apple's version of `binutils` (called `cctools`) contains lots of functionality missing in the
 FSF's `binutils`. In addition to extra linker options for frameworks and sysroots, several
-other tools are needed as well such as `install_name_tool`, `lipo`, and `nmedit`. These
-do not build under Linux, so they have been patched to do so. The work here was used as
-a starting point: [mingwandroid/toolchain4](https://github.com/mingwandroid/toolchain4).
+other tools are needed as well. These do not build under Linux, so they have been patched to
+do so. The work here was used as a starting point: [mingwandroid/toolchain4](https://github.com/mingwandroid/toolchain4).
 
 In order to build a working toolchain, the following source packages are needed from
 Apple: `cctools`, `dyld`, and `ld64`.
@@ -78,19 +79,9 @@ This version of `cctools` has been patched to use the current version of `clang`
 and its `libLTO.so` rather than those from `llvmgcc`, as it was originally done in `toolchain4`.
 
 To complicate things further, all builds must target an Apple SDK. These SDKs are free to
-download, but not redistributable. To obtain it, register for an Apple Developer Account,
-then download [Xcode_12.2](https://download.developer.apple.com/Developer_Tools/Xcode_12.2/Xcode_12.2.xip).
-
-This file is many gigabytes in size, but most (but not all) of what we need is
-contained only in a single directory:
-
-```bash
-Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk
-```
-
-See the SDK Extraction notes above for how to obtain it.
+download, but not redistributable. See the SDK Extraction notes above for how to obtain it.
 
-The Guix process build 2 sets of files: Linux tools, then Apple binaries which are
+The Guix process builds 2 sets of files: Linux tools, then Apple binaries which are
 created using these tools. The build process has been designed to avoid including the
 SDK's files in Guix's outputs. All interim tarballs are fully deterministic and may be freely
 redistributed.