Merge bitcoin/bitcoin#19817: build: macOS toolchain bump

a5550f8 build: use -stdlib++-isystem with Clang 10 (fanquake)
51d9d16 guix: use Clang 10 for the macOS cross compile (fanquake)
b80a6af build: no longer patch threading out of ld64 (fanquake)
c29cba4 build: Xcode 12.1, macOS SDK 10.15.6 (fanquake)
9ed2f19 build: native cctools 973.0.1, ld64 609 (fanquake)
f48f187 build: Clang 10.0.1 (Hennadii Stepanov)
9b193cd build: libtapi 1100.0.11 (fanquake)

Pull request description:

  Bumps our macOS toolchain to be using the following:
  * Clang 10.0.1 (gitian) & Clang 10.0.0 (Guix)
  * ld64 609
  * libtapi 1100.0.11
  * cctools  973.0.1
  * Xcode 12.1
  * macOS SDK 10.15.6

  which are currently the most recent releases available as open source. See upstream [`cctools`](https://github.com/tpoechtrager/cctools-port/commits/973.0.1-ld64-609) and [`libtapi`](https://github.com/tpoechtrager/apple-libtapi/tree/1100.0.11).

  This should improve the possibility of Apple ARM cross-compilation in depends.

  This also removes our [patching out of pthreads usage](https://github.com/bitcoin/bitcoin/blob/master/depends/patches/native_cctools/ld64_disable_threading.patch) in `ld64`. There have been multiple changes since `ld64 450.3`, which have likely fixed the non-determinism we were working around. i.e from [InputFiles.cpp](https://opensource.apple.com/source/ld64/ld64-609/src/ld/InputFiles.cpp.auto.html):
  ```cpp
  // <rdar://problem/15002251> make implicit dylib order be deterministic by sorting by install_name
  std::sort(implicitDylibs.begin(), implicitDylibs.end(), DylibByInstallNameSorter());
  ```

  ```cpp
  // <rdar://problem/42675402> ld64 output is not deterministic due to dylib processing order
  std::sort(unprocessedDylibs.begin(), unprocessedDylibs.end(), [](const ld::dylib::File* lhs, const ld::dylib::File* rhs) {
  return strcmp(lhs->path(), rhs->path()) < 0;
  });
  ```

  Guix Build:
  ```bash
  find guix-build-$(git rev-parse --short=12 HEAD)/output/ -type f -print0 | env LC_ALL=C sort -z | xargs -r0 sha256sum
  f6c3817b8fe5f7370299d1ae2533e4a3acd313ba9f9aa8d423a8956117e52dd5  guix-build-a5550f877a2c/output/dist-archive/bitcoin-a5550f877a2c.tar.gz
  4954dcf563c2d496b8d9fecd48f8e3f7fba2f319ffa254a5bc8ee12cfee6acf0  guix-build-a5550f877a2c/output/x86_64-apple-darwin18/bitcoin-a5550f877a2c-osx-unsigned.dmg
  8f6095b445c7f1a8e6accd86bb7f0696d5849402084927d2b726b7d557831c3a  guix-build-a5550f877a2c/output/x86_64-apple-darwin18/bitcoin-a5550f877a2c-osx-unsigned.tar.gz
  cc40f25477b4defc1617ae694313d80f307ddf6742fe6cc85c6bc0e215ef8be0  guix-build-a5550f877a2c/output/x86_64-apple-darwin18/bitcoin-a5550f877a2c-osx64.tar.gz
  ```

  Gitian Build:
  ```bash
  Generating report
  506a8abdefe559999b43dd9f14905b9b2b5a3363b1cd013d45ae47acc1f7ef6c  bitcoin-a5550f877a2c-osx-unsigned.dmg
  f606997f74026dd12d110d683c6f116b40df324836904ef507dd7ac787e6ebe2  bitcoin-a5550f877a2c-osx-unsigned.tar.gz
  5b495ef15f2c3260c2950921b61326912a9bf533cccd51e13818809fd225489e  bitcoin-a5550f877a2c-osx64.tar.gz
  f6c3817b8fe5f7370299d1ae2533e4a3acd313ba9f9aa8d423a8956117e52dd5  src/bitcoin-a5550f877a2c.tar.gz
  9eb0221e962d2839770963bd03c6c9e98e8bf3078566bee2ae42f06233a710fa  bitcoin-core-osx-22-res.yml
  Done.
  ```

ACKs for top commit:
  hebasto:
    ACK a5550f8

Tree-SHA512: 504c4b0f9cd3b939714a322298320c5bde07e9356a48a9a000060b36f8dce4d6134ed60c3a5188810476a28ec5b108733eabbc6fb8053231b9ea8a494cc91b12

Author: fanquake

ci/test/00_setup_env_mac.sh

@@ -10,8 +10,8 @@ export CONTAINER_NAME=ci_macos_cross
 export DOCKER_NAME_TAG=ubuntu:20.04  # Check that Focal can cross-compile to macos (Focal is used in the gitian build as well)
 export HOST=x86_64-apple-darwin18
 export PACKAGES="cmake imagemagick librsvg2-bin libz-dev libtiff-tools libtinfo5 python3-setuptools xorriso"
-export XCODE_VERSION=11.3.1
-export XCODE_BUILD_ID=11C505
+export XCODE_VERSION=12.1
+export XCODE_BUILD_ID=12A7403
 export RUN_UNIT_TESTS=false
 export RUN_FUNCTIONAL_TESTS=false
 export GOAL="deploy"

contrib/gitian-build.py

@@ -210,7 +210,7 @@ def main():
     args.macos = 'm' in args.os
 
     # Disable for MacOS if no SDK found
-    if args.macos and not os.path.isfile('gitian-builder/inputs/Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz'):
+    if args.macos and not os.path.isfile('gitian-builder/inputs/Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz'):
         print('Cannot build for MacOS, SDK does not exist. Will build for other OSes')
         args.macos = False
 

contrib/gitian-descriptors/gitian-osx.yml

@@ -31,7 +31,7 @@ remotes:
 - "url": "https://github.com/bitcoin/bitcoin.git"
   "dir": "bitcoin"
 files:
-- "Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz"
+- "Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz"
 script: |
   set -e -o pipefail
 
@@ -88,7 +88,7 @@ script: |
   BASEPREFIX="${PWD}/depends"
 
   mkdir -p ${BASEPREFIX}/SDKs
-  tar -C ${BASEPREFIX}/SDKs -xf ${BUILD_DIR}/Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz
+  tar -C ${BASEPREFIX}/SDKs -xf ${BUILD_DIR}/Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz
 
   # Build dependencies for each host
   for i in $HOSTS; do

contrib/guix/README.md

@@ -167,7 +167,7 @@ find output/ -type f -print0 | sort -z | xargs -r0 sha256sum
   Set the path where _extracted_ SDKs can be found. This is passed through to
   the depends tree. Note that this is should be set to the _parent_ directory of
   the actual SDK (e.g. SDK_PATH=$HOME/Downloads/macOS-SDKs instead of
-  $HOME/Downloads/macOS-SDKs/Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers).
+  $HOME/Downloads/macOS-SDKs/Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers).
 
 * _**JOBS**_
 

contrib/guix/manifest.scm

@@ -239,5 +239,5 @@ chain for " target " development."))
           ((string-contains target "-linux-")
            (list (make-bitcoin-cross-toolchain target)))
           ((string-contains target "darwin")
-           (list clang-toolchain-8 binutils imagemagick libtiff librsvg font-tuffy cmake xorriso))
+           (list clang-toolchain-10 binutils imagemagick libtiff librsvg font-tuffy cmake xorriso))
           (else '())))))

contrib/macdeploy/README.md

@@ -13,9 +13,9 @@ When complete, it will have produced `Bitcoin-Core.dmg`.
 ### Step 1: Obtaining `Xcode.app`
 
 Our current macOS SDK
-(`Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz`) can be
+(`Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz`) can be
 extracted from
-[Xcode_11.3.1.xip](https://download.developer.apple.com/Developer_Tools/Xcode_11.3.1/Xcode_11.3.1.xip).
+[Xcode_12.1.xip](https://download.developer.apple.com/Developer_Tools/Xcode_12.1/Xcode_12.1.xip).
 An Apple ID is needed to download this.
 
 After Xcode version 7.x, Apple started shipping the `Xcode.app` in a `.xip`
@@ -27,25 +27,25 @@ approach (tested on Debian Buster) is outlined below:
 apt install cpio
 git clone https://github.com/bitcoin-core/apple-sdk-tools.git
 
-# Unpack Xcode_11.3.1.xip and place the resulting Xcode.app in your current
+# Unpack Xcode_12.1.xip and place the resulting Xcode.app in your current
 # working directory
-python3 apple-sdk-tools/extract_xcode.py -f Xcode_11.3.1.xip | cpio -d -i
+python3 apple-sdk-tools/extract_xcode.py -f Xcode_12.1.xip | cpio -d -i
 ```
 
 On macOS the process is more straightforward:
 
 ```bash
-xip -x Xcode_11.3.1.xip
+xip -x Xcode_12.1.xip
 ```
 
-### Step 2: Generating `Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz` from `Xcode.app`
+### Step 2: Generating `Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz` from `Xcode.app`
 
-To generate `Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz`, run
+To generate `Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz`, run
 the script [`gen-sdk`](./gen-sdk) with the path to `Xcode.app` (extracted in the
 previous stage) as the first argument.
 
 ```bash
-# Generate a Xcode-11.3.1-11C505-extracted-SDK-with-libcxx-headers.tar.gz from
+# Generate a Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz from
 # the supplied Xcode.app
 ./contrib/macdeploy/gen-sdk '/path/to/Xcode.app'
 ```

depends/hosts/darwin.mk

@@ -1,8 +1,8 @@
 OSX_MIN_VERSION=10.14
-OSX_SDK_VERSION=10.15.1
-XCODE_VERSION=11.3.1
-XCODE_BUILD_ID=11C505
-LD64_VERSION=530
+OSX_SDK_VERSION=10.15.6
+XCODE_VERSION=12.1
+XCODE_BUILD_ID=12A7403
+LD64_VERSION=609
 
 OSX_SDK=$(SDK_PATH)/Xcode-$(XCODE_VERSION)-$(XCODE_BUILD_ID)-extracted-SDK-with-libcxx-headers
 
@@ -60,16 +60,11 @@ $(foreach TOOL,$(cctools_TOOLS),$(eval darwin_$(TOOL) = $$(build_prefix)/bin/$$(
 #         Explicitly point to our binaries (e.g. cctools) so that they are
 #         ensured to be found and preferred over other possibilities.
 #
-#     -stdlib=libc++ -nostdinc++ -Xclang -cxx-isystem$(OSX_SDK)/usr/include/c++/v1
+#     -stdlib=libc++ -stdlib++-isystem$(OSX_SDK)/usr/include/c++/v1
 #
 #         Forces clang to use the libc++ headers from our SDK and completely
 #         forget about the libc++ headers from the standard directories
 #
-#         TODO: Once we start requiring a clang version that has the
-#         -stdlib++-isystem<directory> flag first introduced here:
-#         https://reviews.llvm.org/D64089, we should use that instead. Read the
-#         differential summary there for more details.
-#
 #     -Xclang -*system<path_a> \
 #     -Xclang -*system<path_b> \
 #     -Xclang -*system<path_c> ...
@@ -109,8 +104,8 @@ darwin_CXX=env -u C_INCLUDE_PATH -u CPLUS_INCLUDE_PATH \
              $(clangxx_prog) --target=$(host) -mmacosx-version-min=$(OSX_MIN_VERSION) \
                -B$(build_prefix)/bin -mlinker-version=$(LD64_VERSION) \
                -isysroot$(OSX_SDK) \
-               -stdlib=libc++ -nostdinc++ \
-               -Xclang -cxx-isystem$(OSX_SDK)/usr/include/c++/v1 \
+               -stdlib=libc++ \
+               -stdlib++-isystem$(OSX_SDK)/usr/include/c++/v1 \
                -Xclang -internal-externc-isystem$(clang_resource_dir)/include \
                -Xclang -internal-externc-isystem$(OSX_SDK)/usr/include
 

depends/packages/native_cctools.mk

@@ -1,10 +1,9 @@
 package=native_cctools
-$(package)_version=55562e4073dea0fbfd0b20e0bf69ffe6390c7f97
+$(package)_version=2ef2e931cf641547eb8a68cfebde61003587c9fd
 $(package)_download_path=https://github.com/tpoechtrager/cctools-port/archive
 $(package)_file_name=$($(package)_version).tar.gz
-$(package)_sha256_hash=e51995a843533a3dac155dd0c71362dd471597a2d23f13dff194c6285362f875
+$(package)_sha256_hash=6b73269efdf5c58a070e7357b66ee760501388549d6a12b423723f45888b074b
 $(package)_build_subdir=cctools
-$(package)_patches=ld64_disable_threading.patch
 $(package)_dependencies=native_libtapi
 
 define $(package)_set_vars
@@ -17,10 +16,6 @@ define $(package)_set_vars
   $(package)_cxx=$(clangxx_prog)
 endef
 
-define $(package)_preprocess_cmds
-  patch -p1 < $($(package)_patch_dir)/ld64_disable_threading.patch
-endef
-
 define $(package)_config_cmds
   $($(package)_autoconf)
 endef

depends/packages/native_clang.mk

@@ -1,9 +1,9 @@
 package=native_clang
-$(package)_version=8.0.0
-$(package)_download_path=https://releases.llvm.org/$($(package)_version)
-$(package)_download_file=clang+llvm-$($(package)_version)-x86_64-linux-gnu-ubuntu-14.04.tar.xz
-$(package)_file_name=clang-llvm-$($(package)_version)-x86_64-linux-gnu-ubuntu-14.04.tar.xz
-$(package)_sha256_hash=9ef854b71949f825362a119bf2597f744836cb571131ae6b721cd102ffea8cd0
+$(package)_version=10.0.1
+$(package)_download_path=https://github.com/llvm/llvm-project/releases/download/llvmorg-$($(package)_version)
+$(package)_download_file=clang+llvm-$($(package)_version)-x86_64-linux-gnu-ubuntu-16.04.tar.xz
+$(package)_file_name=clang+llvm-$($(package)_version)-x86_64-linux-gnu-ubuntu-16.04.tar.xz
+$(package)_sha256_hash=48b83ef827ac2c213d5b64f5ad7ed082c8bcb712b46644e0dc5045c6f462c231
 
 define $(package)_preprocess_cmds
   rm -f $($(package)_extract_dir)/lib/libc++abi.so*

depends/packages/native_libtapi.mk

@@ -1,9 +1,9 @@
 package=native_libtapi
-$(package)_version=3efb201881e7a76a21e0554906cf306432539cef
+$(package)_version=664b8414f89612f2dfd35a9b679c345aa5389026
 $(package)_download_path=https://github.com/tpoechtrager/apple-libtapi/archive
 $(package)_download_file=$($(package)_version).tar.gz
 $(package)_file_name=$($(package)_version).tar.gz
-$(package)_sha256_hash=380c1ca37cfa04a8699d0887a8d3ee1ad27f3d08baba78887c73b09485c0fbd3
+$(package)_sha256_hash=62e419c12d1c9fad67cc1cd523132bc00db050998337c734c15bc8d73cc02b61
 
 ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
 $(package)_dependencies=native_clang