ARM Mac open-source and packaged build (#677)

This enables cross-compiling for arm64 Mac. This changes the default Xcode version to 12.2, as Xcode 11 doesn't support arm64, and 12.0 fails with vcpkg. It'll still build with 11.7 in the expanded matrix and omit arm64.

Unit tests are built, but not run, on arm64. (They can't be run until GitHub has hosted ARM Mac runners, or we add our own.)

This PR adds a --arch option to the testapp builder but does not yet build for arm64 in the integration_tests workflow.

For packaged builds, we now build LLVM binutils on Mac, in addition to GNU binutils. LLVM binutils are required for the merge_libraries step on Mac, as GNU binutils does not support outputting ARM64 Mach-O files properly. (We still need GNU binutils on Mac for c++filt). Because LLVM binutils can take a long time to build, we cache it.

Author: jonsimantov

.github/workflows/cpp-packaging.yml

@@ -32,9 +32,11 @@ env:
   demumbleVer: "1.1.0"
   # Use SHA256 for hashing files.
   hashCommand: "sha256sum"
-  # Xcode version 12 is the version we build the SDK with.
+  # Xcode version 12.2 is the version we build the SDK with.
   # Our MacOS runners will use the version in /Applications/Xcode_${xcodeVersion}.app
-  xcodeVersion: "12"
+  xcodeVersion: "12.2"
+  # LLVM version with ARM MachO support has no version number yet.
+  llvmVer: "5f187f0afaad33013ba03454c4749d99b1362534"
 
 jobs:
   log_inputs:
@@ -91,7 +93,7 @@ jobs:
         if: runner.os == 'macOS'
         run: sudo xcode-select -s /Applications/Xcode_${{ env.xcodeVersion }}.app/Contents/Developer
 
-      - name: fetch and build binutils
+      - name: Fetch and build binutils
         run: |
           set +e
           # Retry up to 10 times because Curl has a tendency to timeout on
@@ -105,13 +107,46 @@ jobs:
 
           tar -xf binutils.tar.xz
           mv ./binutils-${{ matrix.binutils_version }} ./binutils-src
+          mkdir /tmp/binutils-output
           cd binutils-src
-          ./configure --enable-targets=all --prefix=/tmp/binutils
-          make
+          ./configure --enable-targets=all --prefix=/tmp/binutils-output
+          make -j2
           make install
           cd -
           mkdir -p packaging-tools
-          cp -af /tmp/binutils/bin/* packaging-tools
+          cp -af /tmp/binutils-output/bin/* packaging-tools
+
+      - name: Cache LLVM (Mac only)
+        if: matrix.tools_platform == 'darwin'
+        id: cache_llvm
+        uses: actions/cache@v2
+        with:
+          path: llvm-src/llvm/build/bin
+          key: llvm-bin-${{matrix.tools_platform}}-${{env.xcodeVersion}}-${{env.llvmVer}}
+
+      - name: Fetch LLVM source (Mac only)
+        # Only fetch LLVM if we don't already have binaries from the cache.
+        if: ${{ matrix.tools_platform == 'darwin' && !steps.cache_llvm.outputs.cache-hit }}
+        uses: actions/[email protected]
+        with:
+          repository: llvm/llvm-project
+          path: llvm-src
+          ref: ${{ env.llvmVer }}
+
+      - name: Build LLVM (Mac only) 
+        # Only build LLVM if we don't already have binaries from the cache.
+        if: ${{ matrix.tools_platform == 'darwin' && !steps.cache_llvm.outputs.cache-hit }}
+        run: |
+          mkdir llvm-src/llvm/build
+          cd llvm-src/llvm/build
+          cmake -DLLVM_ENABLE_PROJECTS=clang -DCMAKE_BUILD_TYPE=Release ..
+          cmake --build . -j 3 --target llvm-objcopy --target llvm-nm --target llvm-ar
+
+      - name: Package LLVM output (Mac only)
+        if: matrix.tools_platform == 'darwin'
+        run: |
+          mkdir -p packaging-tools
+          cp -af llvm-src/llvm/build/bin/* packaging-tools
 
       - name: fetch demumble
         uses: actions/[email protected]
@@ -127,7 +162,7 @@ jobs:
           cmake --build .
           python demumble_test.py
           cd -
-          mkdir -p packaging-tools-
+          mkdir -p packaging-tools
           cp -af demumble-src/demumble packaging-tools
 
       - name: archive tools
@@ -254,7 +289,7 @@ jobs:
       matrix:
         os: [windows-latest, ubuntu-latest, macos-latest]
         build_type: ["Release", "Debug"]
-        architecture: ["x64", "x86"]
+        architecture: ["x64", "x86", "arm64"]
         msvc_runtime: ["static", "dynamic"]
         linux_abi: ["legacy", "c++11"]
         python_version: [3.7]
@@ -292,6 +327,10 @@ jobs:
           msvc_runtime: "dynamic"
         - os: ubuntu-latest
           build_type: "Debug"
+        - os: ubuntu-latest
+          architecture: "arm64"
+        - os: windows-latest
+          architecture: "arm64"
 
     steps:
       - name: setup Xcode version (macos)
@@ -308,6 +347,10 @@ jobs:
           echo "VCPKG_RESPONSE_FILE=external/vcpkg_${{ matrix.vcpkg_triplet }}_response_file.txt" >> $GITHUB_ENV
           echo "MATRIX_UNIQUE_NAME=${{ matrix.os }}-${{ matrix.build_type }}-${{ matrix.architecture }}-${{ matrix.python_version }}-${{ matrix.msvc_runtime }}-${{ matrix.linux_abi }}" >> $GITHUB_ENV
           echo "SDK_NAME=${{ matrix.sdk_platform }}-${{ matrix.architecture }}-${{ matrix.build_type }}-${{ matrix.msvc_runtime }}-${{ matrix.linux_abi }}" >> $GITHUB_ENV
+          if [[ '${{ matrix.sdk_platform }}' == 'darwin' ]]; then
+            # If Mac, also hash vcpkg cache on Xcode version.
+            echo "VCPKG_EXTRA_HASH=-xcode${{env.xcodeVersion}}" >> $GITHUB_ENV
+          fi
 
       - name: Add msbuild to PATH (windows)
         if: startsWith(matrix.os, 'windows')
@@ -318,7 +361,7 @@ jobs:
         uses: actions/cache@v2
         with:
           path: external/vcpkg/installed
-          key: dev-vcpkg-${{ matrix.architecture }}-${{ matrix.vcpkg_triplet_suffix }}-${{ matrix.msvc_runtime }}-${{ matrix.linux_abi }}-${{ hashFiles(format('{0}', env.VCPKG_RESPONSE_FILE)) }}-${{ hashFiles('.git/modules/external/vcpkg/HEAD') }}
+          key: dev-vcpkg-${{ matrix.architecture }}-${{ matrix.vcpkg_triplet_suffix }}-${{ matrix.msvc_runtime }}-${{ matrix.linux_abi }}-${{ hashFiles(format('{0}', env.VCPKG_RESPONSE_FILE)) }}-${{ hashFiles('.git/modules/external/vcpkg/HEAD') }}${{env.VCPKG_EXTRA_HASH}}
 
       - name: Cache ccache files
         if: startsWith(matrix.os, 'ubuntu') || startsWith(matrix.os, 'macos')
@@ -503,15 +546,15 @@ jobs:
             variant=$(sdk-src/build_scripts/desktop/get_variant.sh "${pkg}")
             additional_flags=(${verbose_flag})
             # Several build targets require explicitly-set binutils format to be passed
-            # to package.sh (and thus, to merge_libraries).
-            if [[ "${{ matrix.sdk_platform }}" == "darwin" && "${variant}" == "arm64" ]]; then
-              # MacOS ARM
-              additional_flags+=(-f mach-o-arm64)
+            # to package.sh (and thus, to merge_libraries), or use LLVM binutils.
+            if [[ "${{ matrix.sdk_platform }}" == "darwin" ]]; then
+              # MacOS: use LLVM binutils for both X64 and ARM64
+              additional_flags+=(-L)
             elif [[ "${{ matrix.sdk_platform }}" == "windows" && "${variant}" == *"/x64/"* ]]; then
-              # Windows x64
+              # Windows x64: force input and output target format
               additional_flags+=(-f pe-x86-64,pe-bigobj-x86-64)
             elif [[ "${{ matrix.sdk_platform }}" == "windows" && "${variant}" == *"/x86/"* ]]; then
-              # Windows x86
+              # Windows x86: force input and output target format
               additional_flags+=(-f pe-i386,pe-bigobj-i386)
             fi
             sdk-src/build_scripts/desktop/package.sh -b ${pkg} -o firebase-cpp-sdk-${{ matrix.sdk_platform }}${{ matrix.suffix }}-package -p ${{ matrix.sdk_platform }} -t bin -d ${variant} -P python3 -j ${additional_flags[*]}

.github/workflows/desktop.yml

@@ -76,6 +76,14 @@ jobs:
             xcode_version: "12.4"
           - os: windows-latest
             xcode_version: "12.4"
+          # arm64 is only for macos
+          - os: ubuntu-latest
+            architecture: "arm64"
+          - os: windows-latest
+            architecture: "arm64"
+          # arm64 can't be built on Xcode 11
+          - xcode_version: "11.7"
+            architecture: "arm64"
     steps:                
       - name: Setup Xcode version (macos)
         if: runner.os == 'macOS'
@@ -114,21 +122,25 @@ jobs:
           echo "VCPKG_SUFFIX: ${{env.VCPKG_SUFFIX}}"
           echo "VCPKG_RESPONSE_FILE=external/vcpkg_custom_data/response_files/${{ matrix.architecture }}-${{ env.VCPKG_SUFFIX }}.txt" >> $GITHUB_ENV
           echo "MATRIX_UNIQUE_NAME=${{ matrix.os }}-${{ matrix.build_type }}-${{ matrix.architecture }}-${{ matrix.msvc_runtime }}" >> $GITHUB_ENV
+          if [[ '${{ matrix.sdk_platform }}' == 'darwin' ]]; then
+            # If Mac, also hash vcpkg cache on Xcode version.
+            echo "VCPKG_EXTRA_HASH=-xcode${{matrix.xcode_version}}" >> $GITHUB_ENV
+          fi
 
       - name: Cache vcpkg C++ dependencies
         id: cache_vcpkg
         uses: actions/cache@v2
         with:
           path: external/vcpkg/installed
-          key: dev-vcpkg-${{ matrix.architecture }}-${{ env.VCPKG_SUFFIX }}-${{ matrix.msvc_runtime }}-${{ hashFiles(format('{0}', env.VCPKG_RESPONSE_FILE)) }}-${{ hashFiles('.git/modules/external/vcpkg/HEAD') }}
+          key: dev-vcpkg-${{ matrix.architecture }}-${{ env.VCPKG_SUFFIX }}-${{ matrix.msvc_runtime }}-${{ hashFiles(format('{0}', env.VCPKG_RESPONSE_FILE)) }}-${{ hashFiles('.git/modules/external/vcpkg/HEAD') }}${{ env.VCPKG_EXTRA_HASH }}
 
       - name: Cache ccache files
         if: startsWith(matrix.os, 'ubuntu') || startsWith(matrix.os, 'macos')
         id: cache_ccache
         uses: actions/cache@v2
         with:
           path: ccache_dir
-          key: dev-test-ccache-${{ env.MATRIX_UNIQUE_NAME }}
+          key: dev-test-ccache-${{ env.MATRIX_UNIQUE_NAME }}${{ env.VCPKG_EXTRA_HASH }}
 
       - name: Setup python
         uses: actions/setup-python@v2
@@ -193,7 +205,7 @@ jobs:
 
       - name: Run unit tests (windows & macos)
         # TODO: Enable tests for x86 once they are all working
-        if: (startsWith(matrix.os, 'windows') || startsWith(matrix.os, 'macos')) && matrix.architecture != 'x86'
+        if: (startsWith(matrix.os, 'windows') || startsWith(matrix.os, 'macos')) && matrix.architecture != 'x86' && matrix.architecture != 'arm64'
         env:
           LANG: en_US
         run: |

CMakeLists.txt

@@ -136,6 +136,15 @@ else()
   set(DESKTOP OFF)
 endif()
 
+# For OSX, if the architectures we are building for do not match the current
+# host, treat it as cross-compiling.
+if(DESKTOP AND APPLE)
+  if(NOT "${CMAKE_OSX_ARCHITECTURES}" MATCHES "${CMAKE_HOST_SYSTEM_PROCESSOR}")
+    message("Cross-compiling ENABLED")
+    set(CMAKE_CROSSCOMPILING ON CACHE BOOL "" FORCE)
+  endif()
+endif()
+
 if(DESKTOP AND NOT MSVC AND NOT APPLE)
   # Linux-specific option.
   if (FIREBASE_LINUX_USE_CXX11_ABI)
@@ -305,6 +314,12 @@ endif()
 
 # Include Firestore's external build early to resolve conflicts on packages.
 if(FIRESTORE_USE_EXTERNAL_CMAKE_BUILD)
+  if(APPLE)
+    # On Apple, set GRPC_BAZEL_BUILD to fix an issue with __thread.
+    # Setting this define makes gRPC use pthread's thread-local storage
+    # instead of GCC's.
+    add_definitions(-DGRPC_BAZEL_BUILD=1)
+  endif()
   set(FIRESTORE_BINARY_DIR ${FIRESTORE_SOURCE_DIR}-build)
 
   if(FIREBASE_CPP_BUILD_TESTS OR FIREBASE_CPP_BUILD_STUB_TESTS)
@@ -329,7 +344,7 @@ set(FLATBUFFERS_FLATHASH OFF CACHE BOOL "")
 set(LEVELDB_BUILD_TESTS OFF CACHE BOOL "")
 
 # Disable building flatc if cross compiling
-if(IOS OR ANDROID)
+if(IOS OR ANDROID OR CMAKE_CROSSCOMPILING)
   set(FLATBUFFERS_BUILD_FLATC OFF CACHE BOOL "")
 endif()
 
@@ -367,6 +382,14 @@ if(DESKTOP)
   set(CMAKE_USE_LIBSSH2 OFF)
   set(HTTP_ONLY ON)
   set(BUILD_TESTING OFF)
+  if(APPLE AND CMAKE_CROSSCOMPILING AND NOT CMAKE_TOOLCHAIN_FILE)
+    # curl's CMake file checks for CMAKE_TOOLCHAIN_FILE rather than checking
+    # CMAKE_CROSSCOMPILING when determining whether we are cross-compiling.
+    # So if CMAKE_CROSSCOMPILING is set and we don't have a CMAKE_TOOLCHAIN_FILE,
+    # just add a null one now so curl is happy. This only happens on Mac when
+    # building for arm64/x86_64 on the opposite machine.
+    set(CMAKE_TOOLCHAIN_FILE /dev/null)
+  endif()
   add_external_library(curl)
 
   add_external_library(libuv)
@@ -402,13 +425,15 @@ if(DESKTOP)
         ${UWEBSOCKETS_SOURCE_DIR}/src/WebSocket.cpp)
     if(MSVC)
       set(websockets_additional_defines
+        ${websockets_additional_defines}
         -DWIN32_LEAN_AND_MEAN # Ensure that windows doesn't include winsock.h by
                               # default, as it can cause issues when libraries try
                               # to include winsock2.h later on in the process.
         -DUSE_LIBUV=1
       )
     elseif(APPLE)
       set(websockets_additional_defines
+        ${websockets_additional_defines}
         -DUSE_LIBUV=1
       )
     endif()
@@ -440,7 +465,7 @@ if(DESKTOP)
   if(FIRESTORE_USE_EXTERNAL_CMAKE_BUILD)
     # The Firestore build includes protobuf and nanopb already
     list(APPEND CMAKE_MODULE_PATH ${NANOPB_SOURCE_DIR}/extra)
-    find_package(Nanopb REQUIRED)
+    find_package(Nanopb)
 
     set(PROTOBUF_FOUND ON)
   else()
@@ -462,7 +487,7 @@ if(DESKTOP)
   endif()
 endif()
 
-if(DESKTOP)
+if(DESKTOP AND NOT CMAKE_CROSSCOMPILING)
   # Desktop platforms do not need to declare this dependency, as they will build
   # flatc correctly when needed.
   set(FIREBASE_FLATBUFFERS_DEPENDENCIES "")

build_scripts/desktop/finish_darwin.sh

@@ -55,7 +55,7 @@ done
 # means that the lipo step has already occurred.
 if [[ ! -d "${packagepath}/libs/${os}/universal" ]]; then
     echo "Repackaging libraries using Mac libtool..."
-    for arch in "${architectures[*]}"; do
+    for arch in ${architectures[*]}; do
 	for lib in "${packagepath}/libs/${os}/${arch}"/*.a; do
 	    pushd $(dirname ${lib}) > /dev/null
 	    libname=$(basename "${lib}")
@@ -89,7 +89,7 @@ fi
 echo "Creating frameworks..."
 rm -rf "${packagepath}/frameworks/${os}"
 mkdir -p "${packagepath}/frameworks/${os}"
-for arch in universal "${architectures[*]}"; do
+for arch in universal ${architectures[*]}; do
     for f in "${packagepath}/libs/${os}/${arch}/"*.a; do
 	library=$f
 	framework=$(basename "${library}" | sed 's|^lib||' | sed 's|\.a$||')

build_scripts/desktop/package.sh

@@ -18,6 +18,7 @@ options:
   -f, binutils format                             default: [auto-detect]
   -j, run merge_libraries jobs in parallel
   -v, enable verbose mode
+  -L, use LLVM binutils
 example:
   build_scripts/desktop/package.sh -b firebase-cpp-sdk-linux -p linux -o package_out -v x86 -j"
 }
@@ -35,6 +36,7 @@ built_sdk_tarfile=
 binutils_format=
 temp_dir=
 run_in_parallel=0
+use_llvm_binutils=0
 
 . "${root_dir}/build_scripts/packaging.conf"
 
@@ -48,7 +50,7 @@ abspath(){
     fi
 }
 
-while getopts "f:b:o:p:d:m:P:t:hjv" opt; do
+while getopts "f:b:o:p:d:m:P:t:hjLv" opt; do
     case $opt in
         f)
             binutils_format=$OPTARG
@@ -73,6 +75,9 @@ while getopts "f:b:o:p:d:m:P:t:hjv" opt; do
         v)
             verbose=1
             ;;
+        L)
+            use_llvm_binutils=1
+            ;;
         d)
             variant=$OPTARG
             ;;
@@ -228,13 +233,19 @@ readonly -a rename_namespaces=(flatbuffers flexbuffers reflection ZLib bssl uWS
 readonly rename_string=f_b_
 
 readonly demangle_cmds=${tools_path}/c++filt,${tools_path}/demumble
-readonly binutils_objcopy=${tools_path}/objcopy
-if [[ -x ${tools_path}/nm-new ]] ; then
-    readonly binutils_nm=${tools_path}/nm-new
-else
-    readonly binutils_nm=${tools_path}/nm
+if [[ ${use_llvm_binutils} -eq 1 ]]; then
+    readonly binutils_objcopy=${tools_path}/llvm-objcopy
+    readonly binutils_nm=${tools_path}/llvm-nm
+    readonly binutils_ar=${tools_path}/llvm-ar
+else    
+    readonly binutils_objcopy=${tools_path}/objcopy
+    if [[ -x ${tools_path}/nm-new ]] ; then
+        readonly binutils_nm=${tools_path}/nm-new
+    else
+	readonly binutils_nm=${tools_path}/nm
+    fi
+    readonly binutils_ar=${tools_path}/ar
 fi
-readonly binutils_ar=${tools_path}/ar
 
 cache_file=/tmp/merge_libraries_cache.$$
 # Clean up cache file after script is finished.

external/vcpkg_custom_data/response_files/arm64-osx.txt

@@ -0,0 +1,4 @@
+protobuf
+zlib
+--triplet
+arm64-osx