Merge pull request #83809 from finagolfin/cross-tools

[6.2.1][build] Make it possible to build a cross-compilation toolchain for Android, including Testing

Author: shahmishal

stdlib/cmake/modules/AddSwiftStdlib.cmake

@@ -2504,6 +2504,8 @@ function(add_swift_target_library name)
         list(APPEND swiftlib_link_flags_all "-shared")
         # TODO: Instead of `lib${name}.so` find variable or target property which already have this value.
         list(APPEND swiftlib_link_flags_all "-Wl,-soname,lib${name}.so")
+        # Ensure compatibility with Android 15+ devices using 16KB memory pages.
+        list(APPEND swiftlib_link_flags_all "-Wl,-z,max-page-size=16384")
       endif()
 
       if (SWIFTLIB_BACK_DEPLOYMENT_LIBRARY)

test/SILOptimizer/concat_string_literals.32.swift

@@ -4,6 +4,7 @@
 
 // We have a separate test for 64-bit architectures.
 // REQUIRES: PTRSIZE=32
+// XFAIL: OS=linux-androideabi
 
 // NOTE: 25185.byteSwapped = 0x62 'a', 0x61 'b'
 // CHECK-LABEL: test_ascii_scalar_scalar2

utils/build-script-impl

@@ -2351,7 +2351,7 @@ for host in "${ALL_HOSTS[@]}"; do
                     continue
                 fi
 
-                if [[ "${CROSS_COMPILE_BUILD_SWIFT_TOOLS}" == "0" ]]; then
+                if [[ "$(true_false ${CROSS_COMPILE_BUILD_SWIFT_TOOLS})" == "FALSE" ]]; then
                     echo "Skipping building Foundation Macros for ${host}, because the host tools are not being built"
                     continue
                 fi
@@ -2940,7 +2940,7 @@ for host in "${ALL_HOSTS[@]}"; do
                     continue
                 fi
 
-                if [[ "${CROSS_COMPILE_BUILD_SWIFT_TOOLS}" == "0" && "${product}" == "foundation_macros" ]]; then
+                if [[ "$(true_false ${CROSS_COMPILE_BUILD_SWIFT_TOOLS})" == "FALSE" && "${product}" == "foundation_macros" ]]; then
                     echo "Skipping installing Foundation Macros for ${host}, because the host tools are not being built"
                     continue
                 fi

utils/build_swift/build_swift/driver_arguments.py

@@ -677,6 +677,12 @@ def create_argument_parser():
                 "for each cross-compiled toolchain's destdir, useful when building "
                 "multiple toolchains and can be disabled if only cross-compiling one.")
 
+    option('--cross-compile-build-swift-tools', toggle_true,
+           default=True,
+           help="Cross-compile the Swift compiler, other host tools from the "
+                "compiler repository, and various macros for each listed "
+                "--cross-compile-hosts platform.")
+
     option('--stdlib-deployment-targets', store,
            type=argparse.ShellSplitType(),
            default=None,

utils/build_swift/tests/expected_options.py

@@ -155,6 +155,7 @@
     'compiler_vendor': defaults.COMPILER_VENDOR,
     'coverage_db': None,
     'cross_compile_append_host_target_to_destdir': True,
+    'cross_compile_build_swift_tools': True,
     'cross_compile_deps_path': None,
     'cross_compile_hosts': [],
     'infer_cross_compile_hosts_on_darwin': False,
@@ -622,6 +623,7 @@ class BuildScriptImplOption(_BaseOption):
     EnableOption('--build-swift-clang-overlays'),
     EnableOption('--build-swift-remote-mirror'),
     EnableOption('--cross-compile-append-host-target-to-destdir'),
+    EnableOption('--cross-compile-build-swift-tools'),
     EnableOption('--color-in-tests'),
     EnableOption('--distcc'),
     EnableOption('--sccache'),

utils/swift_build_support/swift_build_support/build_script_invocation.py

@@ -119,6 +119,8 @@ def convert_to_impl_arguments(self):
             "--cmake-generator", args.cmake_generator,
             "--cross-compile-append-host-target-to-destdir", str(
                 args.cross_compile_append_host_target_to_destdir).lower(),
+            "--cross-compile-build-swift-tools", str(
+                args.cross_compile_build_swift_tools).lower(),
             "--build-jobs", str(args.build_jobs),
             "--lit-jobs", str(args.lit_jobs),
             "--common-cmake-options=%s" % ' '.join(

utils/swift_build_support/swift_build_support/products/cmake_product.py

@@ -24,7 +24,7 @@ def is_verbose(self):
         return self.args.verbose_build
 
     def build_with_cmake(self, build_targets, build_type, build_args,
-                         prefer_native_toolchain=False):
+                         prefer_native_toolchain=False, build_llvm=True):
         assert self.toolchain.cmake is not None
         cmake_build = []
         _cmake = cmake.CMake(self.args, self.toolchain,
@@ -71,9 +71,7 @@ def build_with_cmake(self, build_targets, build_type, build_args,
                            env=env)
 
         is_llvm = self.product_name() == "llvm"
-        if (not is_llvm and not self.args.skip_build) or (
-            is_llvm and self.args._build_llvm
-        ):
+        if (not is_llvm and not self.args.skip_build) or (is_llvm and build_llvm):
             cmake_opts = [self.build_dir, "--config", build_type]
 
             shell.call(

utils/swift_build_support/swift_build_support/products/llvm.py

@@ -249,10 +249,13 @@ def build(self, host_target):
                 # space/time efficient than -g on that platform.
                 llvm_cmake_options.define('LLVM_USE_SPLIT_DWARF:BOOL', 'YES')
 
-        if not self.args._build_llvm:
+        build = True
+        if not self.args._build_llvm or (not self.args.cross_compile_build_swift_tools
+                                         and self.is_cross_compile_target(host_target)):
             # Indicating we don't want to build LLVM at all should
             # override everything.
             build_targets = []
+            build = False
         elif self.args.skip_build or not self.args.build_llvm:
             # We can't skip the build completely because the standalone
             # build of Swift depends on these.
@@ -399,7 +402,8 @@ def build(self, host_target):
 
         self._handle_cxx_headers(host_target, platform)
 
-        self.build_with_cmake(build_targets, self.args.llvm_build_variant, [])
+        self.build_with_cmake(build_targets, self.args.llvm_build_variant, [],
+                              build_llvm=build)
 
         # copy over the compiler-rt builtins for iOS/tvOS/watchOS to ensure
         # that Swift's stdlib can use compiler-rt builtins when targeting
@@ -484,7 +488,9 @@ def should_install(self, host_target):
         Whether or not this product should be installed with the given
         arguments.
         """
-        return self.args.install_llvm
+        return self.args.install_llvm and (
+            self.args.cross_compile_build_swift_tools or
+            not self.is_cross_compile_target(host_target))
 
     def install(self, host_target):
         """

utils/swift_build_support/swift_build_support/products/product.py

@@ -389,7 +389,7 @@ def get_linux_target(self, platform, arch):
         sysroot_arch, vendor, abi = self.get_linux_target_components(arch)
         return '{}-{}-linux-{}'.format(sysroot_arch, vendor, abi)
 
-    def generate_linux_toolchain_file(self, platform, arch):
+    def generate_linux_toolchain_file(self, platform, arch, crosscompiling=True):
         """
         Generates a new CMake tolchain file that specifies Linux as a target
         platform.
@@ -402,18 +402,34 @@ def generate_linux_toolchain_file(self, platform, arch):
 
         toolchain_args = {}
 
-        toolchain_args['CMAKE_SYSTEM_NAME'] = 'Linux'
-        toolchain_args['CMAKE_SYSTEM_PROCESSOR'] = arch
+        if crosscompiling:
+            if platform == "linux":
+                toolchain_args['CMAKE_SYSTEM_NAME'] = 'Linux'
+                toolchain_args['CMAKE_SYSTEM_PROCESSOR'] = arch
+            elif platform == "android":
+                toolchain_args['CMAKE_SYSTEM_NAME'] = 'Android'
+                toolchain_args['CMAKE_SYSTEM_VERSION'] = self.args.android_api_level
+                toolchain_args['CMAKE_SYSTEM_PROCESSOR'] = arch if not arch == 'armv7' \
+                                                           else 'armv7-a'
+                toolchain_args['CMAKE_ANDROID_NDK'] = self.args.android_ndk
+                toolchain_args['CMAKE_FIND_ROOT_PATH'] = self.args.cross_compile_deps_path
+                # This is a workaround for a CMake 3.30+ bug,
+                # https://gitlab.kitware.com/cmake/cmake/-/issues/26154, and can
+                # be removed once that is fixed.
+                toolchain_args['CMAKE_SHARED_LINKER_FLAGS'] = '\"\"'
 
         # We only set the actual sysroot if we are actually cross
         # compiling. This is important since otherwise cmake seems to change the
         # RUNPATH to be a relative rather than an absolute path, breaking
         # certain cmark tests (and maybe others).
-        maybe_sysroot = self.get_linux_sysroot(platform, arch)
-        if maybe_sysroot is not None:
-            toolchain_args['CMAKE_SYSROOT'] = maybe_sysroot
-
-        target = self.get_linux_target(platform, arch)
+        if platform == "linux":
+            maybe_sysroot = self.get_linux_sysroot(platform, arch)
+            if maybe_sysroot is not None:
+                toolchain_args['CMAKE_SYSROOT'] = maybe_sysroot
+
+            target = self.get_linux_target(platform, arch)
+        elif platform == "android":
+            target = '%s-unknown-linux-android%s' % (arch, self.args.android_api_level)
         if self.toolchain.cc.endswith('clang'):
             toolchain_args['CMAKE_C_COMPILER_TARGET'] = target
         if self.toolchain.cxx.endswith('clang++'):
@@ -459,10 +475,30 @@ def generate_toolchain_file_for_darwin_or_linux(
                 platform, arch,
                 macos_deployment_version=override_macos_deployment_version)
             self.cmake_options.define('CMAKE_TOOLCHAIN_FILE:PATH', toolchain_file)
-        elif platform == "linux":
-            toolchain_file = self.generate_linux_toolchain_file(platform, arch)
+        elif platform == "linux" or platform == "android":
+            # Always cross-compile for linux, but not on Android, as a native
+            # compile on Android does not use the NDK and its CMake config.
+            cross_compile = platform == "linux" or \
+                self.is_cross_compile_target(host_target)
+            toolchain_file = self.generate_linux_toolchain_file(platform, arch,
+                                                                cross_compile)
             self.cmake_options.define('CMAKE_TOOLCHAIN_FILE:PATH', toolchain_file)
 
+            if cross_compile and platform == "android":
+                resource_dir = None
+                # build-script-impl products build before the install and use
+                # the Swift stdlib from the compiler build directory instead,
+                # while products built even before that currently do not support
+                # cross-compiling Swift.
+                if not self.is_before_build_script_impl_product() and \
+                   not self.is_build_script_impl_product():
+                    install_path = self.host_install_destdir(host_target) + \
+                        self.args.install_prefix
+                    resource_dir = '%s/lib/swift' % install_path
+                flags = targets.StdlibDeploymentTarget.get_target_for_name(
+                    host_target).platform.swift_flags(self.args, resource_dir)
+                self.cmake_options.define('CMAKE_Swift_FLAGS', flags)
+
         return toolchain_file
 
     def get_openbsd_toolchain_file(self):

utils/swift_build_support/swift_build_support/products/swift_testing.py

@@ -127,3 +127,11 @@ def install(self, host_target):
         install_prefix = install_destdir + self.args.install_prefix
 
         self.install_with_cmake(['install'], install_prefix)
+
+    @classmethod
+    def is_build_script_impl_product(cls):
+        return False
+
+    @classmethod
+    def is_before_build_script_impl_product(cls):
+        return False