[Concurrency] Build C only libdispatch before Swift on non-Darwin pla… (swiftlang#35738)

* [Concurrency] Build C only libdispatch before Swift on non-Darwin platforms

_Concurrency depends on libdispatch and since it is not available by
default on non-Darwin platforms, it needs to be built before the Swift
project, so that the dependency can be resolved.

* Fix clibdispatch installation and BuildSystem tests

* Fix build system tests on Darwin and formatting

Author: drexin

stdlib/public/Concurrency/CMakeLists.txt

@@ -19,16 +19,17 @@ set(LLVM_OPTIONAL_SOURCES
 set(swift_concurrency_link_libraries
   swiftCore)
 
+set(swift_concurrency_link_flags "${SWIFT_RUNTIME_SWIFT_LINK_FLAGS}")
+
 if(SWIFT_CONCURRENCY_USES_DISPATCH)
   if(NOT CMAKE_SYSTEM_NAME STREQUAL Darwin)
     include_directories(AFTER
                           ${SWIFT_PATH_TO_LIBDISPATCH_SOURCE})
+    link_directories(AFTER
+                        ${SWIFT_PATH_TO_LIBDISPATCH_BUILD})
 
-    # FIXME: we can't rely on libdispatch having been built for the
-    # target at this point in the process.  Currently, we're relying
-    # on soft-linking.
-    #list(APPEND swift_concurrency_link_libraries
-    #  dispatch)
+    list(APPEND swift_concurrency_link_flags
+      "-ldispatch")
   endif()
 endif()
 
@@ -67,6 +68,6 @@ add_swift_target_library(swift_Concurrency ${SWIFT_STDLIB_LIBRARY_BUILD_TYPES} I
     ${SWIFT_STANDARD_LIBRARY_SWIFT_FLAGS}
     -parse-stdlib
     -Xfrontend -enable-experimental-concurrency
-  LINK_FLAGS "${SWIFT_RUNTIME_SWIFT_LINK_FLAGS}"
+  LINK_FLAGS "${swift_concurrency_link_flags}"
   INSTALL_IN_COMPONENT stdlib
 )

utils/build-presets.ini

@@ -824,6 +824,7 @@ long-test
 stress-test
 test-optimized
 foundation
+clibdispatch
 libdispatch
 indexstore-db
 sourcekit-lsp
@@ -966,6 +967,7 @@ release
 test
 validation-test
 foundation
+clibdispatch
 libdispatch
 indexstore-db
 sourcekit-lsp
@@ -1065,6 +1067,7 @@ swiftpm
 swift-driver
 xctest
 foundation
+clibdispatch
 libdispatch
 swiftsyntax
 indexstore-db
@@ -1593,6 +1596,7 @@ mixin-preset=
     mixin_swiftpm_base
     mixin_linux_install_components_with_clang
 
+clibdispatch
 libdispatch
 libicu
 foundation
@@ -1841,6 +1845,7 @@ llbuild
 swiftpm
 xctest
 foundation
+clibdispatch
 libdispatch
 indexstore-db
 sourcekit-lsp
@@ -2456,6 +2461,7 @@ install-symroot=%(install_symroot)s
 mixin-preset=source_compat_suite_base
 build-subdir=compat_linux
 foundation
+clibdispatch
 libdispatch
 xctest
 install-foundation
@@ -2511,6 +2517,7 @@ no-assertions
 lldb
 foundation
 libicu
+clibdispatch
 libdispatch
 build-ninja
 

utils/build-script

@@ -318,6 +318,10 @@ def apply_default_arguments(toolchain, args):
         args.stdlib_deployment_targets = [
             target.name for target in stdlib_targets]
 
+    if (args.build_swift and
+            platform.system() != "Darwin"):
+        args.build_clibdispatch = True
+
     # SwiftPM and XCTest have a dependency on Foundation.
     # On OS X, Foundation is built automatically using xcodebuild.
     # On Linux, we must ensure that it is built manually.
@@ -624,6 +628,7 @@ class BuildScriptInvocation(object):
             (args.build_llbuild, "llbuild"),
             (args.build_libcxx, "libcxx"),
             (args.build_libdispatch, "libdispatch"),
+            (args.build_clibdispatch, "clibdispatch"),
             (args.build_libicu, "libicu")
         ]
         for (should_build, string_name) in conditional_subproject_configs:
@@ -865,6 +870,8 @@ class BuildScriptInvocation(object):
             impl_product_classes.append(products.LibCXX)
         if self.args.build_libicu:
             impl_product_classes.append(products.LibICU)
+        if self.args.build_clibdispatch:
+            impl_product_classes.append(products.CLibDispatch)
         if self.args.build_swift:
             impl_product_classes.append(products.Swift)
         if self.args.build_lldb:

utils/build-script-impl

@@ -230,6 +230,7 @@ KNOWN_SETTINGS=(
 )
 
 components=(
+  clibdispatch
   cmark
   foundation
   libcxx
@@ -1162,6 +1163,7 @@ STRESSTEST_PACKAGE_DIR="${WORKSPACE}/swift-stress-tester"
 XCTEST_SOURCE_DIR="${WORKSPACE}/swift-corelibs-xctest"
 FOUNDATION_SOURCE_DIR="${WORKSPACE}/swift-corelibs-foundation"
 FOUNDATION_STATIC_SOURCE_DIR="${WORKSPACE}/swift-corelibs-foundation"
+CLIBDISPATCH_SOURCE_DIR="${WORKSPACE}/swift-corelibs-libdispatch"
 LIBDISPATCH_SOURCE_DIR="${WORKSPACE}/swift-corelibs-libdispatch"
 LIBDISPATCH_STATIC_SOURCE_DIR="${WORKSPACE}/swift-corelibs-libdispatch"
 LIBICU_SOURCE_DIR="${WORKSPACE}/icu"
@@ -1175,6 +1177,7 @@ PRODUCTS=(llvm)
 [[ "${SKIP_BUILD_CMARK}" ]] || PRODUCTS+=(cmark)
 [[ "${SKIP_BUILD_LIBCXX}" ]] || PRODUCTS+=(libcxx)
 [[ "${SKIP_BUILD_LIBICU}" ]] || PRODUCTS+=(libicu)
+[[ "${SKIP_BUILD_CLIBDISPATCH}" ]] || PRODUCTS+=(clibdispatch)
 [[ "${SKIP_BUILD_SWIFT}" ]] || PRODUCTS+=(swift)
 [[ "${SKIP_BUILD_LLDB}" ]] || PRODUCTS+=(lldb)
 [[ "${SKIP_BUILD_LIBDISPATCH}" ]] || PRODUCTS+=(libdispatch)
@@ -1259,7 +1262,7 @@ function build_directory_bin() {
             foundation|foundation_static)
                 echo "${root}/${FOUNDATION_BUILD_TYPE}/bin"
                 ;;
-            libdispatch|libdispatch_static)
+            clibdispatch|libdispatch|libdispatch_static)
                 echo "${root}/${LIBDISPATCH_BUILD_TYPE}/bin"
                 ;;
             libicu)
@@ -1377,7 +1380,7 @@ function cmake_config_opt() {
             foundation|foundation_static)
                 echo "--config ${FOUNDATION_BUILD_TYPE}"
                 ;;
-            libdispatch|libdispatch_static)
+            clibdispatch|libdispatch|libdispatch_static)
                 echo "--config ${LIBDISPATCH_BUILD_TYPE}"
                 ;;
             libicu)
@@ -1764,7 +1767,6 @@ for host in "${ALL_HOSTS[@]}"; do
                 ;;
 
             swift)
-
                 if [[ "${ANDROID_API_LEVEL}" ]]; then
                     cmake_options=(
                         "${cmake_options[@]}"
@@ -2320,9 +2322,18 @@ for host in "${ALL_HOSTS[@]}"; do
                 )
 
                 ;;
-            libdispatch|libdispatch_static)
+            clibdispatch|libdispatch|libdispatch_static)
                 LIBDISPATCH_BUILD_DIR=$(build_directory ${host} ${product})
                 SWIFT_BUILD_PATH="$(build_directory ${host} swift)"
+                ENABLE_SWIFT=YES
+                if [[ "${product}" = "clibdispatch" ]]; then
+                  if [[ "$(uname -s)" = "Darwin" ]]; then
+                    continue # We don't build clibdispatch on Darwin
+                  fi
+                  ENABLE_SWIFT=NO
+                  LIBDISPATCH_BUILD_DIR=$(build_directory ${host} libdispatch)
+                  build_dir=${LIBDISPATCH_BUILD_DIR}
+                fi
 
                 case "${host}" in
                 macosx-*)
@@ -2338,7 +2349,7 @@ for host in "${ALL_HOSTS[@]}"; do
                   call rm -rf "${LIBDISPATCH_BUILD_DIR}"
 
                   cmake_options=(
-                    -DENABLE_SWIFT=YES
+                    -DENABLE_SWIFT="${ENABLE_SWIFT}"
                     ${cmake_options[@]}
                     -DCMAKE_BUILD_TYPE:STRING="${LIBDISPATCH_BUILD_TYPE}"
                     -DCMAKE_C_COMPILER:PATH="${CLANG_BIN}/clang"
@@ -2506,6 +2517,13 @@ for host in "${ALL_HOSTS[@]}"; do
         if [[ "${product}" = "llvm" ]] && [[ "${BUILD_LLVM}" = "1" ]] && [[ "$(uname -s)" = "Darwin" ]]; then
             copy_embedded_compiler_rt_builtins_from_darwin_host_toolchain "$(build_directory_bin ${host} llvm)/.."
         fi
+
+        # We need to copy libdispatch.so into the swift lib directory
+        # so the tests can find it
+        if [[ "${product}" = "clibdispatch" ]] && [[ "uname -s" != "Darwin" ]]; then
+            clibdispatch_libdir="$(build_directory ${host} swift)/lib/swift/${SWIFT_HOST_VARIANT}/${SWIFT_HOST_VARIANT_ARCH}"
+            call cp "${build_dir}/libdispatch.so" "${clibdispatch_libdir}"
+        fi
     done
 done
 # END OF BUILD PHASE
@@ -2762,13 +2780,17 @@ for host in "${ALL_HOSTS[@]}"; do
             foundation_static)
               continue
             ;;
-            libdispatch)
+            clibdispatch|libdispatch)
                 if [[ "${SKIP_TEST_LIBDISPATCH}" ]]; then
                     continue
                 fi
 
                 case "${host}" in
                 macosx-*)
+                  if [[ "${product}" = clibdispatch ]]; then
+                    # we don't build clibdispatch on Darwin
+                    continue
+                  fi
                   LIBDISPATCH_BUILD_DIR=$(build_directory ${host} ${product})
                   echo "--- Running tests for ${product} ---"
                   with_pushd "${LIBDISPATCH_BUILD_DIR}" \
@@ -2777,6 +2799,9 @@ for host in "${ALL_HOSTS[@]}"; do
                   continue
                 ;;
                 *)
+                  if [[ "${product}" = clibdispatch ]]; then
+                    product=libdispatch
+                  fi
                   results_targets=( "test" )
                   executable_target=""
                 ;;
@@ -2951,13 +2976,21 @@ for host in "${ALL_HOSTS[@]}"; do
                 fi
 
                 ;;
-            libdispatch|libdispatch_static)
-                if [[ -z "${INSTALL_LIBDISPATCH}" ]] ; then
+            clibdispatch|libdispatch|libdispatch_static)
+                if [[ "${product}" = clibdispatch ]]; then
+                  if [[ -z "${INSTALL_CLIBDISPATCH}" ]]; then
+                    continue
+                  fi
+                elif [[ -z "${INSTALL_LIBDISPATCH}" ]]; then
                     continue
                 fi
 
                 case "${host}" in
                 macosx-*)
+                  if [[ "${product}" = clibdispatch ]]; then
+                    # we don't build clibdispatch on Darwin
+                    continue
+                  fi
                   if [[ -z "${INSTALL_DESTDIR}" ]] ; then
                       echo "--install-destdir is required to install products."
                       exit 1
@@ -2976,6 +3009,9 @@ for host in "${ALL_HOSTS[@]}"; do
                   continue
                 ;;
                 *)
+                  if [[ "${product}" = clibdispatch ]]; then
+                    product=libdispatch
+                  fi
                 ;;
                 esac
                 ;;

utils/build_swift/build_swift/driver_arguments.py

@@ -652,6 +652,9 @@ def create_argument_parser():
     option('--libdispatch', toggle_true('build_libdispatch'),
            help='build libdispatch')
 
+    option('--clibdispatch', toggle_true('build_clibdispatch'),
+           help='build clibdispatch')
+
     option('--libicu', toggle_true('build_libicu'),
            help='build libicu')
 

utils/build_swift/tests/expected_options.py

@@ -70,6 +70,7 @@
     'build_ios_simulator': False,
     'build_jobs': multiprocessing.cpu_count(),
     'build_libdispatch': False,
+    'build_clibdispatch': False,
     'build_libicu': False,
     'build_linux': True,
     'build_llbuild': False,
@@ -522,6 +523,7 @@ class BuildScriptImplOption(_BaseOption):
     EnableOption('--only-executable-test'),
     EnableOption('--only-non-executable-test'),
     EnableOption('--libdispatch', dest='build_libdispatch'),
+    EnableOption('--clibdispatch', dest='build_clibdispatch'),
     EnableOption('--libicu', dest='build_libicu'),
     EnableOption('--indexstore-db', dest='build_indexstoredb'),
     EnableOption('--test-indexstore-db-sanitize-all',

utils/swift_build_support/swift_build_support/products/__init__.py

@@ -11,6 +11,7 @@
 # ----------------------------------------------------------------------------
 
 from .benchmarks import Benchmarks
+from .clibdispatch import CLibDispatch
 from .cmark import CMark
 from .foundation import Foundation
 from .indexstoredb import IndexStoreDB
@@ -39,6 +40,7 @@
     'Ninja',
     'Foundation',
     'LibCXX',
+    'CLibDispatch',
     'LibDispatch',
     'LibICU',
     'LLBuild',

utils/swift_build_support/swift_build_support/products/clibdispatch.py

@@ -0,0 +1,44 @@
+# swift_build_support/products/clibdispatch.py -------------------*- python -*-
+#
+# This source file is part of the Swift.org open source project
+#
+# Copyright (c) 2014 - 2021 Apple Inc. and the Swift project authors
+# Licensed under Apache License v2.0 with Runtime Library Exception
+#
+# See https://swift.org/LICENSE.txt for license information
+# See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+#
+# ----------------------------------------------------------------------------
+
+from . import cmark
+from . import libcxx
+from . import llvm
+from . import product
+
+
+class CLibDispatch(product.Product):
+    @classmethod
+    def is_build_script_impl_product(cls):
+        """is_build_script_impl_product -> bool
+
+        Whether this product is produced by build-script-impl.
+        """
+        return True
+
+    @classmethod
+    def product_source_name(cls):
+        """product_source_name() -> str
+
+        The name of the source code directory of this product.
+        """
+        return "swift-corelibs-libdispatch"
+
+    @classmethod
+    def is_nondarwin_only_build_product(cls):
+        return True
+
+    @classmethod
+    def get_dependencies(cls):
+        return [cmark.CMark,
+                llvm.LLVM,
+                libcxx.LibCXX]

utils/swift_build_support/swift_build_support/products/swift.py

@@ -10,6 +10,7 @@
 #
 # ----------------------------------------------------------------------------
 
+from . import clibdispatch
 from . import cmark
 from . import libcxx
 from . import libicu
@@ -147,4 +148,5 @@ def get_dependencies(cls):
         return [cmark.CMark,
                 llvm.LLVM,
                 libcxx.LibCXX,
-                libicu.LibICU]
+                libicu.LibICU,
+                clibdispatch.CLibDispatch]