[build-script][lto] Migrate the num_*_parallel_lto_link_jobs code from build-script-impl into build support.

rdar://24717107

Author: gottesmm

utils/build-script

@@ -37,6 +37,7 @@ sys.path.append(os.path.join(os.path.dirname(__file__), 'swift_build_support'))
 # E402 means module level import not at top of file
 from swift_build_support import arguments  # noqa (E402)
 from swift_build_support import diagnostics  # noqa (E402)
+from swift_build_support import host  # noqa (E402)
 from swift_build_support.toolchain import host_toolchain  # noqa (E402)
 import swift_build_support.debug      # noqa (E402)
 from swift_build_support import migration  # noqa (E402)
@@ -986,6 +987,21 @@ details of the setups of other systems or automated environments.""")
         default=False,
         const=True)
 
+    default_max_lto_link_job_counts = host.max_lto_link_job_counts()
+    parser.add_argument(
+        "--llvm-max-parallel-lto-link-jobs",
+        help="the maximum number of parallel link jobs to use when compiling "
+             "llvm",
+        metavar="COUNT",
+        default=default_max_lto_link_job_counts['llvm'])
+
+    parser.add_argument(
+        "--swift-tools-max-parallel-lto-link-jobs",
+        help="the maximum number of parallel link jobs to use when compiling "
+             "swift tools.",
+        metavar="COUNT",
+        default=default_max_lto_link_job_counts['swift'])
+
     parser.add_argument(
         # Explicitly unavailable options here.
         "--build-jobs",
@@ -1457,7 +1473,17 @@ details of the setups of other systems or automated environments.""")
             "--llvm-enable-lto",
             "--swift-tools-enable-lto"
         ]
-
+        if args.llvm_max_parallel_lto_link_jobs is not None:
+            build_script_impl_args += [
+                "--llvm-num-parallel-lto-link-jobs=%s" %
+                min(args.llvm_max_parallel_lto_link_jobs, args.build_jobs)
+            ]
+        if args.swift_tools_max_parallel_lto_link_jobs is not None:
+            build_script_impl_args += [
+                "--swift-tools-num-parallel-lto-link-jobs=%s" %
+                min(args.swift_tools_max_parallel_lto_link_jobs,
+                    args.build_jobs)
+            ]
     build_script_impl_args += args.build_script_impl_args
 
     if args.dry_run:

utils/build-script-impl

@@ -76,6 +76,8 @@ KNOWN_SETTINGS=(
     swift-analyze-code-coverage "not-merged"     "Code coverage analysis mode for Swift (false, not-merged, merged). Defaults to false if the argument is not present, and not-merged if the argument is present without a modifier."
     swift-tools-enable-lto      "0"              "enable LTO compilation of Swift tools. *NOTE* This does not include the swift standard library and runtime."
     llvm-enable-lto             "0"              "enable LTO compilation of LLVM/Clang."
+    swift-tools-num-parallel-lto-link-jobs ""    "The number of parallel link jobs to use when compiling swift tools"
+    llvm-num-parallel-lto-link-jobs ""           "The number of parallel link jobs to use when compiling llvm"
     swift-stdlib-build-type     "Debug"          "the CMake build variant for Swift"
     swift-stdlib-enable-assertions "1"           "enable assertions in Swift"
     swift-stdlib-enable-resilience "0"           "build the Swift stdlib and overlays with resilience enabled"
@@ -287,28 +289,6 @@ function set_lldb_build_mode() {
     LLDB_BUILD_MODE="CustomSwift-${LLDB_BUILD_TYPE}"
 }
 
-function system_memory_in_bytes() {
-    case "$(uname -s -m)" in
-        Darwin\ x86_64)
-            sysctl hw.memsize | cut -f 2 -d " "
-            ;;
-        *)
-            echo "Unknown operating system"
-            exit 1
-            ;;
-    esac
-}
-
-function float_min() {
-    local LHS=$1
-    local RHS=$2
-    if (( $(echo "${LHS} < ${RHS}" | bc -l) )); then
-        echo ${LHS}
-    else
-        echo ${RHS}
-    fi
-}
-
 # Support for performing isolated actions.
 #
 # This is part of refactoring more work to be done or controllable via
@@ -354,48 +334,6 @@ function should_execute_host_actions_for_phase() {
     fi
 }
 
-function num_llvm_parallel_lto_link_jobs() {
-    case "$(uname -s -m)" in
-        Darwin\ x86_64)
-            # *WARNING! HEURISTIC!*
-            #
-            # Use the formula (GB Memory - 3)/6.0GB to get the number of
-            # parallel link threads we can support. This gives the OS 3 GB of
-            # room to work with.
-            #
-            # This is a bit conservative, but I have found that this number
-            # prevents me from swapping on my test machine.
-            local NUM_MEMORY_THREADS=$(echo \( $(system_memory_in_bytes)/1000000000.0 - 3.0 \) / 6.0 | bc)
-            echo $(float_min "${NUM_MEMORY_THREADS}" "${BUILD_JOBS}")
-            ;;
-        *)
-            echo "Unknown operating system"
-            exit 1
-            ;;
-    esac
-}
-
-function num_swift_parallel_lto_link_jobs() {
-    case "$(uname -s -m)" in
-        Darwin\ x86_64)
-            # *WARNING! HEURISTIC!*
-            #
-            # Use the formula (GB Memory - 3)/8.0GB to get the number of
-            # parallel link threads we can support. This gives the OS 3 GB of
-            # room to work with.
-            #
-            # This is a bit conservative, but I have found that this number
-            # prevents me from swapping on my test machine.
-            local NUM_MEMORY_THREADS=$(echo \( $(system_memory_in_bytes)/1000000000 - 3.0 \) / 8.0 | bc)
-            echo $(float_min "${NUM_MEMORY_THREADS}" "${BUILD_JOBS}")
-            ;;
-        *)
-            echo "Unknown operating system"
-            exit 1
-            ;;
-    esac
-}
-
 function set_build_options_for_host() {
     llvm_cmake_options=()
     swift_cmake_options=()
@@ -638,7 +576,7 @@ function set_build_options_for_host() {
                 fi
 
                 llvm_cmake_options+=(
-                    "-DLLVM_PARALLEL_LINK_JOBS=$(num_llvm_parallel_lto_link_jobs)"
+                    "-DLLVM_PARALLEL_LINK_JOBS=${LLVM_NUM_PARALLEL_LTO_LINK_JOBS}"
                 )
             fi
 
@@ -659,7 +597,7 @@ function set_build_options_for_host() {
                     )
                 fi
                 swift_cmake_options+=(
-                    "-DSWIFT_PARALLEL_LINK_JOBS=$(num_swift_parallel_lto_link_jobs)"
+                    "-DSWIFT_PARALLEL_LINK_JOBS=${SWIFT_TOOLS_NUM_PARALLEL_LTO_LINK_JOBS}"
                 )
             fi
 

utils/swift_build_support/swift_build_support/host.py

@@ -0,0 +1,89 @@
+# swift_build_support/host.py ----------- Migrating build-script -*- python -*-
+#
+# This source file is part of the Swift.org open source project
+#
+# Copyright (c) 2014 - 2016 Apple Inc. and the Swift project authors
+# Licensed under Apache License v2.0 with Runtime Library Exception
+#
+# See http://swift.org/LICENSE.txt for license information
+# See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+#
+# -----------------------------------------------------------------------------
+#
+# This file contains routines for determining information about the host for
+# use in utils/build-script.
+#
+# -----------------------------------------------------------------------------
+
+from __future__ import absolute_import
+
+import platform
+
+from . import shell
+
+
+# Utilities
+def _return_none_fun():
+    return None
+
+
+def _return_none_fun_pair():
+    return (_return_none_fun, _return_none_fun)
+
+
+def _compute_system_key():
+    return (platform.system(), platform.machine())
+
+
+# System Memory
+def _darwin_system_memory():
+    # Output looks like "hw.memsize: \d+\n"
+    return int(shell.capture(["sysctl", "hw.memsize"],
+                             dry_run=False, echo=False,
+                             optional=False).strip().split(" ")[1])
+
+_PER_PLATFORM_SYSTEM_MEMORY = {
+    ('Darwin', 'x86_64'): _darwin_system_memory
+}
+
+
+def system_memory():
+    return _PER_PLATFORM_SYSTEM_MEMORY.get(_compute_system_key(),
+                                           _return_none_fun)()
+
+
+# Max Num CPU Threads for use with LTO
+def _darwin_max_num_llvm_parallel_lto_link_jobs():
+    # *WARNING! HEURISTIC!*
+    #
+    # Use the formula (GB Memory - 3)/6.0GB to get the number of
+    # parallel link threads we can support. This gives the OS 3 GB of
+    # room to work with.
+    #
+    # This is a bit conservative, but I have found that this number
+    # prevents me from swapping on my test machine.
+    return int((_darwin_system_memory()/1000000000.0 - 3.0)/6.0)
+
+
+def _darwin_max_num_swift_parallel_lto_link_jobs():
+    # *WARNING! HEURISTIC!*
+    #
+    # Use the formula (GB Memory - 3)/8.0GB to get the number of
+    # parallel link threads we can support. This gives the OS 3 GB of
+    # room to work with.
+    #
+    # This is a bit conservative, but I have found that this number
+    # prevents me from swapping on my test machine.
+    return int((_darwin_system_memory()/1000000000.0 - 3.0)/8.0)
+
+_PER_PLATFORM_MAX_PARALLEL_LTO_JOBS = {
+    ('Darwin', 'x86_64'): (_darwin_max_num_llvm_parallel_lto_link_jobs,
+                           _darwin_max_num_swift_parallel_lto_link_jobs)
+}
+
+
+def max_lto_link_job_counts():
+    key = _compute_system_key()
+    info = _PER_PLATFORM_MAX_PARALLEL_LTO_JOBS.get(key,
+                                                   _return_none_fun_pair())
+    return {'llvm': info[0](), 'swift': info[1]()}

utils/swift_build_support/tests/test_host.py

@@ -0,0 +1,61 @@
+# test_host.py - Unit tests for swift_build_support.cmake -*-- python -*-
+#
+# This source file is part of the Swift.org open source project
+#
+# Copyright (c) 2014 - 2016 Apple Inc. and the Swift project authors
+# Licensed under Apache License v2.0 with Runtime Library Exception
+#
+# See http://swift.org/LICENSE.txt for license information
+# See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+
+import platform
+import unittest
+
+import swift_build_support.host as sbs_host
+
+
+class HostTestCase(unittest.TestCase):
+
+    def test_system_memory(self):
+        # We make sure that we get an integer back. If we get an integer back,
+        # we know that we at least were able to get some sort of information
+        # from the system and it could be parsed as an integer. This is just a
+        # smoke test.
+        supported_platforms = [('Darwin', 'x86_64')]
+
+        mem = sbs_host.system_memory()
+
+        if (platform.system(), platform.machine()) not in supported_platforms:
+            self.assertIsNone(mem)
+        else:
+            self.assertIsInstance(mem, int)
+
+    def test_lto_link_job_counts(self):
+        # Make sure that:
+        #
+        # 1. we get back a dictionary with two keys in it, the first called
+        # llvm, the other called swift.
+        #
+        # 2. The values associated with these keys is either None (if we do not
+        # support the platform) or is an int that is reasonable (i.e. <
+        # 100). The number 100 is just a heuristic number that is appropriate
+        # currently since LTO uses so much memory. If and when that changes,
+        # this number should change.
+        supported_platforms = [('Darwin', 'x86_64')]
+        reasonable_upper_bound_of_lto_threads = 100
+
+        result = sbs_host.max_lto_link_job_counts()
+        self.assertIsInstance(result, dict)
+        self.assertEqual(len(result), 2)
+
+        if (platform.system(), platform.machine()) not in supported_platforms:
+            self.assertIsNone(result['llvm'])
+            self.assertIsNone(result['swift'])
+            return
+
+        self.assertIsNotNone(result['llvm'])
+        self.assertIsNotNone(result['swift'])
+        self.assertIsInstance(result['llvm'], int)
+        self.assertIsInstance(result['swift'], int)
+        self.assertLess(result['llvm'], reasonable_upper_bound_of_lto_threads)
+        self.assertLess(result['swift'], reasonable_upper_bound_of_lto_threads)