[build-script][SR-237] Migrated calculation of BUILD_ARGS to Python.

Calculate `BUILD_ARGS` in `build-script`, then pass them to
`build-script-impl` using existing `--build-args` argument.

Author: rintaro

utils/build-script

@@ -849,6 +849,19 @@ details of the setups of other systems or automated environments.""")
         type=argparse_shell_split,
         default=[])
 
+    parser.add_argument(
+        "--build-args",
+        help="arguments to the build tool. This would be prepended to the "
+             "default argument that is '-j8' when CMake generator is "
+             "\"Ninja\".",
+        type=argparse_shell_split,
+        default=[])
+
+    parser.add_argument(
+        "--verbose-build",
+        help="print the commands executed during the build",
+        action="store_true")
+
     args = migration.parse_args(parser, sys.argv[1:])
 
     build_script_impl = os.path.join(
@@ -1258,6 +1271,8 @@ details of the setups of other systems or automated environments.""")
         "--workspace", SWIFT_SOURCE_ROOT,
         "--common-cmake-options=%s" % ' '.join(
             pipes.quote(opt) for opt in cmake.common_options()),
+        "--build-args=%s" % ' '.join(
+            pipes.quote(arg) for arg in cmake.build_args()),
     ]
 
     if args.distcc:
@@ -1273,6 +1288,8 @@ details of the setups of other systems or automated environments.""")
         build_script_impl_args += [
             "--clang-compiler-version", args.clang_compiler_version
         ]
+    if args.verbose_build:
+        build_script_impl_args += ["--verbose-build"]
     if args.install_symroot:
         build_script_impl_args += [
             "--install-symroot", os.path.abspath(args.install_symroot)

utils/build-script-impl

@@ -774,6 +774,12 @@ if [[ "${SKIP_BUILD_WATCHOS_SIMULATOR}" ]] ; then
     SKIP_TEST_WATCHOS_SIMULATOR=1
 fi
 
+# FIXME: We currently do not support building compiler-rt with the
+# Xcode generator.
+if [[ "${CMAKE_GENERATOR}" == "Xcode" ]]; then
+    SKIP_COMPILER_RT=1
+fi
+
 # FIXME: We currently do not support cross-compiling swift with compiler-rt.
 if [[ "${CROSS_COMPILE_TOOLS_DEPLOYMENT_TARGETS}" ]]; then
     SKIP_COMPILER_RT=1
@@ -1131,39 +1137,12 @@ COMMON_C_FLAGS=""
 # Convert to an array.
 eval COMMON_CMAKE_OPTIONS=(${COMMON_CMAKE_OPTIONS})
 eval EXTRA_CMAKE_OPTIONS=(${EXTRA_CMAKE_OPTIONS})
+eval BUILD_ARGS=(${BUILD_ARGS})
 
-CMAKE_JOBS="${BUILD_JOBS}"
-if [[ "${DISTCC}" ]] ; then
-    CMAKE_JOBS="$(distcc -j)"
+if [[ "${CMAKE_GENERATOR}" == "Xcode" ]]; then
+    BUILD_TARGET_FLAG="-target"
 fi
 
-case "${CMAKE_GENERATOR}" in
-    Ninja)
-        BUILD_ARGS="${BUILD_ARGS} -j${CMAKE_JOBS}"
-        if [[ "${VERBOSE_BUILD}" ]] ; then
-            BUILD_ARGS="${BUILD_ARGS} -v"
-        fi
-        ;;
-    'Unix Makefiles')
-        BUILD_ARGS="${BUILD_ARGS} -j${CMAKE_JOBS}"
-        if [[ "${VERBOSE_BUILD}" ]] ; then
-            BUILD_ARGS="${BUILD_ARGS} VERBOSE=1"
-        fi
-        ;;
-    Xcode)
-        # -parallelizeTargets is an unsupported flag from the Xcode 3 days,
-        # but since we're not using proper Xcode 4 schemes, this is the
-        # only way to get target-level parallelism.
-        BUILD_ARGS="${BUILD_ARGS} -parallelizeTargets"
-        BUILD_ARGS="${BUILD_ARGS} -jobs ${CMAKE_JOBS}"
-        BUILD_TARGET_FLAG="-target"
-
-        # FIXME: We currently do not support building compiler-rt with the
-        # Xcode generator.
-        SKIP_COMPILER_RT=1
-        ;;
-esac
-
 function build_directory() {
     deployment_target=$1
     product=$2
@@ -2001,7 +1980,7 @@ for deployment_target in "${HOST_TARGET}" "${CROSS_TOOLS_DEPLOYMENT_TARGETS[@]}"
             fi
 
             set -x
-            ${DISTCC_PUMP} "${CMAKE}" --build "${build_dir}" $(cmake_config_opt ${product}) -- ${BUILD_ARGS} ${build_targets[@]}
+            ${DISTCC_PUMP} "${CMAKE}" --build "${build_dir}" $(cmake_config_opt ${product}) -- "${BUILD_ARGS[@]}" ${build_targets[@]}
             { set +x; } 2>/dev/null
         fi
 

utils/swift_build_support/swift_build_support/cmake.py

@@ -14,9 +14,10 @@
 #
 # ----------------------------------------------------------------------------
 
-import re
-import platform
 from numbers import Number
+import platform
+import re
+import subprocess
 
 from . import xcrun
 from .which import which
@@ -136,3 +137,30 @@ def common_options(self):
                 define("LLVM_VERSION_PATCH:STRING", m.group(3))
 
         return options
+
+    def build_args(self):
+        """Return arguments to the build tool used for all products
+        """
+        args = self.args
+        jobs = args.build_jobs
+        if args.distcc:
+            jobs = str(
+                subprocess.check_output([self.host_distcc, '-j'])).rstrip()
+
+        build_args = list(args.build_args)
+
+        if args.cmake_generator == 'Ninja':
+            build_args += ['-j%s' % jobs]
+            if args.verbose_build:
+                build_args += ['-v']
+
+        elif args.cmake_generator == 'Unix Makefiles':
+            build_args += ['-j%s' % jobs]
+            if args.verbose_build:
+                build_args += ['VERBOSE=1']
+
+        elif args.cmake_generator == 'Xcode':
+            build_args += ['-parallelizeTargets',
+                           '-jobs', str(jobs)]
+
+        return build_args

utils/swift_build_support/tests/mock-distcc

@@ -0,0 +1,25 @@
+#!/usr/bin/env python
+# mock-distcc - discc mock used from tests ----------------------*- python -*-
+#
+# This source file is part of the Swift.org open source project
+#
+# Copyright (c) 2014 - 2016 Apple Inc. and the Swift project authors
+# Licensed under Apache License v2.0 with Runtime Library Exception
+#
+# See http://swift.org/LICENSE.txt for license information
+# See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+
+from __future__ import print_function
+
+import sys
+
+
+try:
+    if sys.argv[1] == '-j':
+        print('6')
+        sys.exit(0)
+except Exception:
+    pass
+
+print("Error: invalid option", file=sys.stderr)
+sys.exit(1)

utils/swift_build_support/tests/test_cmake.py

@@ -29,23 +29,37 @@ def test_cmake_available_on_this_platform(self):
 
 class CMakeTestCase(unittest.TestCase):
 
+    def mock_distcc_path(self):
+        """Return a path string of mock distcc executable
+        """
+        return os.path.join(os.path.dirname(__file__),
+                            'mock-distcc')
+
     def default_args(self):
-        "Return new args object with default values"
+        """Return new args object with default values
+        """
         return Namespace(host_cc="/path/to/clang",
                          host_cxx="/path/to/clang++",
                          enable_asan=False,
                          enable_ubsan=False,
                          export_compile_commands=False,
                          distcc=False,
                          cmake_generator="Ninja",
-                         clang_compiler_version=None)
+                         clang_compiler_version=None,
+                         build_jobs=8,
+                         build_args=[],
+                         verbose_build=False)
 
     def cmake(self, args):
-        "Return new CMake object initialized with given args"
+        """Return new CMake object initialized with given args
+        """
+        host_distcc = None
+        if args.distcc:
+            host_distcc = self.mock_distcc_path()
         return CMake(args=args,
                      host_cc=args.host_cc,
                      host_cxx=args.host_cxx,
-                     host_distcc="/path/to/distcc")
+                     host_distcc=host_distcc)
 
     def test_common_options_defaults(self):
         args = self.default_args()
@@ -108,9 +122,9 @@ def test_common_options_distcc(self):
         self.assertEqual(
             list(cmake.common_options()),
             ["-G", "Ninja",
-             "-DCMAKE_C_COMPILER:PATH=/path/to/distcc",
+             "-DCMAKE_C_COMPILER:PATH=" + self.mock_distcc_path(),
              "-DCMAKE_C_COMPILER_ARG1=/path/to/clang",
-             "-DCMAKE_CXX_COMPILER:PATH=/path/to/distcc",
+             "-DCMAKE_CXX_COMPILER:PATH=" + self.mock_distcc_path(),
              "-DCMAKE_CXX_COMPILER_ARG1=/path/to/clang++"])
 
     def test_common_options_xcode(self):
@@ -152,16 +166,84 @@ def test_common_options_full(self):
             ["-G", "Xcode",
              "-DLLVM_USE_SANITIZER=Address;Undefined",
              "-DCMAKE_EXPORT_COMPILE_COMMANDS=ON",
-             "-DCMAKE_C_COMPILER:PATH=/path/to/distcc",
+             "-DCMAKE_C_COMPILER:PATH=" + self.mock_distcc_path(),
              "-DCMAKE_C_COMPILER_ARG1=/path/to/clang",
-             "-DCMAKE_CXX_COMPILER:PATH=/path/to/distcc",
+             "-DCMAKE_CXX_COMPILER:PATH=" + self.mock_distcc_path(),
              "-DCMAKE_CXX_COMPILER_ARG1=/path/to/clang++",
              "-DCMAKE_CONFIGURATION_TYPES=" +
              "Debug;Release;MinSizeRel;RelWithDebInfo",
              "-DLLVM_VERSION_MAJOR:STRING=3",
              "-DLLVM_VERSION_MINOR:STRING=8",
              "-DLLVM_VERSION_PATCH:STRING=0"])
 
+    def test_build_args_ninja(self):
+        args = self.default_args()
+        cmake = self.cmake(args)
+        self.assertEqual(
+            list(cmake.build_args()),
+            ["-j8"])
+
+        args.verbose_build = True
+        cmake = self.cmake(args)
+        self.assertEqual(
+            list(cmake.build_args()),
+            ["-j8", "-v"])
+
+    def test_build_args_makefile(self):
+        args = self.default_args()
+        args.cmake_generator = "Unix Makefiles"
+        cmake = self.cmake(args)
+        self.assertEqual(
+            list(cmake.build_args()),
+            ["-j8"])
+
+        args.verbose_build = True
+        cmake = self.cmake(args)
+        self.assertEqual(
+            list(cmake.build_args()),
+            ["-j8", "VERBOSE=1"])
+
+    def test_build_args_xcode(self):
+        args = self.default_args()
+        args.cmake_generator = "Xcode"
+        cmake = self.cmake(args)
+        self.assertEqual(
+            list(cmake.build_args()),
+            ["-parallelizeTargets", "-jobs", "8"])
+
+        # NOTE: Xcode generator DOES NOT take 'verbose-build' into account.
+        args.verbose_build = True
+        cmake = self.cmake(args)
+        self.assertEqual(
+            list(cmake.build_args()),
+            ["-parallelizeTargets", "-jobs", "8"])
+
+    def test_build_args_eclipse_ninja(self):
+        # NOTE: Eclipse generator DOES NOT take 'build-jobs' into account,
+        #       nor 'verbose-build'.
+        args = self.default_args()
+        args.cmake_generator = "Eclipse CDT4 - Ninja"
+        args.verbose_build = True
+        cmake = self.cmake(args)
+        self.assertEqual(
+            list(cmake.build_args()), [])
+
+    def test_build_args_custom_build_args(self):
+        args = self.default_args()
+        args.build_args = ["-foo", "bar baz"]
+        cmake = self.cmake(args)
+        self.assertEqual(
+            list(cmake.build_args()),
+            ["-foo", "bar baz", "-j8"])
+
+    def test_build_args_distcc(self):
+        args = self.default_args()
+        args.distcc = True
+        cmake = self.cmake(args)
+        self.assertEqual(
+            list(cmake.build_args()),
+            ["-j6"])
+
 
 class CMakeOptionsTestCase(unittest.TestCase):
     def test_define(self):