Merge pull request #2352 from rintaro/build-script-refactor-toolchain

Author: swift-ci

utils/build-script

@@ -42,11 +42,9 @@ from SwiftBuildSupport import (
 sys.path.append(os.path.join(os.path.dirname(__file__), 'swift_build_support'))
 
 # E402 means module level import not at top of file
-import swift_build_support.toolchain  # noqa (E402)
-import swift_build_support.cmake      # noqa (E402)
+from swift_build_support.toolchain import host_toolchain  # noqa (E402)
 import swift_build_support.debug      # noqa (E402)
 from swift_build_support import migration  # noqa (E402)
-import swift_build_support.ninja      # noqa (E402)
 import swift_build_support.tar        # noqa (E402)
 import swift_build_support.targets    # noqa (E402)
 from swift_build_support.cmake import CMake  # noqa (E402)
@@ -87,6 +85,15 @@ def argparse_clang_compiler_version(string):
         "%r is invalid version value. must be 'MAJOR.MINOR.PATCH'" % string)
 
 
+# Check the string is executable path string.
+# Convert it to absolute path.
+def argparse_executable(string):
+    if os.access(string, os.X_OK):
+        return os.path.abspath(string)
+    raise argparse.ArgumentTypeError(
+        "%r is not execuable" % string)
+
+
 # Main entry point for the preset mode.
 def main_preset():
     parser = argparse.ArgumentParser(
@@ -773,7 +780,9 @@ details of the setups of other systems or automated environments.""")
     parser.add_argument(
         "--cmake",
         help="the path to a CMake executable that will be used to build "
-             "Swift")
+             "Swift",
+        type=argparse_executable,
+        metavar="PATH")
     parser.add_argument(
         "--show-sdks",
         help="print installed Xcode and SDK versions",
@@ -828,11 +837,13 @@ details of the setups of other systems or automated environments.""")
         "--host-cc",
         help="the absolute path to CC, the 'clang' compiler for the host "
              "platform. Default is auto detected.",
+        type=argparse_executable,
         metavar="PATH")
     parser.add_argument(
         "--host-cxx",
         help="the absolute path to CXX, the 'clang++' compiler for the host "
              "platform. Default is auto detected.",
+        type=argparse_executable,
         metavar="PATH")
     parser.add_argument(
         "--distcc",
@@ -897,6 +908,35 @@ details of the setups of other systems or automated environments.""")
         if '--check-args-only' in args.build_script_impl_args:
             return 0
 
+    # Prepare and validate toolchain
+    toolchain = host_toolchain(xcrun_toolchain=args.darwin_xcrun_toolchain)
+
+    if args.host_cc is not None:
+        toolchain.cc = args.host_cc
+    if args.host_cxx is not None:
+        toolchain.cxx = args.host_cxx
+    if args.cmake is not None:
+        toolchain.cmake = args.cmake
+
+    if toolchain.cc is None or toolchain.cxx is None:
+        print_with_argv0(
+            "Can't find clang. Please install clang-3.5 or a later version.")
+        return 1
+
+    if toolchain.cmake is None:
+        print_with_argv0("Can't find CMake. Please install CMake.")
+        return 1
+
+    if args.distcc:
+        if toolchain.distcc is None:
+            print_with_argv0(
+                "Can't find distcc. Please install distcc")
+            return 1
+        if toolchain.distcc_pump is None:
+            print_with_argv0(
+                "Can't find distcc-pump. Please install distcc-pump")
+            return 1
+
     if args.host_target is None or args.stdlib_deployment_targets is None:
         print_with_argv0("Unknown operating system.")
         return 1
@@ -1221,61 +1261,19 @@ details of the setups of other systems or automated environments.""")
     if args.clean and os.path.isdir(build_dir):
         shutil.rmtree(build_dir)
 
-    host_clang = swift_build_support.toolchain.host_clang(
-        xcrun_toolchain=args.darwin_xcrun_toolchain)
-    if args.host_cc is None and host_clang is not None:
-        args.host_cc = str(host_clang.cc)
-    if args.host_cxx is None and host_clang is not None:
-        args.host_cxx = str(host_clang.cxx)
-    is_host_clang_ok = (
-        args.host_cc is not None and os.access(args.host_cc, os.X_OK) and
-        args.host_cxx is not None and os.access(args.host_cxx, os.X_OK))
-    if not is_host_clang_ok:
-        print_with_argv0(
-            "Can't find clang. Please install clang-3.5 or a later version.")
-        return 1
-
-    host_cmake = args.cmake
-    if not host_cmake:
-        host_cmake = swift_build_support.cmake.host_cmake(
-            args.darwin_xcrun_toolchain)
-    if not host_cmake:
-        print_with_argv0("Can't find CMake. Please install CMake.")
-        return 1
-
-    # distcc usage.
-    host_distcc = None
-    host_distcc_pump = None
-    if args.distcc:
-        host_distcc = swift_build_support.which('distcc')
-        # On some platforms, 'pump' may be unrelated to distcc, in which case
-        # it's called 'distcc-pump'.
-        host_distcc_pump = swift_build_support.which('distcc-pump')
-        if host_distcc_pump is None:
-            host_distcc_pump = swift_build_support.which('pump')
-
-        if host_distcc is None or host_distcc_pump is None:
-            print_with_argv0(
-                "Can't find distcc. Please install distcc")
-
-        host_distcc = str(host_distcc)
-        host_distcc_pump = str(host_distcc_pump)
-
     cmake = CMake(args=args,
-                  host_cc=args.host_cc,
-                  host_cxx=args.host_cxx,
-                  host_distcc=host_distcc)
+                  toolchain=toolchain)
 
     build_script_impl_args = [
         "--build-dir", build_dir,
         "--install-prefix", os.path.abspath(args.install_prefix),
         "--host-target", args.host_target,
         "--stdlib-deployment-targets",
         " ".join(args.stdlib_deployment_targets),
-        "--host-cc", args.host_cc,
-        "--host-cxx", args.host_cxx,
+        "--host-cc", toolchain.cc,
+        "--host-cxx", toolchain.cxx,
         "--darwin-xcrun-toolchain", args.darwin_xcrun_toolchain,
-        "--cmake", host_cmake,
+        "--cmake", toolchain.cmake,
         "--cmark-build-type", args.cmark_build_variant,
         "--llvm-build-type", args.llvm_build_variant,
         "--swift-build-type", args.swift_build_variant,
@@ -1299,7 +1297,7 @@ details of the setups of other systems or automated environments.""")
     if args.distcc:
         build_script_impl_args += [
             "--distcc",
-            "--distcc-pump=%s" % host_distcc_pump
+            "--distcc-pump=%s" % toolchain.distcc_pump
         ]
     if args.enable_asan:
         build_script_impl_args += ["--enable-asan"]
@@ -1319,8 +1317,7 @@ details of the setups of other systems or automated environments.""")
         build_script_impl_args += [
             "--foundation-build-type", args.foundation_build_variant
         ]
-    if args.cmake_generator == 'Ninja' and \
-       not swift_build_support.ninja.is_ninja_installed():
+    if args.cmake_generator == 'Ninja' and toolchain.ninja is None:
         build_script_impl_args += ["--build-ninja"]
 
     if args.skip_build_linux:

utils/profdata_merge/process.py

@@ -24,11 +24,11 @@
 sys.path.append(parent_dir)
 support_dir = os.path.join(parent_dir, 'swift_build_support')
 sys.path.append(support_dir)
-from swift_build_support import xcrun  # noqa (E402)
+from swift_build_support.toolchain import host_toolchain  # noqa (E402)
 from SwiftBuildSupport import check_output, check_call  # noqa (E402)
 
-# FIXME: This doesn't work on non-Darwin platforms.
-LLVM_PROFDATA_PATH = xcrun.find('default', 'llvm-profdata')
+toolchain = host_toolchain()
+LLVM_PROFDATA_PATH = toolchain.llvm_profdata
 _profdata_help = check_output([LLVM_PROFDATA_PATH, 'merge', '-help'])
 LLVM_PROFDATA_SUPPORTS_SPARSE = 'sparse' in _profdata_help
 

utils/swift_build_support/swift_build_support/cache_util.py

@@ -0,0 +1,58 @@
+# swift_build_support/cache_util.py -----------------------------*- python -*-
+#
+# This source file is part of the Swift.org open source project
+#
+# Copyright (c) 2014 - 2016 Apple Inc. and the Swift project authors
+# Licensed under Apache License v2.0 with Runtime Library Exception
+#
+# See http://swift.org/LICENSE.txt for license information
+# See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+#
+# ----------------------------------------------------------------------------
+"""
+Cache related utilities
+"""
+# ----------------------------------------------------------------------------
+
+from functools import update_wrapper
+
+__all__ = [
+    'cached',
+    'reify'
+]
+
+
+def cached(func):
+    """Decorator that caches result of method or function.
+
+    Note: Support method or function.
+    """
+    cache = {}
+
+    def wrapper(*args, **kwargs):
+        key = tuple(args) + tuple(kwargs.items())
+        if key not in cache:
+            result = func(*args, **kwargs)
+            cache[key] = result
+            return result
+        else:
+            return cache[key]
+
+    return update_wrapper(wrapper, func)
+
+
+def reify(func):
+    """Decorator that replaces the wrapped method with the result after the
+    first call.
+
+    Note: Support method that takes no arguments.
+    """
+    class wrapper(object):
+        def __get__(self, obj, objtype=None):
+            if obj is None:
+                return self
+            result = func(obj)
+            setattr(obj, func.__name__, result)
+            return result
+
+    return update_wrapper(wrapper(), func)

utils/swift_build_support/swift_build_support/cmake.py

@@ -14,30 +14,10 @@
 #
 # ----------------------------------------------------------------------------
 
-import platform
 import subprocess
 
 from numbers import Number
 
-from . import xcrun
-from .which import which
-
-
-def host_cmake(xcrun_toolchain):
-    """
-    Return the path to `cmake`, using tools provided by the host platform.
-    If `cmake` cannot be found on OS X, return None.
-    If `cmake` cannot be found on Linux, return a probable path.
-    """
-    if platform.system() == 'Darwin':
-        return xcrun.find(xcrun_toolchain, 'cmake')
-    else:
-        cmake = which('cmake')
-        if cmake:
-            return cmake
-        else:
-            return '/usr/local/bin/cmake'
-
 
 class CMakeOptions(object):
     """List like object used to define cmake options
@@ -89,16 +69,15 @@ def __iadd__(self, other):
 
 class CMake(object):
 
-    def __init__(self, args, host_cc, host_cxx, host_distcc):
+    def __init__(self, args, toolchain):
         self.args = args
-        self.host_cc = host_cc
-        self.host_cxx = host_cxx
-        self.host_distcc = host_distcc
+        self.toolchain = toolchain
 
     def common_options(self):
         """Return options used for all products, including LLVM/Clang
         """
         args = self.args
+        toolchain = self.toolchain
         options = CMakeOptions()
         define = options.define
 
@@ -116,13 +95,13 @@ def common_options(self):
             define("CMAKE_EXPORT_COMPILE_COMMANDS", "ON")
 
         if args.distcc:
-            define("CMAKE_C_COMPILER:PATH", self.host_distcc)
-            define("CMAKE_C_COMPILER_ARG1", self.host_cc)
-            define("CMAKE_CXX_COMPILER:PATH", self.host_distcc)
-            define("CMAKE_CXX_COMPILER_ARG1", self.host_cxx)
+            define("CMAKE_C_COMPILER:PATH", toolchain.distcc)
+            define("CMAKE_C_COMPILER_ARG1", toolchain.cc)
+            define("CMAKE_CXX_COMPILER:PATH", toolchain.distcc)
+            define("CMAKE_CXX_COMPILER_ARG1", toolchain.cxx)
         else:
-            define("CMAKE_C_COMPILER:PATH", self.host_cc)
-            define("CMAKE_CXX_COMPILER:PATH", self.host_cxx)
+            define("CMAKE_C_COMPILER:PATH", toolchain.cc)
+            define("CMAKE_CXX_COMPILER:PATH", toolchain.cxx)
 
         if args.cmake_generator == 'Xcode':
             define("CMAKE_CONFIGURATION_TYPES",
@@ -140,10 +119,11 @@ def build_args(self):
         """Return arguments to the build tool used for all products
         """
         args = self.args
+        toolchain = self.toolchain
         jobs = args.build_jobs
         if args.distcc:
-            jobs = str(
-                subprocess.check_output([self.host_distcc, '-j'])).rstrip()
+            jobs = str(subprocess.check_output(
+                [toolchain.distcc, '-j']).decode()).rstrip()
 
         build_args = list(args.build_args)