[benchmarks] Add support for building the benchmarks via build-script against the just created toolchain.

Now one can on Darwin/Linux build the benchmarks via swiftpm from build-script by passing in:

```
build-script $NORMAL_ARGS --install-swift --install-swiftpm --install-llbuild --toolchain-benchmarks --swiftpm --llbuild
```

This is done using the infrastructure that BenL added for sourcekit-lsp.

Author: gottesmm

benchmark/utils/build_script_helper.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python
+
+from __future__ import print_function
+
+import argparse
+import os
+import subprocess
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--verbose', '-v', action='store_true')
+    parser.add_argument('--package-path', type=str, required=True)
+    parser.add_argument('--build-path', type=str, required=True)
+    parser.add_argument('--toolchain', type=str, required=True)
+
+    # Build the debug/release versions.
+    args = parser.parse_args()
+    swiftbuild_path = os.path.join(args.toolchain, 'usr', 'bin', 'swift-build')
+    swiftbuild_args = [
+        swiftbuild_path,
+        '--package-path', args.package_path,
+        '--build-path', args.build_path,
+        '--configuration', 'debug',
+    ]
+    if args.verbose:
+        swiftbuild_args.append('--verbose')
+    subprocess.call(swiftbuild_args)
+
+    swiftbuild_args = [
+        swiftbuild_path,
+        '--package-path', args.package_path,
+        '--build-path', args.build_path,
+        '--configuration', 'release',
+        '-Xswiftc', '-Xllvm',
+        '-Xswiftc', '-align-module-to-page-size',
+    ]
+    if args.verbose:
+        swiftbuild_args.append('--verbose')
+    subprocess.call(swiftbuild_args)
+
+
+if __name__ == "__main__":
+    main()

utils/build-script

@@ -244,11 +244,17 @@ class BuildScriptInvocation(object):
                     "--android-icu-i18n-include, and --android-icu-data "
                     "must be specified")
 
-        if args.legacy_impl and (args.build_indexstoredb or
-                                 args.build_sourcekitlsp):
+        targets_needing_toolchain = [
+            'build_indexstoredb',
+            'build_sourcekitlsp',
+            'build_toolchainbenchmarks',
+        ]
+        has_target_needing_toolchain = \
+            bool(sum(getattr(args, x) for x in targets_needing_toolchain))
+        if args.legacy_impl and has_target_needing_toolchain:
             diagnostics.fatal(
                 "--legacy-impl is incompatible with building packages needing "
-                "a toolchain (indexstore-db or sourcekit-lsp)")
+                "a toolchain (%s)" % ", ".join(targets_needing_toolchain))
 
     @staticmethod
     def apply_default_arguments(toolchain, args):
@@ -879,6 +885,8 @@ class BuildScriptInvocation(object):
             product_classes.append(products.IndexStoreDB)
         if self.args.build_sourcekitlsp:
             product_classes.append(products.SourceKitLSP)
+        if self.args.build_toolchainbenchmarks:
+            product_classes.append(products.ToolchainBenchmarks)
         return product_classes
 
     def execute(self):

utils/build_swift/driver_arguments.py

@@ -546,6 +546,10 @@ def create_argument_parser():
            help='build IndexStoreDB')
     option(['--sourcekit-lsp'], toggle_true('build_sourcekitlsp'),
            help='build SourceKitLSP')
+    option(['--toolchain-benchmarks'],
+           toggle_true('build_toolchainbenchmarks'),
+           help='build Swift Benchmarks using swiftpm against the just built '
+                'toolchain')
 
     option('--xctest', toggle_true('build_xctest'),
            help='build xctest')

utils/build_swift/tests/expected_options.py

@@ -88,6 +88,7 @@
     'build_swiftevolve': False,
     'build_indexstoredb': False,
     'build_sourcekitlsp': False,
+    'build_toolchainbenchmarks': False,
     'build_tvos': True,
     'build_tvos_device': False,
     'build_tvos_simulator': False,
@@ -448,6 +449,7 @@ class IgnoreOption(_BaseOption):
     EnableOption('--libicu', dest='build_libicu'),
     EnableOption('--indexstore-db', dest='build_indexstoredb'),
     EnableOption('--sourcekit-lsp', dest='build_sourcekitlsp'),
+    EnableOption('--toolchain-benchmarks', dest='build_toolchainbenchmarks'),
     EnableOption('--long-test'),
     EnableOption('--show-sdks'),
     EnableOption('--stress-test'),

utils/swift_build_support/swift_build_support/products/__init__.py

@@ -10,6 +10,7 @@
 #
 # ----------------------------------------------------------------------------
 
+from .benchmarks import ToolchainBenchmarks
 from .cmark import CMark
 from .foundation import Foundation
 from .indexstoredb import IndexStoreDB
@@ -47,4 +48,5 @@
     'SwiftEvolve',
     'IndexStoreDB',
     'SourceKitLSP',
+    'ToolchainBenchmarks',
 ]

utils/swift_build_support/swift_build_support/products/benchmarks.py

@@ -0,0 +1,71 @@
+# swift_build_support/products/benchmarks.py --------------------*- python -*-
+#
+# This source file is part of the Swift.org open source project
+#
+# Copyright (c) 2014 - 2019 Apple Inc. and the Swift project authors
+# Licensed under Apache License v2.0 with Runtime Library Exception
+#
+# See https://swift.org/LICENSE.txt for license information
+# See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+#
+# ----------------------------------------------------------------------------
+
+import os
+import platform
+
+from . import product
+from .. import shell
+from .. import targets
+
+
+# Build against the current installed toolchain.
+class ToolchainBenchmarks(product.Product):
+    @classmethod
+    def product_source_name(cls):
+        return "benchmarks"
+
+    @classmethod
+    def is_build_script_impl_product(cls):
+        return False
+
+    def do_build(self, host_target):
+        run_build_script_helper(host_target, self, self.args)
+
+    def do_test(self, host_target):
+        """Just run a single instance of the command for both .debug and
+           .release.
+        """
+        cmdline = ['--num-iters=1', 'XorLoop']
+        debug_bench = os.path.join(self.build_dir, 'debug', 'SwiftBench')
+        shell.call([debug_bench] + cmdline)
+
+        release_bench = os.path.join(self.build_dir, 'release', 'SwiftBench')
+        shell.call([release_bench] + cmdline)
+
+
+def run_build_script_helper(host_target, product, args):
+    toolchain_path = args.install_destdir
+    if platform.system() == 'Darwin':
+        # The prefix is an absolute path, so concatenate without os.path.
+        toolchain_path += \
+            targets.darwin_toolchain_prefix(args.install_prefix)
+
+    # Our source_dir is expected to be './$SOURCE_ROOT/benchmarks'. That is due
+    # the assumption that each product is in its own build directory. This
+    # product is not like that and has its package/tools instead in
+    # ./$SOURCE_ROOT/swift/benchmark.
+    package_path = os.path.join(product.source_dir, '..', 'swift', 'benchmark')
+    package_path = os.path.abspath(package_path)
+
+    # We use a separate python helper to enable quicker iteration when working
+    # on this by avoiding going through build-script to test small changes.
+    helper_path = os.path.join(package_path, 'utils', 'build_script_helper.py')
+
+    build_cmd = [
+        helper_path,
+        '--verbose',
+        '--package-path', package_path,
+        '--build-path', product.build_dir,
+        '--toolchain', toolchain_path,
+    ]
+    shell.call(build_cmd)