[build-script] Add a default-enabled "clean" step for swift-driver and swiftpm

Along with options to disable the mandatory clean using: `--skip-clean-swift-driver` and `--skip-clean-swiftpm`.

This will ensure that every invocation of build-script will, by default, clean up all build artifacts of these projects and re-build them from scratch.
This is needed for all builds because today arbitrary changes to the compiler can lead to us being unable to incrementally build components that are themselves written in Swift. This causes now-frequent failures in incremental build bots, and is a scenario that is encountered by developers. (For example see: rdar://65006593)

The proper long-term solution is to enable library evolution for these projects. Until this is done, the only safe thing to do is to always rebuild them.

Resolves rdar://65006593

Author: artemcm

utils/build-script

@@ -964,6 +964,9 @@ class BuildScriptInvocation(object):
                     toolchain=self.toolchain,
                     source_dir=self.workspace.source_dir(product_source),
                     build_dir=build_dir)
+                if product.should_clean(host_target):
+                    print("--- Cleaning %s ---" % product_name)
+                    product.clean(host_target)
                 if product.should_build(host_target):
                     print("--- Building %s ---" % product_name)
                     product.build(host_target)

utils/build_swift/build_swift/driver_arguments.py

@@ -1008,6 +1008,10 @@ def create_argument_parser():
            help='skip testing Android device targets on the host machine (the '
                 'phone itself)')
 
+    option('--skip-clean-swiftpm', toggle_false('clean_swiftpm'),
+           help='skip cleaning up swiftpm')
+    option('--skip-clean-swift-driver', toggle_false('clean_swift_driver'),
+           help='skip cleaning up Swift driver')
     option('--skip-test-swiftpm', toggle_false('test_swiftpm'),
            help='skip testing swiftpm')
     option('--skip-test-swift-driver', toggle_false('test_swift_driver'),

utils/build_swift/tests/expected_options.py

@@ -205,6 +205,8 @@
         defaults.SWIFT_MAX_PARALLEL_LTO_LINK_JOBS,
     'swift_user_visible_version': defaults.SWIFT_USER_VISIBLE_VERSION,
     'symbols_package': None,
+    'clean_swiftpm': True,
+    'clean_swift_driver': True,
     'test': None,
     'test_android': False,
     'test_android_host': False,
@@ -567,6 +569,8 @@ class BuildScriptImplOption(_BaseOption):
                   dest='build_watchos_device'),
     DisableOption('--skip-build-watchos-simulator',
                   dest='build_watchos_simulator'),
+    DisableOption('--skip-clean-swiftpm', dest='clean_swiftpm'),
+    DisableOption('--skip-clean-swift-driver', dest='clean_swift_driver'),
     DisableOption('--skip-test-android', dest='test_android'),
     DisableOption('--skip-test-android-host', dest='test_android_host'),
     DisableOption('--skip-test-cygwin', dest='test_cygwin'),

utils/swift_build_support/swift_build_support/products/product.py

@@ -78,6 +78,20 @@ def get_dependencies(cls):
         """Return a list of products that this product depends upon"""
         raise NotImplementedError
 
+    def should_clean(self, host_target):
+        """should_clean() -> Bool
+
+        Whether or not this product should be cleaned before being built
+        """
+        return False
+
+    def clean(self, host_target):
+        """clean() -> void
+
+        Perform the clean, for a non-build-script-impl product.
+        """
+        raise NotImplementedError
+
     def should_build(self, host_target):
         """should_build() -> Bool
 

utils/swift_build_support/swift_build_support/products/swiftdriver.py

@@ -47,6 +47,13 @@ def get_dependencies(cls):
                 xctest.XCTest,
                 llbuild.LLBuild]
 
+    def should_clean(self, host_target):
+        return self.args.clean_swift_driver
+
+    def clean(self, host_target):
+        indexstoredb.run_build_script_helper(
+            'clean', host_target, self, self.args)
+
     def build(self, host_target):
         indexstoredb.run_build_script_helper(
             'build', host_target, self, self.args)

utils/swift_build_support/swift_build_support/products/swiftpm.py

@@ -51,6 +51,11 @@ def run_bootstrap_script(self, action, host_target, additional_params=[]):
 
         helper_cmd = [script_path, action]
 
+        if action == 'clean':
+            helper_cmd += ["--build-dir", self.build_dir]
+            shell.call(helper_cmd)
+            return
+
         if self.is_release():
             helper_cmd.append("--release")
 
@@ -94,6 +99,12 @@ def should_test(self, host_target):
     def test(self, host_target):
         self.run_bootstrap_script('test', host_target)
 
+    def should_clean(self, host_target):
+        return self.args.clean_swiftpm
+
+    def clean(self, host_target):
+        self.run_bootstrap_script('clean', host_target)
+
     def should_install(self, host_target):
         return self.args.install_swiftpm