Merge pull request swiftlang#29991 from apple/add-flag-non-executable-test

[Build System] Add support to execute non-executable tests (30308095)

Author: shahmishal

utils/build-presets.ini

@@ -175,6 +175,14 @@ skip-test-ios-host
 skip-test-tvos-host
 skip-test-watchos-host
 
+
+[preset: buildbot,tools=RA,stdlib=DA,test=non_executable]
+mixin-preset=
+     mixin_buildbot_tools_RA_stdlib_DA
+
+only-non-executable-test
+
+
 [preset: buildbot,tools=RA,stdlib=RD]
 mixin-preset=
     mixin_buildbot_tools_RA_stdlib_RD
@@ -187,6 +195,14 @@ skip-test-ios-host
 skip-test-tvos-host
 skip-test-watchos-host
 
+
+[preset: buildbot,tools=RA,stdlib=RD,test=non_executable]
+mixin-preset=
+     mixin_buildbot_tools_RA_stdlib_RD
+
+only-non-executable-test
+
+
 [preset: buildbot,tools=RA,stdlib=RD,single-thread]
 mixin-preset=
     mixin_buildbot_tools_RA_stdlib_RD
@@ -215,6 +231,11 @@ skip-test-tvos-host
 skip-test-watchos-host
 check-incremental-compilation
 
+[preset: buildbot,tools=R,stdlib=RD,test=non_executable]
+mixin-preset=
+     mixin_buildbot_tools_R_stdlib_RD
+
+only-non-executable-test
 
 [preset: buildbot,tools=RA,stdlib=RDA]
 mixin-preset=
@@ -228,6 +249,12 @@ skip-test-ios-host
 skip-test-tvos-host
 skip-test-watchos-host
 
+[preset: buildbot,tools=RA,stdlib=RDA,test=non_executable]
+mixin-preset=
+     mixin_buildbot_tools_RA_stdlib_RDA
+
+only-non-executable-test
+
 
 [preset: buildbot,tools=RA,stdlib=RDA,long_test]
 mixin-preset=

utils/build_swift/build_swift/driver_arguments.py

@@ -825,6 +825,8 @@ def create_argument_parser():
     option('--only-executable-test', toggle_true,
            help='Only run executable tests. Does nothing if host-test is not '
                 'allowed')
+    option('--only-non-executable-test', toggle_true,
+           help='Only run non-executable tests.')
 
     option('--test-paths', append,
            type=argparse.ShellSplitType(),

utils/build_swift/tests/expected_options.py

@@ -155,6 +155,7 @@
     'host_target': targets.StdlibDeploymentTarget.host_target().name,
     'host_test': False,
     'only_executable_test': False,
+    'only_non_executable_test': False,
     'install_prefix': targets.install_prefix(),
     'install_symroot': None,
     'install_destdir': None,
@@ -498,6 +499,7 @@ class BuildScriptImplOption(_BaseOption):
     EnableOption('--foundation', dest='build_foundation'),
     EnableOption('--host-test'),
     EnableOption('--only-executable-test'),
+    EnableOption('--only-non-executable-test'),
     EnableOption('--libdispatch', dest='build_libdispatch'),
     EnableOption('--libicu', dest='build_libicu'),
     EnableOption('--indexstore-db', dest='build_indexstoredb'),

utils/swift_build_support/swift_build_support/host_specific_configuration.py

@@ -136,6 +136,8 @@ def __init__(self, host_target, args):
                     suffix = "-only_non_executable"
                 elif args.only_executable_test:
                     suffix = "-only_executable"
+                elif args.only_non_executable_test:
+                    suffix = "-only_non_executable"
                 else:
                     suffix = ""
                 subset_suffix = ""
@@ -214,25 +216,25 @@ def __platforms_to_skip_test(self, args):
             platforms_to_skip_test.add(StdlibDeploymentTarget.Cygwin)
         if not args.test_osx:
             platforms_to_skip_test.add(StdlibDeploymentTarget.OSX)
-        if not args.test_ios_host:
+        if not args.test_ios_host and not args.only_non_executable_test:
             platforms_to_skip_test.add(StdlibDeploymentTarget.iOS)
-        else:
+        elif not args.only_non_executable_test:
             raise ArgumentError(None,
                                 "error: iOS device tests are not " +
                                 "supported in open-source Swift.")
         if not args.test_ios_simulator:
             platforms_to_skip_test.add(StdlibDeploymentTarget.iOSSimulator)
-        if not args.test_tvos_host:
+        if not args.test_tvos_host and not args.only_non_executable_test:
             platforms_to_skip_test.add(StdlibDeploymentTarget.AppleTV)
-        else:
+        elif not args.only_non_executable_test:
             raise ArgumentError(None,
                                 "error: tvOS device tests are not " +
                                 "supported in open-source Swift.")
         if not args.test_tvos_simulator:
             platforms_to_skip_test.add(StdlibDeploymentTarget.AppleTVSimulator)
-        if not args.test_watchos_host:
+        if not args.test_watchos_host and not args.only_non_executable_test:
             platforms_to_skip_test.add(StdlibDeploymentTarget.AppleWatch)
-        else:
+        elif not args.only_non_executable_test:
             raise ArgumentError(None,
                                 "error: watchOS device tests are not " +
                                 "supported in open-source Swift.")
@@ -255,10 +257,10 @@ def __platforms_to_skip_test_host(self, args):
         platforms_to_skip_test_host = set()
         if not args.test_android_host:
             platforms_to_skip_test_host.add(StdlibDeploymentTarget.Android)
-        if not args.test_ios_host:
+        if not args.test_ios_host and not args.only_non_executable_test:
             platforms_to_skip_test_host.add(StdlibDeploymentTarget.iOS)
-        if not args.test_tvos_host:
+        if not args.test_tvos_host and not args.only_non_executable_test:
             platforms_to_skip_test_host.add(StdlibDeploymentTarget.AppleTV)
-        if not args.test_watchos_host:
+        if not args.test_watchos_host and not args.only_non_executable_test:
             platforms_to_skip_test_host.add(StdlibDeploymentTarget.AppleWatch)
         return platforms_to_skip_test_host

utils/swift_build_support/tests/test_host_specific_configuration.py

@@ -320,6 +320,23 @@ def test_should_allow_testing_only_executable_tests(self):
         self.assertIn('check-swift-only_executable-macosx-x86_64',
                       after.swift_test_run_targets)
 
+    def test_should_allow_testing_only_non_executable_tests(self):
+        args = self.default_args()
+        args.build_osx = True
+        args.test_osx = True
+        args.host_target = 'macosx-x86_64'
+        args.stdlib_deployment_targets = ['macosx-x86_64']
+        args.build_stdlib_deployment_targets = 'all'
+
+        before = HostSpecificConfiguration('macosx-x86_64', args)
+        self.assertIn('check-swift-macosx-x86_64',
+                      before.swift_test_run_targets)
+
+        args.only_non_executable_test = True
+        after = HostSpecificConfiguration('macosx-x86_64', args)
+        self.assertIn('check-swift-only_non_executable-macosx-x86_64',
+                      after.swift_test_run_targets)
+
     def generate_should_build_benchmarks(host_target, build_arg_name):
         def test(self):
             args = self.default_args()
@@ -620,6 +637,7 @@ def default_args(self):
             maccatalyst_ios_tests=False,
             long_test=False,
             only_executable_test=False,
+            only_non_executable_test=False,
             stress_test=False,
             test_android=False,
             test_android_host=False,