Skip to content

Commit fed7a75

Browse files
ahoppenahoppen
committed
Manage the project XFails under a different job type for the stress tester
Currently we’re in a state again in which some project succeed to build in the stress tester CI job but fail to build in the source compatibility suite CI job (rdar://82333521). We are thus mistakingly receiving UPASSes in the stress tester CI job. To work around this, introduce the notion of job types to the source compatibility runner and its associated XFails. By default everything runs as the "source-compat" job, only the stress tester overrides the job to "stress-tester". That way we can selectively XFail jobs only for the stress tester in `projects.json`. Since the stress tester now controls its own project build XFails, a project build failure will also cause a stress tester failure. rdar://62080167
1 parent efb8f15 commit fed7a75

File tree

5 files changed

+65
-36
lines changed

5 files changed

+65
-36
lines changed

‎build_incremental.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ def main():
4444
args.swiftc,
4545
args.swift_version,
4646
args.swift_branch,
47+
args.job_type,
4748
args.sandbox_profile_xcodebuild,
4849
args.sandbox_profile_package,
4950
args.add_swift_flags,

‎project_future.py

Lines changed: 17 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -414,8 +414,8 @@ def dispatch(root_path, repo, action, swiftc, swift_version,
414414
raise common.Unimplemented("Unknown action: %s" % action['action'])
415415

416416

417-
def is_xfailed(xfail_args, compatible_version, platform, swift_branch, build_config):
418-
"""Return whether the specified swift version/platform/branch/configuration is xfailed."""
417+
def is_xfailed(xfail_args, compatible_version, platform, swift_branch, build_config, job_type):
418+
"""Return whether the specified swift version/platform/branch/configuration/job is xfailed."""
419419
if isinstance(xfail_args, dict):
420420
xfail_args = [xfail_args]
421421

@@ -427,7 +427,8 @@ def matches(spec):
427427
current = {
428428
'compatibility': compatible_version,
429429
'branch': swift_branch,
430-
'platform': platform
430+
'platform': platform,
431+
'job': job_type,
431432
}
432433
if 'configuration' in spec:
433434
if build_config is None:
@@ -583,6 +584,9 @@ def add_arguments(parser):
583584
parser.add_argument("--report-time-path",
584585
help='export time for building each xcode build target to the specified json file',
585586
type=os.path.abspath)
587+
parser.add_argument("--job-type",
588+
help="The type of job to run. This influences which projects are XFailed, for example the stress tester tracks its XFails under a different job type. Defaults to 'source-compat'.",
589+
default='source-compat')
586590

587591
def add_minimal_arguments(parser):
588592
"""Add common arguments to parser."""
@@ -942,7 +946,7 @@ def output_fd(self, subtarget):
942946

943947

944948
class ActionBuilder(Factory):
945-
def __init__(self, swiftc, swift_version, swift_branch,
949+
def __init__(self, swiftc, swift_version, swift_branch, job_type,
946950
sandbox_profile_xcodebuild,
947951
sandbox_profile_package,
948952
added_swift_flags,
@@ -968,6 +972,7 @@ def __init__(self, swiftc, swift_version, swift_branch,
968972
self.build_config = build_config
969973
self.strip_resource_phases = strip_resource_phases
970974
self.time_reporter = time_reporter
975+
self.job_type = job_type
971976
self.init()
972977

973978
def init(self):
@@ -1055,7 +1060,7 @@ def succeeded(self, identifier):
10551060

10561061
class CompatActionBuilder(ActionBuilder):
10571062
def __init__(self,
1058-
swiftc, swift_version, swift_branch,
1063+
swiftc, swift_version, swift_branch, job_type,
10591064
sandbox_profile_xcodebuild,
10601065
sandbox_profile_package,
10611066
added_swift_flags,
@@ -1067,7 +1072,7 @@ def __init__(self,
10671072
time_reporter,
10681073
action, version, project):
10691074
super(CompatActionBuilder, self).__init__(
1070-
swiftc, swift_version, swift_branch,
1075+
swiftc, swift_version, swift_branch, job_type,
10711076
sandbox_profile_xcodebuild,
10721077
sandbox_profile_package,
10731078
added_swift_flags,
@@ -1136,7 +1141,8 @@ def failed(self, identifier, error):
11361141
self.version['version'],
11371142
self.current_platform,
11381143
self.swift_branch,
1139-
build_config)
1144+
build_config,
1145+
self.job_type)
11401146
if bug_identifier:
11411147
error_str = 'XFAIL: {bug}, {project}, {compatibility}, {commit}, {action_target}'.format(
11421148
bug=bug_identifier,
@@ -1170,7 +1176,8 @@ def succeeded(self, identifier):
11701176
self.version['version'],
11711177
self.current_platform,
11721178
self.swift_branch,
1173-
build_config)
1179+
build_config,
1180+
self.job_type)
11741181
if bug_identifier:
11751182
error_str = 'UPASS: {bug}, {project}, {compatibility}, {commit}, {action_target}'.format(
11761183
bug=bug_identifier,
@@ -1243,15 +1250,15 @@ def have_same_trees(full, incr, d):
12431250

12441251
class IncrementalActionBuilder(ActionBuilder):
12451252

1246-
def __init__(self, swiftc, swift_version, swift_branch,
1253+
def __init__(self, swiftc, swift_version, swift_branch, job_type,
12471254
sandbox_profile_xcodebuild,
12481255
sandbox_profile_package,
12491256
added_swift_flags, build_config,
12501257
strip_resource_phases,
12511258
time_reporter,
12521259
project, action):
12531260
super(IncrementalActionBuilder,
1254-
self).__init__(swiftc, swift_version, swift_branch,
1261+
self).__init__(swiftc, swift_version, swift_branch, job_type,
12551262
sandbox_profile_xcodebuild,
12561263
sandbox_profile_package,
12571264
added_swift_flags,

‎projects.json

Lines changed: 42 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,8 @@
2626
{
2727
"issue": "https://bugs.swift.org/browse/SR-15116",
2828
"compatibility": ["5.1"],
29-
"branch": ["main", "release/5.5"]
29+
"branch": ["main", "release/5.5"],
30+
"job": ["source-compat"]
3031
}
3132
]
3233
}
@@ -680,12 +681,14 @@
680681
{
681682
"issue": "https://bugs.swift.org/browse/SR-12736",
682683
"compatibility": "5.3",
683-
"branch": ["main", "release/5.5", "release/5.4"]
684+
"branch": ["main", "release/5.5", "release/5.4"],
685+
"job": ["source-compat"]
684686
},
685687
{
686688
"issue": "https://bugs.swift.org/browse/SR-14659",
687689
"compatibility": "5.3",
688-
"branch": ["main", "release/5.5", "release/5.4"]
690+
"branch": ["main", "release/5.5", "release/5.4"],
691+
"job": ["source-compat"]
689692
}
690693
]
691694
},
@@ -816,7 +819,8 @@
816819
{
817820
"issue": "https://bugs.swift.org/browse/SR-15106",
818821
"compatibility": ["4.0"],
819-
"branch": ["main", "release/5.5"]
822+
"branch": ["main", "release/5.5"],
823+
"job": ["source-compat"]
820824
}
821825
]
822826
}
@@ -1236,7 +1240,8 @@
12361240
{
12371241
"issue": "rdar://problem/75437284",
12381242
"compatibility": ["5.1"],
1239-
"branch": ["main", "release/5.5"]
1243+
"branch": ["main", "release/5.5"],
1244+
"job": ["source-compat"]
12401245
}
12411246
]
12421247
},
@@ -1250,12 +1255,14 @@
12501255
{
12511256
"issue": "rdar://problem/75437284",
12521257
"compatibility": ["5.1"],
1253-
"branch": ["main", "release/5.5"]
1258+
"branch": ["main", "release/5.5"],
1259+
"job": ["source-compat"]
12541260
},
12551261
{
12561262
"issue": "https://bugs.swift.org/browse/SR-15107",
12571263
"compatibility": ["4.0"],
1258-
"branch": ["main", "release/5.5"]
1264+
"branch": ["main", "release/5.5"],
1265+
"job": ["source-compat"]
12591266
}
12601267
]
12611268
}
@@ -1287,7 +1294,8 @@
12871294
{
12881295
"issue": "https://bugs.swift.org/browse/SR-15109",
12891296
"compatibility": ["5.1"],
1290-
"branch": ["main", "release/5.5"]
1297+
"branch": ["main", "release/5.5"],
1298+
"job": ["source-compat"]
12911299
}
12921300
]
12931301
},
@@ -1641,7 +1649,8 @@
16411649
{
16421650
"issue": "https://bugs.swift.org/browse/SR-15116",
16431651
"compatibility": ["5.1"],
1644-
"branch": ["main", "release/5.5"]
1652+
"branch": ["main", "release/5.5"],
1653+
"job": ["source-compat"]
16451654
}
16461655
]
16471656
}
@@ -2184,7 +2193,8 @@
21842193
"issue": "rdar://problem/68590422",
21852194
"compatibility": ["4.0", "5.1"],
21862195
"branch": ["release/5.4"],
2187-
"configuration": "debug"
2196+
"configuration": "debug",
2197+
"job": ["source-compat"]
21882198
}
21892199
]
21902200
},
@@ -2199,7 +2209,8 @@
21992209
"issue": "rdar://problem/68590422",
22002210
"compatibility": ["4.0", "5.1"],
22012211
"branch": ["release/5.4"],
2202-
"configuration": "debug"
2212+
"configuration": "debug",
2213+
"job": ["source-compat"]
22032214
}
22042215
]
22052216
},
@@ -2246,7 +2257,8 @@
22462257
{
22472258
"issue": "https://bugs.swift.org/browse/SR-15109",
22482259
"compatibility": ["5.1"],
2249-
"branch": ["main", "release/5.5"]
2260+
"branch": ["main", "release/5.5"],
2261+
"job": ["source-compat"]
22502262
}
22512263
]
22522264
},
@@ -2420,7 +2432,8 @@
24202432
{
24212433
"issue": "https://bugs.swift.org/browse/SR-15232",
24222434
"compatibility": ["4.2", "5.3"],
2423-
"branch": ["main", "release/5.5"]
2435+
"branch": ["main", "release/5.5"],
2436+
"job": ["source-compat"]
24242437
}
24252438
],
24262439
"configuration": "release"
@@ -2485,7 +2498,8 @@
24852498
"xfail": {
24862499
"issue": "https://bugs.swift.org/browse/SR-11141",
24872500
"compatibility": "5.1",
2488-
"branch": ["main", "release/5.3", "release/5.4", "release/5.5"]
2501+
"branch": ["main", "release/5.3", "release/5.4", "release/5.5"],
2502+
"job": ["source-compat"]
24892503
}
24902504
},
24912505
{
@@ -2549,7 +2563,8 @@
25492563
{
25502564
"issue": "https://bugs.swift.org/browse/SR-14655;rdar://81276100",
25512565
"compatibility": ["5.1"],
2552-
"branch": ["main", "release/5.5"]
2566+
"branch": ["main", "release/5.5"],
2567+
"job": ["source-compat"]
25532568
}
25542569
]
25552570
},
@@ -2563,7 +2578,8 @@
25632578
{
25642579
"issue": "https://bugs.swift.org/browse/SR-14655;rdar://81276100",
25652580
"compatibility": ["5.1"],
2566-
"branch": ["main", "release/5.5"]
2581+
"branch": ["main", "release/5.5"],
2582+
"job": ["source-compat"]
25672583
}
25682584
]
25692585
},
@@ -2577,7 +2593,8 @@
25772593
{
25782594
"issue": "https://bugs.swift.org/browse/SR-14655;rdar://81276100",
25792595
"compatibility": ["5.1"],
2580-
"branch": ["main", "release/5.5"]
2596+
"branch": ["main", "release/5.5"],
2597+
"job": ["source-compat"]
25812598
}
25822599
]
25832600
}
@@ -2611,7 +2628,8 @@
26112628
{
26122629
"issue": "https://bugs.swift.org/browse/SR-13190",
26132630
"compatibility": ["4.0", "5.1"],
2614-
"branch": ["main", "release/5.4", "release/5.5"]
2631+
"branch": ["main", "release/5.4", "release/5.5"],
2632+
"job": ["source-compat"]
26152633
}
26162634
]
26172635
},
@@ -3128,7 +3146,8 @@
31283146
{
31293147
"issue": "rdar://81180448",
31303148
"compatibility": ["4.2", "5.0", "5.1"],
3131-
"branch": ["main", "release/5.5"]
3149+
"branch": ["main", "release/5.5"],
3150+
"job": ["source-compat"]
31323151
}
31333152
]
31343153
},
@@ -3953,7 +3972,8 @@
39533972
{
39543973
"issue": "https://bugs.swift.org/browse/SR-15105",
39553974
"compatibility": ["5.1"],
3956-
"branch": ["main", "release/5.5"]
3975+
"branch": ["main", "release/5.5"],
3976+
"job": ["source-compat"]
39573977
}
39583978
]
39593979
},
@@ -4297,7 +4317,8 @@
42974317
"issue": "https://bugs.swift.org/browse/SR-15105",
42984318
"compatibility": ["5.1"],
42994319
"branch": ["main", "release/5.5"],
4300-
"configuration": "debug"
4320+
"configuration": "debug",
4321+
"job": ["source-compat"]
43014322
}
43024323
]
43034324
},

‎run_sk_stress_test

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -258,6 +258,7 @@ class StressTesterRunner(object):
258258
'--verbose',
259259
'--swiftc', self.wrapper,
260260
'--swift-branch', self.swift_branch,
261+
'--job-type', 'stress-tester',
261262
'--default-timeout', str(-1),
262263
'--only-latest-versions',
263264
# archs override is set to arm64 for generic/iOS actions that would otherwise invoke the stress tester for both arm64 and armv7
@@ -284,7 +285,7 @@ class StressTesterRunner(object):
284285

285286
def _process_output(self, results_path, xfails_path):
286287
if not os.path.isfile(results_path):
287-
return True
288+
return not self.compat_runner_failed
288289

289290
with open(results_path, 'rb') as results_file:
290291
results = json.load(results_file, encoding='utf-8')
@@ -305,7 +306,7 @@ class StressTesterRunner(object):
305306
num_xfail_issues = len(results['expectedIssues'])
306307
unmatched = results['unmatchedExpectedIssues']
307308

308-
success = num_failures == 0 and len(unmatched) == 0
309+
success = self.compat_runner_failed == False and num_failures == 0 and len(unmatched) == 0
309310

310311
if num_xfails > 0:
311312
print('Expected stress tester issues:')
@@ -336,9 +337,7 @@ class StressTesterRunner(object):
336337
print('SourceKit Stress Tester summary:')
337338

338339
print(' {} underlying source compatibility build'.format('failed' if self.compat_runner_failed else 'passed'))
339-
if self.compat_runner_failed:
340-
print(' > treat this as a source compatibility failure')
341-
340+
342341
print(' {} XFails not processed'.format(len(xfails_not_processed)))
343342
if num_failures > 0:
344343
print(' > see "XFails not processed" how to handle them. This is an info, not an error')

‎runner.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@ def main():
6565
args.swiftc,
6666
args.swift_version,
6767
args.swift_branch,
68+
args.job_type,
6869
args.sandbox_profile_xcodebuild,
6970
args.sandbox_profile_package,
7071
swift_flags,

0 commit comments

Comments
 (0)