Skip to content

Commit a1723d8

Browse files
authored
feat: Associate crashes with last started test (#58)
When a test crashes on assert()/precondition() (SIGTRAP, signal 5, etc.), xcsift now tracks the last "Test Case '...' started." line and associates the crash with that test in failed_tests. Three-level crash detection: - "Restarting after..." with signal code → FailedTest with signal info - Fatal error with lastStartedTest → FailedTest with file/line from error - Safety net: started without completion + TEST FAILED → FailedTest Supports both XCTest and Swift Testing "started" formats.
1 parent 7cdf88a commit a1723d8

File tree

2 files changed

+267
-0
lines changed

2 files changed

+267
-0
lines changed

Sources/OutputParser.swift

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,10 @@ class OutputParser {
3535
private var passedTestDurations: [String: Double] = [:]
3636
private var failedTestDurations: [String: Double] = [:]
3737

38+
// Crash / signal tracking
39+
private var lastStartedTestName: String?
40+
private var pendingSignalCode: Int?
41+
3842
// Build info tracking - phases grouped by target
3943
private var targetPhases: [String: [String]] = [:] // target -> [phase names]
4044
private var targetDurations: [String: String] = [:] // target -> duration
@@ -451,6 +455,20 @@ class OutputParser {
451455
return nil
452456
}()
453457

458+
// Safety net: if a test was started but never completed and testRunFailed is set
459+
if testRunFailed, let testName = lastStartedTestName {
460+
let normalizedName = normalizeTestName(testName)
461+
if !hasSeenSimilarTest(normalizedName) {
462+
let message =
463+
pendingSignalCode.map { "Crashed (signal \($0)): last test started before crash" }
464+
?? "Test did not complete (possible crash or timeout)"
465+
failedTests.append(FailedTest(test: testName, message: message, file: nil, line: nil))
466+
seenTestNames.insert(normalizedName)
467+
}
468+
lastStartedTestName = nil
469+
pendingSignalCode = nil
470+
}
471+
454472
// Determine build status with priority on parsed results over testRunFailed flag
455473
// Issue #52: -skipMacroValidation can set testRunFailed even when tests pass
456474
let status: String = {
@@ -647,6 +665,8 @@ class OutputParser {
647665
shouldParseBuildInfo = false
648666
targetDependencies = [:]
649667
currentDependencyTarget = nil
668+
lastStartedTestName = nil
669+
pendingSignalCode = nil
650670
}
651671

652672
private func parseLine(_ line: String) {
@@ -700,11 +720,49 @@ class OutputParser {
700720
|| line.hasPrefix("RegisterWithLaunchServices")
701721
|| line.hasPrefix("Validate") || line.contains("Fatal error")
702722
|| (line.hasPrefix("/") && line.contains(".swift:")) // runtime warnings
723+
|| line.contains("' started") // XCTest: "Test Case '...' started."
724+
|| line.contains("\" started") // Swift Testing: Test "..." started
725+
|| line.contains("signal code ") // "Exited with [unexpected] signal code N"
726+
|| line.hasPrefix("Restarting after") // crash confirmation
703727

704728
if !containsRelevant {
705729
return
706730
}
707731

732+
// Track "Test Case '...' started." for crash association
733+
if parseStartedTest(line) {
734+
return
735+
}
736+
737+
// Signal code: "Exited with [unexpected] signal code N"
738+
if line.contains("signal code ") {
739+
if let lastSpace = line.lastIndex(of: " ") {
740+
let codeStr = String(line[line.index(after: lastSpace)...])
741+
pendingSignalCode = Int(codeStr)
742+
}
743+
return
744+
}
745+
746+
// Crash confirmation: "Restarting after unexpected exit, crash, or test timeout"
747+
if line.hasPrefix("Restarting after") {
748+
if let testName = lastStartedTestName {
749+
let message: String
750+
if let code = pendingSignalCode {
751+
message = "Crashed (signal \(code)): last test started before crash"
752+
} else {
753+
message = "Crashed: last test started before crash"
754+
}
755+
let normalizedName = normalizeTestName(testName)
756+
if !hasSeenSimilarTest(normalizedName) {
757+
failedTests.append(FailedTest(test: testName, message: message, file: nil, line: nil))
758+
seenTestNames.insert(normalizedName)
759+
}
760+
}
761+
lastStartedTestName = nil
762+
pendingSignalCode = nil
763+
return
764+
}
765+
708766
// Parse parallel test scheduling lines: [N/TOTAL] Testing Module.Class/method
709767
if line.contains("] Testing "), let match = line.firstMatch(of: Self.parallelTestSchedulingRegex) {
710768
if let _ = Int(match.1), let total = Int(match.2) {
@@ -727,6 +785,11 @@ class OutputParser {
727785
if let failedTest = parseFailedTest(line) {
728786
let normalizedTestName = normalizeTestName(failedTest.test)
729787

788+
// Clear lastStartedTestName if this failed test matches
789+
if normalizeTestName(lastStartedTestName ?? "") == normalizedTestName {
790+
lastStartedTestName = nil
791+
}
792+
730793
// Check if we've already seen this test name or a similar one
731794
if !hasSeenSimilarTest(normalizedTestName) {
732795
failedTests.append(failedTest)
@@ -761,6 +824,21 @@ class OutputParser {
761824
seenErrors.insert(key)
762825
errors.append(error)
763826
}
827+
// Fatal error + lastStartedTestName → also create FailedTest
828+
if line.contains("Fatal error"), let testName = lastStartedTestName {
829+
let normalizedName = normalizeTestName(testName)
830+
if !hasSeenSimilarTest(normalizedName) {
831+
failedTests.append(
832+
FailedTest(
833+
test: testName,
834+
message: "Crashed (Fatal error): last test started before crash",
835+
file: error.file,
836+
line: error.line
837+
)
838+
)
839+
seenTestNames.insert(normalizedName)
840+
}
841+
}
764842
} else if let warning = parseWarning(line) {
765843
let key = "\(warning.file ?? ""):\(warning.line ?? 0):\(warning.message)"
766844
if !seenWarnings.contains(key) {
@@ -1008,8 +1086,48 @@ class OutputParser {
10081086
return false
10091087
}
10101088

1089+
// MARK: - Crash Association
1090+
1091+
/// Parses "Test Case '...' started." or "◇ Test "..." started." lines.
1092+
/// Updates `lastStartedTestName` for crash association.
1093+
private func parseStartedTest(_ line: String) -> Bool {
1094+
// XCTest: Test Case '-[Module.Class testMethod]' started.
1095+
// XCTest parallel: Test case '-[Module.Class testMethod]' started.
1096+
if (line.hasPrefix("Test Case '") || line.hasPrefix("Test case '"))
1097+
&& line.contains("' started")
1098+
{
1099+
let prefixLength = 11 // "Test Case '" or "Test case '"
1100+
let startIndex = line.index(line.startIndex, offsetBy: prefixLength)
1101+
if let endQuote = line.range(of: "' started", range: startIndex ..< line.endIndex) {
1102+
lastStartedTestName = String(line[startIndex ..< endQuote.lowerBound])
1103+
}
1104+
return true
1105+
}
1106+
1107+
// Swift Testing: ◇ Test "shouldCrash()" started.
1108+
// or: ◇ Test functionName() started.
1109+
if line.hasPrefix("◇ Test ") {
1110+
let afterPrefix = line.index(line.startIndex, offsetBy: "◇ Test ".count)
1111+
if let result = extractSwiftTestingName(from: line, after: afterPrefix) {
1112+
let afterName = line[result.endIndex...]
1113+
if afterName.hasPrefix(" started") {
1114+
lastStartedTestName = result.name
1115+
return true
1116+
}
1117+
}
1118+
}
1119+
1120+
return false
1121+
}
1122+
10111123
private func recordPassedTest(named testName: String, duration: Double? = nil) {
10121124
let normalizedTestName = normalizeTestName(testName)
1125+
1126+
// Clear lastStartedTestName if this passed test matches
1127+
if normalizeTestName(lastStartedTestName ?? "") == normalizedTestName {
1128+
lastStartedTestName = nil
1129+
}
1130+
10131131
guard seenPassedTestNames.insert(normalizedTestName).inserted else {
10141132
return
10151133
}

Tests/ParsingTests.swift

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1324,6 +1324,155 @@ final class ParsingTests: XCTestCase {
13241324
XCTAssertEqual(result.errors[0].file, "TestProjectTests/TestProjectTests.swift")
13251325
XCTAssertEqual(result.errors[0].line, 5)
13261326
XCTAssertEqual(result.errors[0].message, "Fatal error")
1327+
// Crash should also be associated with the last started test
1328+
XCTAssertEqual(result.failedTests.count, 1)
1329+
guard result.failedTests.count == 1 else { return }
1330+
XCTAssertTrue(result.failedTests[0].test.contains("testExample"))
1331+
}
1332+
1333+
// MARK: - Crash Association Tests
1334+
1335+
func testCrashSignalAssociatedWithLastStartedTest() {
1336+
let parser = OutputParser()
1337+
let input = """
1338+
Test Case '-[MyTests.CrashTests testDivideByZero]' started.
1339+
Exited with unexpected signal code 5
1340+
Restarting after unexpected exit, crash, or test timeout
1341+
** TEST FAILED **
1342+
"""
1343+
1344+
let result = parser.parse(input: input)
1345+
1346+
XCTAssertEqual(result.status, "failed")
1347+
XCTAssertEqual(result.failedTests.count, 1)
1348+
guard result.failedTests.count == 1 else { return }
1349+
XCTAssertTrue(result.failedTests[0].test.contains("testDivideByZero"))
1350+
XCTAssertTrue(result.failedTests[0].message.contains("signal 5"))
1351+
}
1352+
1353+
func testCrashSignalWithoutUnexpected() {
1354+
let parser = OutputParser()
1355+
let input = """
1356+
Test Case '-[MyTests.CrashTests testAbort]' started.
1357+
Exited with signal code 6
1358+
Restarting after unexpected exit, crash, or test timeout
1359+
** TEST FAILED **
1360+
"""
1361+
1362+
let result = parser.parse(input: input)
1363+
1364+
XCTAssertEqual(result.failedTests.count, 1)
1365+
guard result.failedTests.count == 1 else { return }
1366+
XCTAssertTrue(result.failedTests[0].test.contains("testAbort"))
1367+
XCTAssertTrue(result.failedTests[0].message.contains("signal 6"))
1368+
}
1369+
1370+
func testFatalErrorAssociatedWithLastStartedTest() {
1371+
let parser = OutputParser()
1372+
let input = """
1373+
Test Case '-[MyTests.CrashTests testPrecondition]' started.
1374+
/path/to/MyTests.swift:42: Fatal error: Precondition failed
1375+
Restarting after unexpected exit, crash, or test timeout
1376+
** TEST FAILED **
1377+
"""
1378+
1379+
let result = parser.parse(input: input)
1380+
1381+
// Fatal error should be in errors (parser strips "Fatal error: " prefix)
1382+
XCTAssertEqual(result.errors.count, 1)
1383+
XCTAssertEqual(result.errors[0].message, "Precondition failed")
1384+
// AND also in failedTests (associated with the started test)
1385+
XCTAssertEqual(result.failedTests.count, 1)
1386+
guard result.failedTests.count == 1 else { return }
1387+
XCTAssertTrue(result.failedTests[0].test.contains("testPrecondition"))
1388+
}
1389+
1390+
func testCrashWithNoStartedTestProducesNoFailedTest() {
1391+
let parser = OutputParser()
1392+
let input = """
1393+
Exited with unexpected signal code 5
1394+
Restarting after unexpected exit, crash, or test timeout
1395+
** TEST FAILED **
1396+
"""
1397+
1398+
let result = parser.parse(input: input)
1399+
1400+
// No "started" line → no test to associate with
1401+
XCTAssertEqual(result.failedTests.count, 0)
1402+
}
1403+
1404+
func testStartedTestClearsAfterPass() {
1405+
let parser = OutputParser()
1406+
let input = """
1407+
Test Case '-[MyTests.OKTests testPass]' started.
1408+
Test Case '-[MyTests.OKTests testPass]' passed (0.001 seconds).
1409+
Test Case '-[MyTests.CrashTests testCrash]' started.
1410+
Exited with unexpected signal code 5
1411+
Restarting after unexpected exit, crash, or test timeout
1412+
** TEST FAILED **
1413+
"""
1414+
1415+
let result = parser.parse(input: input)
1416+
1417+
// Only testCrash should be in failedTests, not testPass
1418+
XCTAssertEqual(result.failedTests.count, 1)
1419+
guard result.failedTests.count == 1 else { return }
1420+
XCTAssertTrue(result.failedTests[0].test.contains("testCrash"))
1421+
XCTAssertFalse(result.failedTests[0].test.contains("testPass"))
1422+
}
1423+
1424+
func testSwiftTestingStartedFormat() {
1425+
let parser = OutputParser()
1426+
let input = """
1427+
◇ Test "shouldCrash()" started.
1428+
Exited with unexpected signal code 5
1429+
Restarting after unexpected exit, crash, or test timeout
1430+
** TEST FAILED **
1431+
"""
1432+
1433+
let result = parser.parse(input: input)
1434+
1435+
XCTAssertEqual(result.failedTests.count, 1)
1436+
guard result.failedTests.count == 1 else { return }
1437+
XCTAssertTrue(result.failedTests[0].test.contains("shouldCrash"))
1438+
}
1439+
1440+
func testCrashInFullTestSuiteOutput() {
1441+
let parser = OutputParser()
1442+
let input = """
1443+
Test Suite 'All tests' started at 2024-01-15 10:00:00.000.
1444+
Test Suite 'MyTests.xctest' started at 2024-01-15 10:00:00.000.
1445+
Test Suite 'CrashTests' started at 2024-01-15 10:00:00.000.
1446+
Test Case '-[MyTests.CrashTests testAssertCrash]' started.
1447+
Exited with unexpected signal code 5
1448+
Restarting after unexpected exit, crash, or test timeout
1449+
Test Suite 'CrashTests' failed at 2024-01-15 10:00:01.000.
1450+
Executed 1 test, with 1 failure in 0.500 seconds
1451+
** TEST FAILED **
1452+
"""
1453+
1454+
let result = parser.parse(input: input)
1455+
1456+
XCTAssertEqual(result.status, "failed")
1457+
XCTAssertEqual(result.failedTests.count, 1)
1458+
guard result.failedTests.count == 1 else { return }
1459+
XCTAssertTrue(result.failedTests[0].test.contains("testAssertCrash"))
1460+
}
1461+
1462+
func testEndOfParseSafetyNet() {
1463+
let parser = OutputParser()
1464+
// "started" but NO "Restarting" line — safety net should catch it
1465+
let input = """
1466+
Test Case '-[MyTests.CrashTests testHang]' started.
1467+
** TEST FAILED **
1468+
"""
1469+
1470+
let result = parser.parse(input: input)
1471+
1472+
XCTAssertEqual(result.status, "failed")
1473+
XCTAssertEqual(result.failedTests.count, 1)
1474+
guard result.failedTests.count == 1 else { return }
1475+
XCTAssertTrue(result.failedTests[0].test.contains("testHang"))
13271476
}
13281477

13291478
// MARK: - Parallel Testing Format Tests

0 commit comments

Comments
 (0)