diff --git a/Sources/OutputParser.swift b/Sources/OutputParser.swift index 568f136..10c6f65 100644 --- a/Sources/OutputParser.swift +++ b/Sources/OutputParser.swift @@ -35,6 +35,10 @@ class OutputParser { private var passedTestDurations: [String: Double] = [:] private var failedTestDurations: [String: Double] = [:] + // Crash / signal tracking + private var lastStartedTestName: String? + private var pendingSignalCode: Int? + // Build info tracking - phases grouped by target private var targetPhases: [String: [String]] = [:] // target -> [phase names] private var targetDurations: [String: String] = [:] // target -> duration @@ -451,6 +455,20 @@ class OutputParser { return nil }() + // Safety net: if a test was started but never completed and testRunFailed is set + if testRunFailed, let testName = lastStartedTestName { + let normalizedName = normalizeTestName(testName) + if !hasSeenSimilarTest(normalizedName) { + let message = + pendingSignalCode.map { "Crashed (signal \($0)): last test started before crash" } + ?? "Test did not complete (possible crash or timeout)" + failedTests.append(FailedTest(test: testName, message: message, file: nil, line: nil)) + seenTestNames.insert(normalizedName) + } + lastStartedTestName = nil + pendingSignalCode = nil + } + // Determine build status with priority on parsed results over testRunFailed flag // Issue #52: -skipMacroValidation can set testRunFailed even when tests pass let status: String = { @@ -647,6 +665,8 @@ class OutputParser { shouldParseBuildInfo = false targetDependencies = [:] currentDependencyTarget = nil + lastStartedTestName = nil + pendingSignalCode = nil } private func parseLine(_ line: String) { @@ -700,11 +720,49 @@ class OutputParser { || line.hasPrefix("RegisterWithLaunchServices") || line.hasPrefix("Validate") || line.contains("Fatal error") || (line.hasPrefix("/") && line.contains(".swift:")) // runtime warnings + || line.contains("' started") // XCTest: "Test Case '...' started." + || line.contains("\" started") // Swift Testing: Test "..." started + || line.contains("signal code ") // "Exited with [unexpected] signal code N" + || line.hasPrefix("Restarting after") // crash confirmation if !containsRelevant { return } + // Track "Test Case '...' started." for crash association + if parseStartedTest(line) { + return + } + + // Signal code: "Exited with [unexpected] signal code N" + if line.contains("signal code ") { + if let lastSpace = line.lastIndex(of: " ") { + let codeStr = String(line[line.index(after: lastSpace)...]) + pendingSignalCode = Int(codeStr) + } + return + } + + // Crash confirmation: "Restarting after unexpected exit, crash, or test timeout" + if line.hasPrefix("Restarting after") { + if let testName = lastStartedTestName { + let message: String + if let code = pendingSignalCode { + message = "Crashed (signal \(code)): last test started before crash" + } else { + message = "Crashed: last test started before crash" + } + let normalizedName = normalizeTestName(testName) + if !hasSeenSimilarTest(normalizedName) { + failedTests.append(FailedTest(test: testName, message: message, file: nil, line: nil)) + seenTestNames.insert(normalizedName) + } + } + lastStartedTestName = nil + pendingSignalCode = nil + return + } + // Parse parallel test scheduling lines: [N/TOTAL] Testing Module.Class/method if line.contains("] Testing "), let match = line.firstMatch(of: Self.parallelTestSchedulingRegex) { if let _ = Int(match.1), let total = Int(match.2) { @@ -727,6 +785,11 @@ class OutputParser { if let failedTest = parseFailedTest(line) { let normalizedTestName = normalizeTestName(failedTest.test) + // Clear lastStartedTestName if this failed test matches + if normalizeTestName(lastStartedTestName ?? "") == normalizedTestName { + lastStartedTestName = nil + } + // Check if we've already seen this test name or a similar one if !hasSeenSimilarTest(normalizedTestName) { failedTests.append(failedTest) @@ -761,6 +824,21 @@ class OutputParser { seenErrors.insert(key) errors.append(error) } + // Fatal error + lastStartedTestName → also create FailedTest + if line.contains("Fatal error"), let testName = lastStartedTestName { + let normalizedName = normalizeTestName(testName) + if !hasSeenSimilarTest(normalizedName) { + failedTests.append( + FailedTest( + test: testName, + message: "Crashed (Fatal error): last test started before crash", + file: error.file, + line: error.line + ) + ) + seenTestNames.insert(normalizedName) + } + } } else if let warning = parseWarning(line) { let key = "\(warning.file ?? ""):\(warning.line ?? 0):\(warning.message)" if !seenWarnings.contains(key) { @@ -1008,8 +1086,48 @@ class OutputParser { return false } + // MARK: - Crash Association + + /// Parses "Test Case '...' started." or "◇ Test "..." started." lines. + /// Updates `lastStartedTestName` for crash association. + private func parseStartedTest(_ line: String) -> Bool { + // XCTest: Test Case '-[Module.Class testMethod]' started. + // XCTest parallel: Test case '-[Module.Class testMethod]' started. + if (line.hasPrefix("Test Case '") || line.hasPrefix("Test case '")) + && line.contains("' started") + { + let prefixLength = 11 // "Test Case '" or "Test case '" + let startIndex = line.index(line.startIndex, offsetBy: prefixLength) + if let endQuote = line.range(of: "' started", range: startIndex ..< line.endIndex) { + lastStartedTestName = String(line[startIndex ..< endQuote.lowerBound]) + } + return true + } + + // Swift Testing: ◇ Test "shouldCrash()" started. + // or: ◇ Test functionName() started. + if line.hasPrefix("◇ Test ") { + let afterPrefix = line.index(line.startIndex, offsetBy: "◇ Test ".count) + if let result = extractSwiftTestingName(from: line, after: afterPrefix) { + let afterName = line[result.endIndex...] + if afterName.hasPrefix(" started") { + lastStartedTestName = result.name + return true + } + } + } + + return false + } + private func recordPassedTest(named testName: String, duration: Double? = nil) { let normalizedTestName = normalizeTestName(testName) + + // Clear lastStartedTestName if this passed test matches + if normalizeTestName(lastStartedTestName ?? "") == normalizedTestName { + lastStartedTestName = nil + } + guard seenPassedTestNames.insert(normalizedTestName).inserted else { return } diff --git a/Tests/ParsingTests.swift b/Tests/ParsingTests.swift index ab88ede..c5d175d 100644 --- a/Tests/ParsingTests.swift +++ b/Tests/ParsingTests.swift @@ -1324,6 +1324,155 @@ final class ParsingTests: XCTestCase { XCTAssertEqual(result.errors[0].file, "TestProjectTests/TestProjectTests.swift") XCTAssertEqual(result.errors[0].line, 5) XCTAssertEqual(result.errors[0].message, "Fatal error") + // Crash should also be associated with the last started test + XCTAssertEqual(result.failedTests.count, 1) + guard result.failedTests.count == 1 else { return } + XCTAssertTrue(result.failedTests[0].test.contains("testExample")) + } + + // MARK: - Crash Association Tests + + func testCrashSignalAssociatedWithLastStartedTest() { + let parser = OutputParser() + let input = """ + Test Case '-[MyTests.CrashTests testDivideByZero]' started. + Exited with unexpected signal code 5 + Restarting after unexpected exit, crash, or test timeout + ** TEST FAILED ** + """ + + let result = parser.parse(input: input) + + XCTAssertEqual(result.status, "failed") + XCTAssertEqual(result.failedTests.count, 1) + guard result.failedTests.count == 1 else { return } + XCTAssertTrue(result.failedTests[0].test.contains("testDivideByZero")) + XCTAssertTrue(result.failedTests[0].message.contains("signal 5")) + } + + func testCrashSignalWithoutUnexpected() { + let parser = OutputParser() + let input = """ + Test Case '-[MyTests.CrashTests testAbort]' started. + Exited with signal code 6 + Restarting after unexpected exit, crash, or test timeout + ** TEST FAILED ** + """ + + let result = parser.parse(input: input) + + XCTAssertEqual(result.failedTests.count, 1) + guard result.failedTests.count == 1 else { return } + XCTAssertTrue(result.failedTests[0].test.contains("testAbort")) + XCTAssertTrue(result.failedTests[0].message.contains("signal 6")) + } + + func testFatalErrorAssociatedWithLastStartedTest() { + let parser = OutputParser() + let input = """ + Test Case '-[MyTests.CrashTests testPrecondition]' started. + /path/to/MyTests.swift:42: Fatal error: Precondition failed + Restarting after unexpected exit, crash, or test timeout + ** TEST FAILED ** + """ + + let result = parser.parse(input: input) + + // Fatal error should be in errors (parser strips "Fatal error: " prefix) + XCTAssertEqual(result.errors.count, 1) + XCTAssertEqual(result.errors[0].message, "Precondition failed") + // AND also in failedTests (associated with the started test) + XCTAssertEqual(result.failedTests.count, 1) + guard result.failedTests.count == 1 else { return } + XCTAssertTrue(result.failedTests[0].test.contains("testPrecondition")) + } + + func testCrashWithNoStartedTestProducesNoFailedTest() { + let parser = OutputParser() + let input = """ + Exited with unexpected signal code 5 + Restarting after unexpected exit, crash, or test timeout + ** TEST FAILED ** + """ + + let result = parser.parse(input: input) + + // No "started" line → no test to associate with + XCTAssertEqual(result.failedTests.count, 0) + } + + func testStartedTestClearsAfterPass() { + let parser = OutputParser() + let input = """ + Test Case '-[MyTests.OKTests testPass]' started. + Test Case '-[MyTests.OKTests testPass]' passed (0.001 seconds). + Test Case '-[MyTests.CrashTests testCrash]' started. + Exited with unexpected signal code 5 + Restarting after unexpected exit, crash, or test timeout + ** TEST FAILED ** + """ + + let result = parser.parse(input: input) + + // Only testCrash should be in failedTests, not testPass + XCTAssertEqual(result.failedTests.count, 1) + guard result.failedTests.count == 1 else { return } + XCTAssertTrue(result.failedTests[0].test.contains("testCrash")) + XCTAssertFalse(result.failedTests[0].test.contains("testPass")) + } + + func testSwiftTestingStartedFormat() { + let parser = OutputParser() + let input = """ + ◇ Test "shouldCrash()" started. + Exited with unexpected signal code 5 + Restarting after unexpected exit, crash, or test timeout + ** TEST FAILED ** + """ + + let result = parser.parse(input: input) + + XCTAssertEqual(result.failedTests.count, 1) + guard result.failedTests.count == 1 else { return } + XCTAssertTrue(result.failedTests[0].test.contains("shouldCrash")) + } + + func testCrashInFullTestSuiteOutput() { + let parser = OutputParser() + let input = """ + Test Suite 'All tests' started at 2024-01-15 10:00:00.000. + Test Suite 'MyTests.xctest' started at 2024-01-15 10:00:00.000. + Test Suite 'CrashTests' started at 2024-01-15 10:00:00.000. + Test Case '-[MyTests.CrashTests testAssertCrash]' started. + Exited with unexpected signal code 5 + Restarting after unexpected exit, crash, or test timeout + Test Suite 'CrashTests' failed at 2024-01-15 10:00:01.000. + Executed 1 test, with 1 failure in 0.500 seconds + ** TEST FAILED ** + """ + + let result = parser.parse(input: input) + + XCTAssertEqual(result.status, "failed") + XCTAssertEqual(result.failedTests.count, 1) + guard result.failedTests.count == 1 else { return } + XCTAssertTrue(result.failedTests[0].test.contains("testAssertCrash")) + } + + func testEndOfParseSafetyNet() { + let parser = OutputParser() + // "started" but NO "Restarting" line — safety net should catch it + let input = """ + Test Case '-[MyTests.CrashTests testHang]' started. + ** TEST FAILED ** + """ + + let result = parser.parse(input: input) + + XCTAssertEqual(result.status, "failed") + XCTAssertEqual(result.failedTests.count, 1) + guard result.failedTests.count == 1 else { return } + XCTAssertTrue(result.failedTests[0].test.contains("testHang")) } // MARK: - Parallel Testing Format Tests