Merge pull request #258 from ahoppen/ahoppen/no-timeout-errors

Don’t report timeout errors

Author: ahoppen

SourceKitStressTester/Sources/StressTester/SourceKitDocument.swift

@@ -533,21 +533,7 @@ class SourceKitDocument {
         try throwIfInvalid(response, request: info)
         try validateResponse(response)
 
-        // Hack to make ACHNBrowserUI/extensions/Path.swift insideOut-893:884 a
-        // soft timeout. It seems to vary *wildly* between runs. We should
-        // investigate why that is, but just get the stress tester to stop
-        // failing for now.
-        var forceSoftTimeout = false
-        if case let .codeCompleteOpen(document, offset, _) = info,
-           offset == 884,
-           let modification = document.modification,
-           modification.mode == .insideOut,
-           modification.content.count == 893,
-           document.path.hasSuffix("ACHNBrowserUI/extensions/Path.swift") {
-          forceSoftTimeout = true
-        }
-
-        if forceSoftTimeout || requestDuration > TimeInterval(SOURCEKIT_REQUEST_TIMEOUT) / 10 {
+        if requestDuration > TimeInterval(SOURCEKIT_REQUEST_TIMEOUT) / 10 {
           // There was no error in the response, but the request took too long
           // throw a soft timeout.
           throw SourceKitError.softTimeout(request: info, duration: requestDuration, instructions: nil)

SourceKitStressTester/Sources/StressTester/StressTester.swift

@@ -129,7 +129,13 @@ public class StressTester {
         if case SourceKitError.softTimeout(request: let request, duration: _, instructions: let .some(instructions)) = error {
           reportPerformanceMeasurement(request: request, instructions: instructions, reusingASTContext: nil)
         }
-        errors.append(error)
+        if case SourceKitError.timedOut = error {
+          // Ignore timeout errors. In practice, we have always just added the timeouts to the XFails and keeping track
+          // of these timeouts is the major cause of stress tester failures, producing noise.
+          // We use instruction count measurements to keep track of performance.
+        } else {
+          errors.append(error)
+        }
       }
     }