Add swift crash producer support

When swift compiler crashed with swift caching enabled, create a crash
reproducer automatically.

Author: cachemeifyoucan

Sources/SwiftDriver/Driver/Driver.swift

@@ -23,6 +23,7 @@ import struct TSCBasic.AbsolutePath
 import struct TSCBasic.ByteString
 import struct TSCBasic.Diagnostic
 import struct TSCBasic.FileInfo
+import struct TSCBasic.ProcessResult
 import struct TSCBasic.RelativePath
 import struct TSCBasic.SHA256
 import var TSCBasic.localFileSystem
@@ -71,6 +72,7 @@ public struct Driver {
     case conditionalCompilationFlagIsNotValidIdentifier(String)
     case baselineGenerationRequiresTopLevelModule(String)
     case optionRequiresAnother(String, String)
+    case unableToCreateReproducer
     // Explicit Module Build Failures
     case malformedModuleDependency(String, String)
     case missingModuleDependency(String)
@@ -142,6 +144,8 @@ public struct Driver {
         return "generating a baseline with '\(arg)' is only supported with '-emit-module' or '-emit-module-path'"
       case .optionRequiresAnother(let first, let second):
         return "'\(first)' cannot be specified if '\(second)' is not present"
+      case .unableToCreateReproducer:
+        return "failed to create reproducer"
       }
     }
   }
@@ -962,7 +966,7 @@ public struct Driver {
                                                       negative: .disableIncrementalFileHashing,
                                                       default: false)
     self.recordedInputMetadata = .init(uniqueKeysWithValues:
-      Set(inputFiles).compactMap { inputFile -> (TypedVirtualPath, FileMetadata)? in 
+      Set(inputFiles).compactMap { inputFile -> (TypedVirtualPath, FileMetadata)? in
         guard let modTime = try? fileSystem.lastModificationTime(for: inputFile.file) else { return nil }
         if incrementalFileHashes {
             guard let data = try? fileSystem.readFileContents(inputFile.file)  else { return nil }
@@ -1957,7 +1961,8 @@ extension Driver {
       buildRecordInfo: buildRecordInfo,
       showJobLifecycle: showJobLifecycle,
       argsResolver: executor.resolver,
-      diagnosticEngine: diagnosticEngine)
+      diagnosticEngine: diagnosticEngine,
+      reproducerCallback: supportsReproducer ? Driver.generateReproducer : nil)
   }
 
   private mutating func performTheBuild(

Sources/SwiftDriver/Driver/ToolExecutionDelegate.swift

@@ -30,6 +30,7 @@ import struct TSCBasic.Diagnostic
 import struct TSCBasic.ProcessResult
 import var TSCBasic.stderrStream
 import var TSCBasic.stdoutStream
+import class TSCBasic.Process
 
 /// Delegate for printing execution information on the command-line.
 @_spi(Testing) public final class ToolExecutionDelegate: JobExecutionDelegate {
@@ -49,6 +50,8 @@ import var TSCBasic.stdoutStream
     case silent
   }
 
+  public typealias ReproducerCallback = (Job, VirtualPath) -> Job
+
   public let mode: Mode
   public let buildRecordInfo: BuildRecordInfo?
   public let showJobLifecycle: Bool
@@ -58,18 +61,21 @@ import var TSCBasic.stdoutStream
   private var nextBatchQuasiPID: Int
   private let argsResolver: ArgsResolver
   private var batchJobInputQuasiPIDMap = TwoLevelMap<Job, TypedVirtualPath, Int>()
+  private let reproducerCallback: ReproducerCallback?
 
   @_spi(Testing) public init(mode: ToolExecutionDelegate.Mode,
                              buildRecordInfo: BuildRecordInfo?,
                              showJobLifecycle: Bool,
                              argsResolver: ArgsResolver,
-                             diagnosticEngine: DiagnosticsEngine) {
+                             diagnosticEngine: DiagnosticsEngine,
+                             reproducerCallback: ReproducerCallback? = nil) {
     self.mode = mode
     self.buildRecordInfo = buildRecordInfo
     self.showJobLifecycle = showJobLifecycle
     self.diagnosticEngine = diagnosticEngine
     self.argsResolver = argsResolver
     self.nextBatchQuasiPID = ToolExecutionDelegate.QUASI_PID_START
+    self.reproducerCallback = reproducerCallback
   }
 
   public func jobStarted(job: Job, arguments: [String], pid: Int) {
@@ -170,6 +176,13 @@ import var TSCBasic.stdoutStream
     }
   }
 
+  public func getReproducerJob(job: Job, output: VirtualPath) -> Job? {
+    guard let reproducerCallback = reproducerCallback else {
+      return nil
+    }
+    return reproducerCallback(job, output)
+  }
+
   private func emit(_ message: ParsableMessage) {
     // FIXME: Do we need to do error handling here? Can this even fail?
     guard let json = try? message.toJSON() else { return }

Sources/SwiftDriver/Execution/DriverExecutor.swift

@@ -206,6 +206,9 @@ public protocol JobExecutionDelegate {
 
   /// Called when a job is skipped.
   func jobSkipped(job: Job)
+
+  /// Create a new job that constructs a reproducer for the providing job.
+  func getReproducerJob(job: Job, output: VirtualPath) -> Job?
 }
 
 @_spi(Testing) public extension ProcessEnvironmentBlock {

Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift

@@ -1071,6 +1071,23 @@ extension Driver {
   }
 }
 
+// Generate reproducer.
+extension Driver {
+  var supportsReproducer: Bool {
+    isFrontendArgSupported(.genReproducer) && enableCaching
+  }
+
+  static func generateReproducer(_ job: Job, _ output: VirtualPath) -> Job {
+    var reproJob = job
+    reproJob.commandLine.appendFlag(.genReproducer)
+    reproJob.commandLine.appendFlag(.genReproducerDir)
+    reproJob.commandLine.appendPath(output)
+    reproJob.outputs.removeAll()
+    reproJob.outputCacheKeys.removeAll()
+    return reproJob
+  }
+}
+
 extension ParsedOptions {
   /// Checks whether experimental embedded mode is enabled.
   var isEmbeddedEnabled: Bool {

Sources/SwiftDriver/Jobs/PrebuiltModulesJob.swift

@@ -270,13 +270,19 @@ fileprivate class ModuleCompileDelegate: JobExecutionDelegate {
   static func canHandle(job: Job) -> Bool {
     return job.kind == .compile
   }
+  func getReproducerJob(job: Job, output: VirtualPath) -> Job? {
+    nil
+  }
 }
 
 fileprivate class ABICheckingDelegate: JobExecutionDelegate {
   let verbose: Bool
   let logPath: AbsolutePath?
 
   func jobSkipped(job: Job) {}
+  func getReproducerJob(job: Job, output: VirtualPath) -> Job? {
+    nil
+  }
 
   public init(_ verbose: Bool, _ logPath: AbsolutePath?) {
     self.verbose = verbose
@@ -339,6 +345,9 @@ public class PrebuiltModuleGenerationDelegate: JobExecutionDelegate {
   public func jobSkipped(job: Job) {
     selectDelegate(job: job).jobSkipped(job: job)
   }
+  public func getReproducerJob(job: Job, output: VirtualPath) -> Job? {
+    selectDelegate(job: job).getReproducerJob(job: job, output: output)
+  }
   public var shouldRunDanglingJobs: Bool {
     return compileDelegate.shouldRunDanglingJobs
   }

Sources/SwiftDriverExecution/MultiJobExecutor.swift

@@ -25,6 +25,7 @@ import protocol TSCBasic.DiagnosticData
 import protocol TSCBasic.FileSystem
 import struct TSCBasic.Diagnostic
 import struct TSCBasic.ProcessResult
+import func TSCBasic.withTemporaryDirectory
 import typealias TSCBasic.ProcessEnvironmentBlock
 import enum TSCUtility.Diagnostics
 
@@ -632,12 +633,14 @@ class ExecuteJobRule: LLBuildRule {
 #if os(Windows)
         case let .abnormal(exception):
           context.diagnosticsEngine.emit(.error_command_exception(kind: job.kind, exception: exception))
+          try handleSignalledJob(for: job)
 #else
         case let .signalled(signal):
           // An interrupt of an individual compiler job means it was deliberately cancelled,
           // most likely by the driver itself. This does not constitute an error.
           if signal != SIGINT {
             context.diagnosticsEngine.emit(.error_command_signalled(kind: job.kind, signal: signal))
+            try handleSignalledJob(for: job)
           }
 #endif
         }
@@ -673,6 +676,23 @@ class ExecuteJobRule: LLBuildRule {
 
     engine.taskIsComplete(value)
   }
+
+  private func handleSignalledJob(for job: Job) throws {
+    try withTemporaryDirectory(dir: fileSystem.tempDirectory, prefix: "swift-reproducer", removeTreeOnDeinit: false) { tempDir in
+      guard let reproJob = context.executorDelegate.getReproducerJob(job: job, output: VirtualPath.absolute(tempDir)) else {
+        return
+      }
+      let arguments: [String] = try context.argsResolver.resolveArgumentList(for: reproJob,
+                                                                             useResponseFiles: .heuristic)
+      let process = try context.processType.launchProcess(arguments: arguments, env: context.env)
+      let reproResult = try process.waitUntilExit()
+      if case .terminated(let code) = reproResult.exitStatus, code == 0 {
+        context.diagnosticsEngine.emit(.note_reproducer_created(tempDir.pathString))
+      } else {
+        context.diagnosticsEngine.emit(.error_failed_to_create_reproducer)
+      }
+    }
+  }
 }
 
 fileprivate extension Job {
@@ -700,4 +720,12 @@ private extension TSCBasic.Diagnostic.Message {
   static func error_command_exception(kind: Job.Kind, exception: UInt32) -> TSCBasic.Diagnostic.Message {
     .error("\(kind.rawValue) command failed due to exception \(exception) (use -v to see invocation)")
   }
+
+  static var error_failed_to_create_reproducer: Diagnostic.Message {
+    .error("failed to create crash reproducer")
+  }
+
+  static func note_reproducer_created(_ path: String) -> Diagnostic.Message {
+    .note("crash reproducer is created at: \(path)")
+  }
 }

Sources/SwiftOptions/Options.swift

@@ -586,6 +586,8 @@ extension Option {
   public static let F: Option = Option("-F", .joinedOrSeparate, attributes: [.frontend, .synthesizeInterface, .argumentIsPath], helpText: "Add directory to framework search path")
   public static let gccToolchain: Option = Option("-gcc-toolchain", .separate, attributes: [.helpHidden, .argumentIsPath], metaVar: "<path>", helpText: "Specify a directory where the clang importer and clang linker can find headers and libraries")
   public static let gdwarfTypes: Option = Option("-gdwarf-types", .flag, attributes: [.frontend], helpText: "Emit full DWARF type info.", group: .g)
+  public static let genReproducerDir: Option = Option("-gen-reproducer-dir", .separate, attributes: [.helpHidden, .frontend, .noDriver], helpText: "Path to directory where reproducers write to.")
+  public static let genReproducer: Option = Option("-gen-reproducer", .flag, attributes: [.helpHidden, .frontend, .noDriver], helpText: "Generate a reproducer for current compilation.")
   public static let generateEmptyBaseline: Option = Option("-generate-empty-baseline", .flag, attributes: [.noDriver], helpText: "Generate an empty baseline")
   public static let generateEmptyBaseline_: Option = Option("--generate-empty-baseline", .flag, alias: Option.generateEmptyBaseline, attributes: [.noDriver], helpText: "Generate an empty baseline")
   public static let generateMigrationScript: Option = Option("-generate-migration-script", .flag, attributes: [.noDriver], helpText: "Compare SDK content in JSON file and generate migration script")
@@ -1553,6 +1555,8 @@ extension Option {
       Option.F,
       Option.gccToolchain,
       Option.gdwarfTypes,
+      Option.genReproducerDir,
+      Option.genReproducer,
       Option.generateEmptyBaseline,
       Option.generateEmptyBaseline_,
       Option.generateMigrationScript,

Tests/SwiftDriverTests/JobExecutorTests.swift

@@ -58,6 +58,10 @@ class JobCollectingDelegate: JobExecutionDelegate {
   }
 
   func jobSkipped(job: Job) {}
+
+  func getReproducerJob(job: Job, output: VirtualPath) -> Job? {
+    nil
+  }
 }
 
 extension DarwinToolchain {