Merge pull request #1949 from sina-mahdavi/sina-mahdavi/new-scanner-prefix-map-option

Add `-scanner-prefix-map-paths` option and fix multiArg option parsing

Author: cachemeifyoucan

Sources/SwiftDriver/Driver/Driver.swift

@@ -303,7 +303,7 @@ public struct Driver {
       return ($0.key, $0.value)
     }
     do {
-      guard isFrontendArgSupported(.scannerPrefixMap) else {
+      guard isFrontendArgSupported(.cacheReplayPrefixMap) else {
         return []
       }
       if let sdkMapping = scannerPrefixMapSDK,
@@ -4015,14 +4015,20 @@ extension Driver {
 
   static func computeScanningPrefixMapper(_ parsedOptions: inout ParsedOptions) throws -> [AbsolutePath: AbsolutePath] {
     var mapping: [AbsolutePath: AbsolutePath] = [:]
-    for opt in parsedOptions.arguments(for: .scannerPrefixMap) {
-      let pluginArg = opt.argument.asSingle.split(separator: "=", maxSplits: 1)
-      if pluginArg.count != 2 {
-        throw Error.invalidArgumentValue(Option.scannerPrefixMap.spelling, opt.argument.asSingle)
+    for opt in parsedOptions.arguments(for: .scannerPrefixMapPaths, .scannerPrefixMap) {
+      if opt.option == .scannerPrefixMapPaths {
+        let key = try AbsolutePath(validating: opt.argument.asMultiple[0])
+        let value = try AbsolutePath(validating: opt.argument.asMultiple[1])
+        mapping[key] = value
+      } else {
+        let pluginArg = opt.argument.asSingle.split(separator: "=", maxSplits: 1)
+        if pluginArg.count != 2 {
+          throw Error.invalidArgumentValue(Option.scannerPrefixMap.spelling, opt.argument.asSingle)
+        }
+        let key = try AbsolutePath(validating: String(pluginArg[0]))
+        let value = try AbsolutePath(validating: String(pluginArg[1]))
+        mapping[key] = value
       }
-      let key = try AbsolutePath(validating: String(pluginArg[0]))
-      let value = try AbsolutePath(validating: String(pluginArg[1]))
-      mapping[key] = value
     }
     return mapping
   }

Sources/SwiftDriver/ExplicitModuleBuilds/ExplicitDependencyBuildPlanner.swift

@@ -62,6 +62,9 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT
   /// Does this compile support `.explicitInterfaceModuleBuild`
   private let supportsExplicitInterfaceBuild: Bool
 
+  /// status of the scanner prefix mapping option supported by the frontend
+  private let supportsScannerPrefixMapPaths: Bool
+
   /// Cached command-line additions for all main module compile jobs
   private struct ResolvedModuleDependenciesCommandLineComponents {
     let inputs: [TypedVirtualPath]
@@ -88,14 +91,16 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT
               supportsExplicitInterfaceBuild: Bool = false,
               cas: SwiftScanCAS? = nil,
               prefixMap:  [(AbsolutePath, AbsolutePath)] = [],
-              supportsBridgingHeaderPCHCommand: Bool = false) throws {
+              supportsBridgingHeaderPCHCommand: Bool = false,
+              supportsScannerPrefixMapPaths: Bool = false) throws {
     self.dependencyGraph = dependencyGraph
     self.toolchain = toolchain
     self.integratedDriver = integratedDriver
     self.mainModuleName = dependencyGraph.mainModuleName
     self.reachabilityMap = try dependencyGraph.computeTransitiveClosure()
     self.supportsExplicitInterfaceBuild = supportsExplicitInterfaceBuild
     self.supportsBridgingHeaderPCHCommand = supportsBridgingHeaderPCHCommand
+    self.supportsScannerPrefixMapPaths = supportsScannerPrefixMapPaths
     self.cas = cas
     self.prefixMap = prefixMap
     let mainModuleId: ModuleDependencyId = .swift(dependencyGraph.mainModuleName)
@@ -216,7 +221,12 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT
       // Add prefix mapping. The option is cache invariant so it can be added without affecting cache key.
       for (key, value) in prefixMap {
         commandLine.appendFlag("-cache-replay-prefix-map")
-        commandLine.appendFlag(value.pathString + "=" + key.pathString)
+        if supportsScannerPrefixMapPaths {
+          commandLine.appendFlag(value.pathString)
+          commandLine.appendFlag(key.pathString)
+        } else {
+          commandLine.appendFlag(value.pathString + "=" + key.pathString)
+        }
       }
 
       jobs.append(Job(
@@ -277,7 +287,12 @@ public typealias ExternalTargetModuleDetailsMap = [ModuleDependencyId: ExternalT
       // Add prefix mapping. The option is cache invariant so it can be added without affecting cache key.
       for (key, value) in prefixMap {
         commandLine.appendFlag("-cache-replay-prefix-map")
-        commandLine.appendFlag(value.pathString + "=" + key.pathString)
+        if supportsScannerPrefixMapPaths {
+          commandLine.appendFlag(value.pathString)
+          commandLine.appendFlag(key.pathString)
+        } else {
+          commandLine.appendFlag(value.pathString + "=" + key.pathString)
+        }
       }
 
       jobs.append(Job(

Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift

@@ -125,8 +125,14 @@ public extension Driver {
       commandLine.appendFlag(.scannerModuleValidation)
     }
 
-    if isFrontendArgSupported(.scannerPrefixMap) {
-      // construct `-scanner-prefix-mapper` for scanner.
+    if isFrontendArgSupported(.scannerPrefixMapPaths) {
+      // construct `-scanner-prefix-map-paths` for scanner.
+      for (key, value) in prefixMapping {
+        commandLine.appendFlag(.scannerPrefixMapPaths)
+        commandLine.appendFlag(key.pathString)
+        commandLine.appendFlag(value.pathString)
+      }
+    } else if isFrontendArgSupported(.scannerPrefixMap) {
       for (key, value) in prefixMapping {
         commandLine.appendFlag(.scannerPrefixMap)
         commandLine.appendFlag(key.pathString + "=" + value.pathString)

Sources/SwiftDriver/Jobs/FrontendJobHelpers.swift

@@ -1035,10 +1035,15 @@ extension Driver {
   }
 
   public mutating func addCacheReplayMapping(to commandLine: inout [Job.ArgTemplate]) {
-    if isCachingEnabled && isFrontendArgSupported(.scannerPrefixMap) {
+    if isCachingEnabled && isFrontendArgSupported(.cacheReplayPrefixMap) {
       for (key, value) in prefixMapping {
         commandLine.appendFlag("-cache-replay-prefix-map")
-        commandLine.appendFlag(value.pathString + "=" + key.pathString)
+        if isFrontendArgSupported(.scannerPrefixMapPaths) {
+          commandLine.appendFlag(value.pathString)
+          commandLine.appendFlag(key.pathString)
+        } else {
+          commandLine.appendFlag(value.pathString + "=" + key.pathString)
+        }
       }
     }
   }

Sources/SwiftDriver/Jobs/Planning.swift

@@ -122,7 +122,9 @@ extension Driver {
                                                 cas: cas,
                                                 prefixMap: prefixMapping,
                                                 supportsBridgingHeaderPCHCommand:
-                                                  interModuleDependencyOracle.supportsBridgingHeaderPCHCommand)
+                                                  interModuleDependencyOracle.supportsBridgingHeaderPCHCommand,
+                                                supportsScannerPrefixMapPaths:
+                                                  isFrontendArgSupported(.scannerPrefixMapPaths))
     } else {
       return nil
     }

Sources/SwiftOptions/OptionParsing.swift

@@ -182,10 +182,8 @@ extension OptionTable {
           throw OptionParseError.missingArgument(
             index: index - 1, argument: argument)
         }
-        parsedOptions.addOption(option, argument: .multiple(Array()))
-        arguments[index..<endIdx].map { String($0) }.forEach { parsedOptions.addInput($0) }
+        parsedOptions.addOption(option, argument: .multiple(arguments[index..<endIdx].map { String($0) }))
         index = endIdx
-
       }
     }
     parsedOptions.buildIndex()

Sources/SwiftOptions/Options.swift

@@ -70,7 +70,7 @@ extension Option {
   public static let bypassResilience: Option = Option("-bypass-resilience-checks", .flag, attributes: [.helpHidden, .frontend, .noDriver], helpText: "Ignore all checks for module resilience.")
   public static let cacheCompileJob: Option = Option("-cache-compile-job", .flag, attributes: [.frontend], helpText: "Enable compiler caching")
   public static let cacheDisableReplay: Option = Option("-cache-disable-replay", .flag, attributes: [.frontend, .cacheInvariant], helpText: "Skip loading the compilation result from cache")
-  public static let cacheReplayPrefixMap: Option = Option("-cache-replay-prefix-map", .separate, attributes: [.frontend, .noDriver, .cacheInvariant], metaVar: "<prefix=replacement>", helpText: "Remap paths when replaying outputs from cache")
+  public static let cacheReplayPrefixMap: Option = Option("-cache-replay-prefix-map", .multiArg, attributes: [.frontend, .noDriver, .cacheInvariant], metaVar: "<prefix> <replacement>", helpText: "Remap paths when replaying outputs from cache", numArgs: 2)
   public static let candidateModuleFile: Option = Option("-candidate-module-file", .separate, attributes: [.helpHidden, .frontend, .noDriver], metaVar: "<path>", helpText: "Specify Swift module may be ready to use for an interface")
   public static let casBackendMode: Option = Option("-cas-backend-mode=", .joined, attributes: [.frontend, .noDriver], metaVar: "native|casid|verify", helpText: "CASBackendMode for output kind")
   public static let casBackend: Option = Option("-cas-backend", .flag, attributes: [.frontend, .noDriver], helpText: "Enable using CASBackend for object file output")
@@ -835,7 +835,8 @@ extension Option {
   public static let scannerOutputDir: Option = Option("-scanner-output-dir", .separate, attributes: [.helpHidden, .frontend, .noDriver], helpText: "Directory for generated files from swift dependency scanner")
   public static let scannerPrefixMapSdk: Option = Option("-scanner-prefix-map-sdk", .separate, attributes: [], metaVar: "<path>", helpText: "Remap paths within SDK reported by dependency scanner")
   public static let scannerPrefixMapToolchain: Option = Option("-scanner-prefix-map-toolchain", .separate, attributes: [], metaVar: "<path>", helpText: "Remap paths within toolchain directory reported by dependency scanner")
-  public static let scannerPrefixMap: Option = Option("-scanner-prefix-map", .separate, attributes: [.frontend], metaVar: "<prefix=replacement>", helpText: "Remap paths reported by dependency scanner")
+  public static let scannerPrefixMap: Option = Option("-scanner-prefix-map", .separate, attributes: [], metaVar: "<prefix=replacement>", helpText: "Remap paths reported by dependency scanner")
+  public static let scannerPrefixMapPaths: Option = Option("-scanner-prefix-map-paths", .multiArg, attributes: [.frontend], metaVar: "<prefix> <replacement>", helpText: "Remap paths reported by dependency scanner", numArgs: 2)
   public static let sdkModuleCachePath: Option = Option("-sdk-module-cache-path", .separate, attributes: [.frontend, .doesNotAffectIncrementalBuild, .argumentIsPath], helpText: "Specifies the module cache path for explicitly-built SDK modules")
   public static let sdk: Option = Option("-sdk", .separate, attributes: [.frontend, .synthesizeInterface, .argumentIsPath], metaVar: "<sdk>", helpText: "Compile against <sdk>")
   public static let serializeBreakingChangesPath: Option = Option("-serialize-breaking-changes-path", .separate, attributes: [.noInteractive, .argumentIsPath], metaVar: "<path>", helpText: "Serialize breaking changes found by the API digester to <path>")
@@ -1805,6 +1806,7 @@ extension Option {
       Option.scannerPrefixMapSdk,
       Option.scannerPrefixMapToolchain,
       Option.scannerPrefixMap,
+      Option.scannerPrefixMapPaths,
       Option.sdkModuleCachePath,
       Option.sdk,
       Option.serializeBreakingChangesPath,

Tests/SwiftDriverTests/CachingBuildTests.swift

@@ -902,33 +902,33 @@ final class CachingBuildTests: XCTestCase {
                                      main.nativePathString(escaped: true)] + sdkArgumentsForTesting,
                               env: env,
                               interModuleDependencyOracle: dependencyOracle)
-      guard driver.isFrontendArgSupported(.scannerPrefixMap) else {
+      guard driver.isFrontendArgSupported(.scannerPrefixMapPaths) else {
         throw XCTSkip("frontend doesn't support prefix map")
       }
       let scanLibPath = try XCTUnwrap(driver.getSwiftScanLibPath())
       try dependencyOracle.verifyOrCreateScannerInstance(swiftScanLibPath: scanLibPath)
       let resolver = try ArgsResolver(fileSystem: localFileSystem)
       let scannerCommand = try driver.dependencyScannerInvocationCommand().1.map { try resolver.resolve($0) }
 
-      XCTAssertTrue(scannerCommand.contains("-scanner-prefix-map"))
-      XCTAssertTrue(scannerCommand.contains(try testInputsPath.description + "=/^src"))
+      XCTAssertTrue(scannerCommand.contains("-scanner-prefix-map-paths"))
+      XCTAssertTrue(scannerCommand.contains(try testInputsPath.description))
+      XCTAssertTrue(scannerCommand.contains("/^src"))
 
       let jobs = try driver.planBuild()
       for job in jobs {
         if !job.kind.supportCaching {
           continue
         }
         let command = try job.commandLine.map { try resolver.resolve($0) }
-        // Check all the arguments that are in the temporary directory are remapped.
-        // The only one that is not remapped should be the `-cas-path` that points to
-        // `casPath`.
-        XCTAssertFalse(command.contains {
-          $0.starts(with: path.description) && $0 != casPath.description
-        })
-        /// All source location path should be remapped as well.
-        XCTAssertFalse(try command.contains {
-          $0.starts(with: try testInputsPath.description)
-        })
+        for i in 0..<command.count {
+          if i >= 2 && command[i - 2] == "-cache-replay-prefix-map" { continue }
+          // Check all the arguments that are in the temporary directory are remapped.
+          // The only one that is not remapped should be the `-cas-path` that points to
+          // `casPath`.
+          XCTAssertFalse(command[i] != casPath.description && command[i].starts(with: path.description))
+          /// All source location path should be remapped as well.
+          XCTAssertFalse(command[i].starts(with: try testInputsPath.description))
+        }
         /// command-line that compiles swift should contains -cache-replay-prefix-map
         XCTAssertTrue(command.contains { $0 == "-cache-replay-prefix-map" })
         if job.kind == .compile {

Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift

@@ -174,7 +174,8 @@ final class ExplicitModuleBuildTests: XCTestCase {
               from: ModuleDependenciesInputs.fastDependencyScannerOutput.data(using: .utf8)!)
       var explicitDependencyBuildPlanner =
         try ExplicitDependencyBuildPlanner(dependencyGraph: moduleDependencyGraph,
-                                           toolchain: driver.toolchain)
+                                           toolchain: driver.toolchain,
+                                           supportsScannerPrefixMapPaths: driver.isFrontendArgSupported(.scannerPrefixMapPaths))
       let modulePrebuildJobs =
         try explicitDependencyBuildPlanner.generateExplicitModuleDependenciesBuildJobs()
       XCTAssertEqual(modulePrebuildJobs.count, 4)

Tests/SwiftOptionsTests/OptionParsingTests.swift

@@ -42,8 +42,21 @@ final class SwiftDriverTests: XCTestCase {
     let three = Option("-three", .multiArg, attributes: [], numArgs: 3)
     options.addNewOption(two)
     options.addNewOption(three)
-    let results = try options.parse(["-two", "1", "2", "-three", "1", "2", "3", "-two", "2", "3"], for: .batch)
+    var results = try options.parse(["-two", "1", "2", "-three", "1", "2", "3", "-two", "2", "3"], for: .batch)
     XCTAssertEqual(results.description, "-two 1 2 -three 1 2 3 -two 2 3")
+    // test that the arguments are assigned to their corresponding flag correctly
+    XCTAssertEqual(results.allInputs.count, 0)
+    let twoOpts = results.arguments(for: two)
+    XCTAssertEqual(twoOpts.count, 2)
+    XCTAssertEqual(twoOpts[0].argument.asMultiple[0], "1")
+    XCTAssertEqual(twoOpts[0].argument.asMultiple[1], "2")
+    XCTAssertEqual(twoOpts[1].argument.asMultiple[0], "2")
+    XCTAssertEqual(twoOpts[1].argument.asMultiple[1], "3")
+    let threeOpts = results.arguments(for: three)
+    XCTAssertEqual(threeOpts.count, 1)
+    XCTAssertEqual(threeOpts[0].argument.asMultiple[0], "1")
+    XCTAssertEqual(threeOpts[0].argument.asMultiple[1], "2")
+    XCTAssertEqual(threeOpts[0].argument.asMultiple[2], "3")
     // Check not enough arguments are passed.
     XCTAssertThrowsError(try options.parse(["-two", "1"], for: .batch)) { error in
       XCTAssertEqual(error as? OptionParseError, .missingArgument(index: 0, argument: "-two"))