[DepScan] Teach dependency scanner to remap path for canonicalization

Allow DependencyScanner to canonicalize path using a prefix map. When
option `-scanner-prefix-map` option is used, dependency scanner will
remap all the input paths in following:
* all the paths in the CAS file system or clang include tree
* all the paths related to input on the command-line returned by scanner

This allows all the input paths to be canonicalized so cache key can be
computed reguardless of the exact on disk path.

The sourceFile field is not remapped so build system can track the exact
file as on the local file system.

Author: cachemeifyoucan

include/swift/AST/DiagnosticsFrontend.def

@@ -485,6 +485,7 @@ REMARK(matching_output_produced,none,
 // Caching related diagnostics
 ERROR(error_caching_no_cas_fs, none,
     "caching is enabled without -cas-fs option, input is not immutable", ())
+ERROR(error_prefix_mapping, none, "cannot create scanner prefix mapping: '%0'", (StringRef))
 
 REMARK(replay_output, none, "replay output file '%0': key '%1'", (StringRef, StringRef))
 REMARK(output_cache_miss, none, "cache miss output file '%0': key '%1'", (StringRef, StringRef))

include/swift/AST/ModuleDependencies.h

@@ -36,6 +36,7 @@
 #include "llvm/CAS/ObjectStore.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/Mutex.h"
+#include "llvm/Support/PrefixMapper.h"
 #include "llvm/Support/VirtualFileSystem.h"
 #include <string>
 #include <unordered_map>
@@ -790,16 +791,18 @@ using ModuleDependenciesKindRefMap =
 /// Track swift dependency
 class SwiftDependencyTracker {
 public:
-  SwiftDependencyTracker(llvm::cas::CachingOnDiskFileSystem &FS)
-      : FS(FS.createProxyFS()) {}
+  SwiftDependencyTracker(llvm::cas::CachingOnDiskFileSystem &FS,
+                         llvm::TreePathPrefixMapper *Mapper)
+      : FS(FS.createProxyFS()), Mapper(Mapper) {}
 
   void startTracking();
-  void addCommonSearchPathDeps(const SearchPathOptions& Opts);
+  void addCommonSearchPathDeps(const SearchPathOptions &Opts);
   void trackFile(const Twine &path) { (void)FS->status(path); }
   llvm::Expected<llvm::cas::ObjectProxy> createTreeFromDependencies();
 
 private:
   llvm::IntrusiveRefCntPtr<llvm::cas::CachingOnDiskFileSystem> FS;
+  llvm::TreePathPrefixMapper *Mapper;
 };
 
 // MARK: SwiftDependencyScanningService
@@ -840,6 +843,9 @@ class SwiftDependencyScanningService {
   /// The common dependencies that is needed for every swift compiler instance.
   std::vector<std::string> CommonDependencyFiles;
 
+  /// File prefix mapper.
+  std::unique_ptr<llvm::TreePathPrefixMapper> Mapper;
+
   /// The global file system cache.
   llvm::Optional<
       clang::tooling::dependencies::DependencyScanningFilesystemSharedCache>
@@ -894,11 +900,11 @@ class SwiftDependencyScanningService {
     return *CacheFS;
   }
 
-  llvm::Optional<SwiftDependencyTracker> createSwiftDependencyTracker() const {
+  llvm::Optional<SwiftDependencyTracker> createSwiftDependencyTracker() {
     if (!CacheFS)
       return llvm::None;
 
-    return SwiftDependencyTracker(*CacheFS);
+    return SwiftDependencyTracker(*CacheFS, Mapper.get());
   }
 
   llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> getClangScanningFS() const {
@@ -912,6 +918,16 @@ class SwiftDependencyScanningService {
     return llvm::vfs::createPhysicalFileSystem();
   }
 
+  bool hasPathMapping() const {
+    return Mapper && !Mapper->getMappings().empty();
+  }
+  llvm::TreePathPrefixMapper *getPrefixMapper() const { return Mapper.get(); }
+  std::string remapPath(StringRef Path) const {
+    if (!Mapper)
+      return Path.str();
+    return Mapper->mapToString(Path);
+  }
+
   /// Wrap the filesystem on the specified `CompilerInstance` with a
   /// caching `DependencyScanningWorkerFilesystem`
   void overlaySharedFilesystemCacheForCompilation(CompilerInstance &Instance);

include/swift/AST/ModuleLoader.h

@@ -34,6 +34,7 @@
 
 namespace llvm {
 class FileCollectorBase;
+class TreePathPrefixMapper;
 namespace vfs {
 class OutputBackend;
 }
@@ -345,6 +346,7 @@ class ModuleLoader {
                         const llvm::DenseSet<clang::tooling::dependencies::ModuleID> &alreadySeenClangModules,
                         clang::tooling::dependencies::DependencyScanningTool &clangScanningTool,
                         InterfaceSubContextDelegate &delegate,
+                        llvm::TreePathPrefixMapper *mapper = nullptr,
                         bool isTestableImport = false) = 0;
 };
 

include/swift/AST/SearchPathOptions.h

@@ -461,6 +461,9 @@ class SearchPathOptions {
   /// Don't look in for compiler-provided modules.
   bool SkipRuntimeLibraryImportPaths = false;
 
+  /// Scanner Prefix Mapper.
+  std::vector<std::string> ScannerPrefixMapper;
+
   /// When set, don't validate module system dependencies.
   ///
   /// If a system header is modified and this is not set, the compiler will

include/swift/ClangImporter/ClangImporter.h

@@ -434,16 +434,18 @@ class ClangImporter final : public ClangModuleLoader {
 
   void verifyAllModules() override;
 
+  using RemapPathCallback = llvm::function_ref<std::string(StringRef)>;
   llvm::SmallVector<std::pair<ModuleDependencyID, ModuleDependencyInfo>, 1> bridgeClangModuleDependencies(
       const clang::tooling::dependencies::ModuleDepsGraph &clangDependencies,
-      StringRef moduleOutputPath);
+      StringRef moduleOutputPath, RemapPathCallback remapPath = nullptr);
 
   llvm::SmallVector<std::pair<ModuleDependencyID, ModuleDependencyInfo>, 1>
   getModuleDependencies(StringRef moduleName, StringRef moduleOutputPath,
                         llvm::IntrusiveRefCntPtr<llvm::cas::CachingOnDiskFileSystem> CacheFS,
                         const llvm::DenseSet<clang::tooling::dependencies::ModuleID> &alreadySeenClangModules,
                         clang::tooling::dependencies::DependencyScanningTool &clangScanningTool,
                         InterfaceSubContextDelegate &delegate,
+                        llvm::TreePathPrefixMapper *mapper,
                         bool isTestableImport = false) override;
 
   void recordBridgingHeaderOptions(

include/swift/Frontend/CachingUtils.h

@@ -20,6 +20,7 @@
 #include "llvm/CAS/ActionCache.h"
 #include "llvm/CAS/CASReference.h"
 #include "llvm/CAS/ObjectStore.h"
+#include "llvm/Support/PrefixMapper.h"
 #include "llvm/Support/VirtualFileSystem.h"
 #include "llvm/Support/VirtualOutputBackend.h"
 #include <memory>
@@ -59,6 +60,10 @@ llvm::Error storeCachedCompilerOutput(llvm::cas::ObjectStore &CAS,
 llvm::Expected<llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem>>
 createCASFileSystem(llvm::cas::ObjectStore &CAS, ArrayRef<std::string> FSRoots,
                     ArrayRef<std::string> IncludeTreeRoots);
-}
+
+std::vector<std::string> remapPathsFromCommandLine(
+    ArrayRef<std::string> Args,
+    llvm::function_ref<std::string(StringRef)> RemapCallback);
+} // namespace swift
 
 #endif

include/swift/Frontend/CompileJobCacheKey.h

@@ -43,6 +43,6 @@ createCompileJobCacheKeyForOutput(llvm::cas::ObjectStore &CAS,
                                   llvm::cas::ObjectRef BaseKey,
                                   StringRef ProducingInput,
                                   file_types::ID OutputType);
-}
+} // namespace swift
 
 #endif

include/swift/Option/FrontendOptions.td

@@ -79,14 +79,14 @@ def backup_module_interface_path_EQ :
 
 } // end let Flags = [FrontendOption, NoDriverOption, CacheInvariant]
 
+def primary_file : Separate<["-"], "primary-file">,
+  Flags<[FrontendOption, NoDriverOption, ArgumentIsPath]>,
+  HelpText<"Produce output for this file, not the whole module">;
 
 let Flags = [FrontendOption, NoDriverOption] in {
 
 def triple : Separate<["-"], "triple">, Alias<target>;
 
-def primary_file : Separate<["-"], "primary-file">,
-  HelpText<"Produce output for this file, not the whole module">;
-
 def frontend_parseable_output : Flag<["-"], "frontend-parseable-output">,
   HelpText<"Emit textual output in a parseable format">;
 

include/swift/Option/Options.td

@@ -1853,6 +1853,18 @@ def clang_scanner_module_cache_path : Separate<["-"], "clang-scanner-module-cach
   Flags<[FrontendOption, DoesNotAffectIncrementalBuild, ArgumentIsPath]>,
   HelpText<"Specifies the Clang dependency scanner module cache path">;
 
+def scanner_prefix_map : Separate<["-"], "scanner-prefix-map">,
+  Flags<[FrontendOption, NewDriverOnlyOption]>,
+  HelpText<"Remap paths reported by dependency scanner">, MetaVarName<"<prefix=replacement>">;
+
+def scanner_prefix_map_sdk : Separate<["-"], "scanner-prefix-map-sdk">,
+  Flags<[NewDriverOnlyOption]>,
+  HelpText<"Remap paths within SDK reported by dependency scanner">, MetaVarName<"<path>">;
+
+def scanner_prefix_map_toolchain : Separate<["-"], "scanner-prefix-map-toolchain">,
+  Flags<[NewDriverOnlyOption]>,
+  HelpText<"Remap paths within toolchain directory reported by dependency scanner">, MetaVarName<"<path>">;
+
 // END ONLY SUPPORTED IN NEW DRIVER
 
 

include/swift/Sema/SourceLoader.h

@@ -103,6 +103,7 @@ class SourceLoader : public ModuleLoader {
                         const llvm::DenseSet<clang::tooling::dependencies::ModuleID> &alreadySeenClangModules,
                         clang::tooling::dependencies::DependencyScanningTool &clangScanningTool,
                         InterfaceSubContextDelegate &delegate,
+                        llvm::TreePathPrefixMapper *mapper,
                         bool isTestableImport) override;
 };
 }