Merge pull request #68684 from cachemeifyoucan/eng/PR-path-remapping

[CompilerCaching] Path Remapping for canonicalization

Author: cachemeifyoucan

include/swift/AST/DiagnosticsFrontend.def

@@ -485,6 +485,7 @@ REMARK(matching_output_produced,none,
 // Caching related diagnostics
 ERROR(error_caching_no_cas_fs, none,
     "caching is enabled without -cas-fs option, input is not immutable", ())
+ERROR(error_prefix_mapping, none, "cannot create scanner prefix mapping: '%0'", (StringRef))
 
 REMARK(replay_output, none, "replay output file '%0': key '%1'", (StringRef, StringRef))
 REMARK(output_cache_miss, none, "cache miss output file '%0': key '%1'", (StringRef, StringRef))

include/swift/AST/ModuleDependencies.h

@@ -36,6 +36,7 @@
 #include "llvm/CAS/ObjectStore.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/Mutex.h"
+#include "llvm/Support/PrefixMapper.h"
 #include "llvm/Support/VirtualFileSystem.h"
 #include <string>
 #include <unordered_map>
@@ -790,16 +791,18 @@ using ModuleDependenciesKindRefMap =
 /// Track swift dependency
 class SwiftDependencyTracker {
 public:
-  SwiftDependencyTracker(llvm::cas::CachingOnDiskFileSystem &FS)
-      : FS(FS.createProxyFS()) {}
+  SwiftDependencyTracker(llvm::cas::CachingOnDiskFileSystem &FS,
+                         llvm::TreePathPrefixMapper *Mapper)
+      : FS(FS.createProxyFS()), Mapper(Mapper) {}
 
   void startTracking();
-  void addCommonSearchPathDeps(const SearchPathOptions& Opts);
+  void addCommonSearchPathDeps(const SearchPathOptions &Opts);
   void trackFile(const Twine &path) { (void)FS->status(path); }
   llvm::Expected<llvm::cas::ObjectProxy> createTreeFromDependencies();
 
 private:
   llvm::IntrusiveRefCntPtr<llvm::cas::CachingOnDiskFileSystem> FS;
+  llvm::TreePathPrefixMapper *Mapper;
 };
 
 // MARK: SwiftDependencyScanningService
@@ -840,6 +843,9 @@ class SwiftDependencyScanningService {
   /// The common dependencies that is needed for every swift compiler instance.
   std::vector<std::string> CommonDependencyFiles;
 
+  /// File prefix mapper.
+  std::unique_ptr<llvm::TreePathPrefixMapper> Mapper;
+
   /// The global file system cache.
   llvm::Optional<
       clang::tooling::dependencies::DependencyScanningFilesystemSharedCache>
@@ -894,11 +900,11 @@ class SwiftDependencyScanningService {
     return *CacheFS;
   }
 
-  llvm::Optional<SwiftDependencyTracker> createSwiftDependencyTracker() const {
+  llvm::Optional<SwiftDependencyTracker> createSwiftDependencyTracker() {
     if (!CacheFS)
       return llvm::None;
 
-    return SwiftDependencyTracker(*CacheFS);
+    return SwiftDependencyTracker(*CacheFS, Mapper.get());
   }
 
   llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> getClangScanningFS() const {
@@ -912,6 +918,16 @@ class SwiftDependencyScanningService {
     return llvm::vfs::createPhysicalFileSystem();
   }
 
+  bool hasPathMapping() const {
+    return Mapper && !Mapper->getMappings().empty();
+  }
+  llvm::TreePathPrefixMapper *getPrefixMapper() const { return Mapper.get(); }
+  std::string remapPath(StringRef Path) const {
+    if (!Mapper)
+      return Path.str();
+    return Mapper->mapToString(Path);
+  }
+
   /// Wrap the filesystem on the specified `CompilerInstance` with a
   /// caching `DependencyScanningWorkerFilesystem`
   void overlaySharedFilesystemCacheForCompilation(CompilerInstance &Instance);

include/swift/AST/ModuleLoader.h

@@ -34,6 +34,7 @@
 
 namespace llvm {
 class FileCollectorBase;
+class TreePathPrefixMapper;
 namespace vfs {
 class OutputBackend;
 }
@@ -345,6 +346,7 @@ class ModuleLoader {
                         const llvm::DenseSet<clang::tooling::dependencies::ModuleID> &alreadySeenClangModules,
                         clang::tooling::dependencies::DependencyScanningTool &clangScanningTool,
                         InterfaceSubContextDelegate &delegate,
+                        llvm::TreePathPrefixMapper *mapper = nullptr,
                         bool isTestableImport = false) = 0;
 };
 

include/swift/AST/SearchPathOptions.h

@@ -461,6 +461,9 @@ class SearchPathOptions {
   /// Don't look in for compiler-provided modules.
   bool SkipRuntimeLibraryImportPaths = false;
 
+  /// Scanner Prefix Mapper.
+  std::vector<std::string> ScannerPrefixMapper;
+
   /// When set, don't validate module system dependencies.
   ///
   /// If a system header is modified and this is not set, the compiler will

include/swift/ClangImporter/ClangImporter.h

@@ -434,16 +434,18 @@ class ClangImporter final : public ClangModuleLoader {
 
   void verifyAllModules() override;
 
+  using RemapPathCallback = llvm::function_ref<std::string(StringRef)>;
   llvm::SmallVector<std::pair<ModuleDependencyID, ModuleDependencyInfo>, 1> bridgeClangModuleDependencies(
       const clang::tooling::dependencies::ModuleDepsGraph &clangDependencies,
-      StringRef moduleOutputPath);
+      StringRef moduleOutputPath, RemapPathCallback remapPath = nullptr);
 
   llvm::SmallVector<std::pair<ModuleDependencyID, ModuleDependencyInfo>, 1>
   getModuleDependencies(StringRef moduleName, StringRef moduleOutputPath,
                         llvm::IntrusiveRefCntPtr<llvm::cas::CachingOnDiskFileSystem> CacheFS,
                         const llvm::DenseSet<clang::tooling::dependencies::ModuleID> &alreadySeenClangModules,
                         clang::tooling::dependencies::DependencyScanningTool &clangScanningTool,
                         InterfaceSubContextDelegate &delegate,
+                        llvm::TreePathPrefixMapper *mapper,
                         bool isTestableImport = false) override;
 
   void recordBridgingHeaderOptions(

include/swift/Frontend/CachingUtils.h

@@ -20,6 +20,7 @@
 #include "llvm/CAS/ActionCache.h"
 #include "llvm/CAS/CASReference.h"
 #include "llvm/CAS/ObjectStore.h"
+#include "llvm/Support/PrefixMapper.h"
 #include "llvm/Support/VirtualFileSystem.h"
 #include "llvm/Support/VirtualOutputBackend.h"
 #include <memory>
@@ -59,6 +60,10 @@ llvm::Error storeCachedCompilerOutput(llvm::cas::ObjectStore &CAS,
 llvm::Expected<llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem>>
 createCASFileSystem(llvm::cas::ObjectStore &CAS, ArrayRef<std::string> FSRoots,
                     ArrayRef<std::string> IncludeTreeRoots);
-}
+
+std::vector<std::string> remapPathsFromCommandLine(
+    ArrayRef<std::string> Args,
+    llvm::function_ref<std::string(StringRef)> RemapCallback);
+} // namespace swift
 
 #endif

include/swift/Frontend/CompileJobCacheKey.h

@@ -43,6 +43,6 @@ createCompileJobCacheKeyForOutput(llvm::cas::ObjectStore &CAS,
                                   llvm::cas::ObjectRef BaseKey,
                                   StringRef ProducingInput,
                                   file_types::ID OutputType);
-}
+} // namespace swift
 
 #endif

include/swift/Frontend/FrontendOptions.h

@@ -148,6 +148,9 @@ class FrontendOptions {
   /// CacheKey for input file.
   std::string InputFileKey;
 
+  /// CacheReplay PrefixMap.
+  std::vector<std::string> CacheReplayPrefixMap;
+
   /// Number of retry opening an input file if the previous opening returns
   /// bad file descriptor error.
   unsigned BadFileDescriptorRetryCount = 0;

include/swift/Option/FrontendOptions.td

@@ -79,14 +79,14 @@ def backup_module_interface_path_EQ :
 
 } // end let Flags = [FrontendOption, NoDriverOption, CacheInvariant]
 
+def primary_file : Separate<["-"], "primary-file">,
+  Flags<[FrontendOption, NoDriverOption, ArgumentIsPath]>,
+  HelpText<"Produce output for this file, not the whole module">;
 
 let Flags = [FrontendOption, NoDriverOption] in {
 
 def triple : Separate<["-"], "triple">, Alias<target>;
 
-def primary_file : Separate<["-"], "primary-file">,
-  HelpText<"Produce output for this file, not the whole module">;
-
 def frontend_parseable_output : Flag<["-"], "frontend-parseable-output">,
   HelpText<"Emit textual output in a parseable format">;
 

include/swift/Option/Options.td

@@ -1856,6 +1856,22 @@ def clang_scanner_module_cache_path : Separate<["-"], "clang-scanner-module-cach
   Flags<[FrontendOption, DoesNotAffectIncrementalBuild, ArgumentIsPath]>,
   HelpText<"Specifies the Clang dependency scanner module cache path">;
 
+def scanner_prefix_map : Separate<["-"], "scanner-prefix-map">,
+  Flags<[FrontendOption, NewDriverOnlyOption]>,
+  HelpText<"Remap paths reported by dependency scanner">, MetaVarName<"<prefix=replacement>">;
+
+def scanner_prefix_map_sdk : Separate<["-"], "scanner-prefix-map-sdk">,
+  Flags<[NewDriverOnlyOption]>,
+  HelpText<"Remap paths within SDK reported by dependency scanner">, MetaVarName<"<path>">;
+
+def scanner_prefix_map_toolchain : Separate<["-"], "scanner-prefix-map-toolchain">,
+  Flags<[NewDriverOnlyOption]>,
+  HelpText<"Remap paths within toolchain directory reported by dependency scanner">, MetaVarName<"<path>">;
+
+def cache_replay_prefix_map: Separate<["-"], "cache-replay-prefix-map">,
+  Flags<[FrontendOption, NoDriverOption, CacheInvariant]>,
+  HelpText<"Remap paths when replaying outputs from cache">, MetaVarName<"<prefix=replacement>">;
+
 // END ONLY SUPPORTED IN NEW DRIVER