Merge pull request #42394 from DavidGoldman/hermeticindexstore

bnbarham · web-flow · commit 47721d25e61c · 2022-05-16T14:09:25.000-07:00
Support hermetic indexing information + introduce `-file-prefix-map`
diff --git a/include/swift/AST/DiagnosticsFrontend.def b/include/swift/AST/DiagnosticsFrontend.def
@@ -338,12 +338,9 @@ WARNING(framework_search_path_includes_framework_extension,none,
 ERROR(error_optimization_remark_pattern, none, "%0 in '%1'",
       (StringRef, StringRef))
 
-ERROR(error_invalid_debug_prefix_map, none,
-      "values for '-debug-prefix-map' must be in the format 'original=remapped'"
-      ", but '%0' was provided", (StringRef))
-ERROR(error_invalid_coverage_prefix_map, none,
-      "values for '-coverage-prefix-map' must be in the format "
-      "'original=remapped', but '%0' was provided", (StringRef))
+ERROR(error_opt_invalid_mapping, none,
+      "values for '%0' must be in the format 'original=remapped', but '%1' was "
+      "provided", (StringRef, StringRef))
 
 ERROR(invalid_vfs_overlay_file,none,
       "invalid virtual overlay file '%0'", (StringRef))
diff --git a/include/swift/AST/IRGenOptions.h b/include/swift/AST/IRGenOptions.h
@@ -254,6 +254,11 @@ class IRGenOptions {
   /// Path prefixes that should be rewritten in coverage info.
   PathRemapper CoveragePrefixMap;
 
+  /// Path prefixes that should be rewritten in info besides debug and coverage
+  /// (use DebugPrefixMap and CoveragePrefixMap for those) - currently just
+  /// indexing info.
+  PathRemapper FilePrefixMap;
+
   /// What level of debug info to generate.
   IRGenDebugInfoLevel DebugInfoLevel : 2;
 
diff --git a/include/swift/Basic/PathRemapper.h b/include/swift/Basic/PathRemapper.h
@@ -25,6 +25,7 @@
 #define SWIFT_BASIC_PATHREMAPPER_H
 
 #include "swift/Basic/LLVM.h"
+#include "clang/Basic/PathRemapper.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Twine.h"
 
@@ -57,6 +58,15 @@ class PathRemapper {
                 Path.substr(Mapping.first.size())).str();
     return Path.str();
   }
+
+  /// Returns the Clang PathRemapper equivalent, suitable for use with Clang
+  /// APIs.
+  clang::PathRemapper asClangPathRemapper() const {
+    clang::PathRemapper Remapper;
+    for (const auto &Mapping : PathMappings)
+      Remapper.addMapping(Mapping.first, Mapping.second);
+    return Remapper;
+  }
 };
 
 class PathObfuscator {
diff --git a/include/swift/Index/IndexRecord.h b/include/swift/Index/IndexRecord.h
@@ -14,6 +14,7 @@
 #define SWIFT_INDEX_INDEXRECORD_H
 
 #include "swift/Basic/LLVM.h"
+#include "swift/Basic/PathRemapper.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/StringRef.h"
 
@@ -44,11 +45,14 @@ namespace index {
 /// \param targetTriple The target for this compilation.
 ///
 /// \param dependencyTracker The set of dependencies seen while building.
+///
+/// \param pathRemapper Remapper to use for paths in index data.
 bool indexAndRecord(SourceFile *primarySourceFile, StringRef indexUnitToken,
                     StringRef indexStorePath, bool indexSystemModules,
                     bool skipStdlib, bool isDebugCompilation,
                     StringRef targetTriple,
-                    const DependencyTracker &dependencyTracker);
+                    const DependencyTracker &dependencyTracker,
+                    const PathRemapper &pathRemapper);
 
 /// Index the given module and store the results to \p indexStorePath.
 ///
@@ -76,11 +80,14 @@ bool indexAndRecord(SourceFile *primarySourceFile, StringRef indexUnitToken,
 /// \param targetTriple The target for this compilation.
 ///
 /// \param dependencyTracker The set of dependencies seen while building.
+///
+/// \param pathRemapper Remapper to use for paths in index data.
 bool indexAndRecord(ModuleDecl *module, ArrayRef<std::string> indexUnitTokens,
                     StringRef moduleUnitToken, StringRef indexStorePath,
                     bool indexSystemModules, bool skipStdlib,
                     bool isDebugCompilation, StringRef targetTriple,
-                    const DependencyTracker &dependencyTracker);
+                    const DependencyTracker &dependencyTracker,
+                    const PathRemapper &pathRemapper);
 // FIXME: indexUnitTokens could be StringRef, but that creates an impedance
 // mismatch in the caller.
 
diff --git a/include/swift/Option/Options.td b/include/swift/Option/Options.td
@@ -845,6 +845,9 @@ def debug_prefix_map : Separate<["-"], "debug-prefix-map">,
 def coverage_prefix_map : Separate<["-"], "coverage-prefix-map">,
   Flags<[FrontendOption]>,
   HelpText<"Remap source paths in coverage info">, MetaVarName<"<prefix=replacement>">;
+def file_prefix_map : Separate<["-"], "file-prefix-map">,
+  Flags<[FrontendOption]>,
+  HelpText<"Remap source paths in debug, coverage, and index info">, MetaVarName<"<prefix=replacement>">;
 
 def file_compilation_dir : Separate<["-"], "file-compilation-dir">,
   Flags<[FrontendOption]>, MetaVarName<"<path>">,
diff --git a/lib/Driver/Driver.cpp b/lib/Driver/Driver.cpp
@@ -223,17 +223,16 @@ static void validateDebugInfoArgs(DiagnosticEngine &diags,
     }
   }
 
-  // Check for any -debug-prefix-map options that aren't of the form
+  // Check for any -*-prefix-map options that aren't of the form
   // 'original=remapped' (either side can be empty, however).
-  for (auto A : args.getAllArgValues(options::OPT_debug_prefix_map))
-    if (A.find('=') == StringRef::npos)
-      diags.diagnose(SourceLoc(), diag::error_invalid_debug_prefix_map, A);
-
-  // Check for any -coverage-prefix-map options that aren't of the form
-  // 'original=remapped' (either side can be empty, however).
-  for (auto A : args.getAllArgValues(options::OPT_coverage_prefix_map))
-    if (A.find('=') == StringRef::npos)
-      diags.diagnose(SourceLoc(), diag::error_invalid_coverage_prefix_map, A);
+  for (const Arg *A : args.filtered(options::OPT_debug_prefix_map,
+                                    options::OPT_coverage_prefix_map,
+                                    options::OPT_file_prefix_map)) {
+    StringRef val = A->getValue();
+    if (val.find('=') == StringRef::npos)
+      diags.diagnose(SourceLoc(), diag::error_opt_invalid_mapping,
+                     A->getOption().getPrefixedName(), val);
+  }
 }
 
 static void validateVerifyIncrementalDependencyArgs(DiagnosticEngine &diags,
diff --git a/lib/Driver/ToolChains.cpp b/lib/Driver/ToolChains.cpp
@@ -305,8 +305,9 @@ void ToolChain::addCommonFrontendArgs(const OutputInfo &OI,
   inputArgs.AddAllArgs(arguments, options::OPT_D);
 
   // Pass on file paths that should be remapped in debug info.
-  inputArgs.AddAllArgs(arguments, options::OPT_debug_prefix_map);
-  inputArgs.AddAllArgs(arguments, options::OPT_coverage_prefix_map);
+  inputArgs.AddAllArgs(arguments, options::OPT_debug_prefix_map,
+                                  options::OPT_coverage_prefix_map,
+                                  options::OPT_file_prefix_map);
 
   std::string globalRemapping = getGlobalDebugPathRemapping();
   if (!globalRemapping.empty()) {
diff --git a/lib/Frontend/CompilerInvocation.cpp b/lib/Frontend/CompilerInvocation.cpp
@@ -1165,14 +1165,23 @@ static bool ParseClangImporterArgs(ClangImporterOptions &Opts,
     Opts.ExtraArgs.push_back(A->getValue());
   }
 
-  for (auto A : Args.getAllArgValues(OPT_debug_prefix_map)) {
+  for (const Arg *A : Args.filtered(OPT_file_prefix_map,
+                                    OPT_debug_prefix_map)) {
+    std::string Val(A->getValue());
     // Forward -debug-prefix-map arguments from Swift to Clang as
-    // -fdebug-prefix-map. This is required to ensure DIFiles created there,
-    // like "<swift-imported-modules>", have their paths remapped properly.
+    // -fdebug-prefix-map= and -file-prefix-map as -ffile-prefix-map=.
+    //
+    // This is required to ensure DIFiles created there, like
+    /// "<swift-imported-modules>", as well as index data, have their paths
+    // remapped properly.
+    //
     // (Note, however, that Clang's usage of std::map means that the remapping
     // may not be applied in the same order, which can matter if one mapping is
     // a prefix of another.)
-    Opts.ExtraArgs.push_back("-fdebug-prefix-map=" + A);
+    if (A->getOption().matches(OPT_file_prefix_map))
+      Opts.ExtraArgs.push_back("-ffile-prefix-map=" + Val);
+    else
+      Opts.ExtraArgs.push_back("-fdebug-prefix-map=" + Val);
   }
 
   if (!workingDirectory.empty()) {
@@ -1957,6 +1966,13 @@ static bool ParseIRGenArgs(IRGenOptions &Opts, ArgList &Args,
                      : "-gdwarf_types");
   }
 
+  for (auto A : Args.getAllArgValues(options::OPT_file_prefix_map)) {
+    auto SplitMap = StringRef(A).split('=');
+    Opts.FilePrefixMap.addMapping(SplitMap.first, SplitMap.second);
+    Opts.DebugPrefixMap.addMapping(SplitMap.first, SplitMap.second);
+    Opts.CoveragePrefixMap.addMapping(SplitMap.first, SplitMap.second);
+  }
+
   for (auto A : Args.getAllArgValues(options::OPT_debug_prefix_map)) {
     auto SplitMap = StringRef(A).split('=');
     Opts.DebugPrefixMap.addMapping(SplitMap.first, SplitMap.second);
diff --git a/lib/FrontendTool/FrontendTool.cpp b/lib/FrontendTool/FrontendTool.cpp
@@ -1763,7 +1763,8 @@ static void emitIndexDataForSourceFile(SourceFile *PrimarySourceFile,
                                  opts.IndexStorePath, opts.IndexSystemModules,
                                  opts.IndexIgnoreStdlib, isDebugCompilation,
                                  Invocation.getTargetTriple(),
-                                 *Instance.getDependencyTracker());
+                                 *Instance.getDependencyTracker(),
+                                 Invocation.getIRGenOptions().FilePrefixMap);
   } else {
     std::string moduleToken =
         Invocation.getModuleOutputPathForAtMostOnePrimary();
@@ -1778,7 +1779,8 @@ static void emitIndexDataForSourceFile(SourceFile *PrimarySourceFile,
                                  opts.IndexIgnoreStdlib,
                                  isDebugCompilation,
                                  Invocation.getTargetTriple(),
-                                 *Instance.getDependencyTracker());
+                                 *Instance.getDependencyTracker(),
+                                 Invocation.getIRGenOptions().FilePrefixMap);
   }
 }
 
diff --git a/lib/Index/IndexRecord.cpp b/lib/Index/IndexRecord.cpp
@@ -22,6 +22,7 @@
 #include "swift/AST/SourceFile.h"
 #include "swift/AST/Stmt.h"
 #include "swift/AST/Types.h"
+#include "swift/Basic/PathRemapper.h"
 #include "swift/ClangImporter/ClangModule.h"
 #include "swift/Index/Index.h"
 #include "clang/Basic/FileManager.h"
@@ -381,6 +382,7 @@ emitDataForSwiftSerializedModule(ModuleDecl *module,
                                  const clang::CompilerInstance &clangCI,
                                  DiagnosticEngine &diags,
                                  IndexUnitWriter &parentUnitWriter,
+                                 const PathRemapper &pathRemapper,
                                  SourceFile *initialFile);
 
 static void addModuleDependencies(ArrayRef<ImportedModule> imports,
@@ -392,6 +394,7 @@ static void addModuleDependencies(ArrayRef<ImportedModule> imports,
                                   DiagnosticEngine &diags,
                                   IndexUnitWriter &unitWriter,
                                   StringScratchSpace &moduleNameScratch,
+                                  const PathRemapper &pathRemapper,
                                   SourceFile *initialFile = nullptr) {
   auto &fileMgr = clangCI.getFileManager();
 
@@ -442,7 +445,9 @@ static void addModuleDependencies(ArrayRef<ImportedModule> imports,
               emitDataForSwiftSerializedModule(mod, indexStorePath,
                                                indexSystemModules, skipStdlib,
                                                targetTriple, clangCI, diags,
-                                               unitWriter, initialFile);
+                                               unitWriter,
+                                               pathRemapper,
+                                               initialFile);
               withoutUnitName = false;
             }
 
@@ -473,6 +478,7 @@ emitDataForSwiftSerializedModule(ModuleDecl *module,
                                  const clang::CompilerInstance &clangCI,
                                  DiagnosticEngine &diags,
                                  IndexUnitWriter &parentUnitWriter,
+                                 const PathRemapper &pathRemapper,
                                  SourceFile *initialFile) {
   StringRef filename = module->getModuleFilename();
   std::string moduleName = module->getNameStr().str();
@@ -567,11 +573,13 @@ emitDataForSwiftSerializedModule(ModuleDecl *module,
   // For indexing serialized modules 'debug compilation' is irrelevant, so
   // set it to true by default.
   bool isDebugCompilation = true;
+  auto clangRemapper = pathRemapper.asClangPathRemapper();
 
   IndexUnitWriter unitWriter(fileMgr, indexStorePath,
     "swift", swiftVersion, indexUnitToken, moduleName,
     /*MainFile=*/nullptr, isSystem, /*IsModuleUnit=*/true,
-    isDebugCompilation, targetTriple, sysrootPath, getModuleInfoFromOpaqueModule);
+    isDebugCompilation, targetTriple, sysrootPath,
+    clangRemapper, getModuleInfoFromOpaqueModule);
 
   auto FE = fileMgr.getFile(filename);
   bool isSystemModule = module->isSystemModule();
@@ -590,7 +598,7 @@ emitDataForSwiftSerializedModule(ModuleDecl *module,
   StringScratchSpace moduleNameScratch;
   addModuleDependencies(imports, indexStorePath, indexSystemModules, skipStdlib,
                         targetTriple, clangCI, diags, unitWriter,
-                        moduleNameScratch, initialFile);
+                        moduleNameScratch, pathRemapper, initialFile);
 
   if (unitWriter.write(error)) {
     diags.diagnose(SourceLoc(), diag::error_write_index_unit, error);
@@ -607,19 +615,21 @@ recordSourceFileUnit(SourceFile *primarySourceFile, StringRef indexUnitToken,
                      StringRef targetTriple,
                      ArrayRef<const clang::FileEntry *> fileDependencies,
                      const clang::CompilerInstance &clangCI,
+                     const PathRemapper &pathRemapper,
                      DiagnosticEngine &diags) {
   auto &fileMgr = clangCI.getFileManager();
   auto *module = primarySourceFile->getParentModule();
   bool isSystem = module->isSystemModule();
   auto mainFile = fileMgr.getFile(primarySourceFile->getFilename());
+  auto clangRemapper = pathRemapper.asClangPathRemapper();
   // FIXME: Get real values for the following.
   StringRef swiftVersion;
   StringRef sysrootPath = clangCI.getHeaderSearchOpts().Sysroot;
   IndexUnitWriter unitWriter(
       fileMgr, indexStorePath, "swift", swiftVersion, indexUnitToken,
       module->getNameStr(), mainFile ? *mainFile : nullptr, isSystem,
       /*isModuleUnit=*/false, isDebugCompilation, targetTriple, sysrootPath,
-      getModuleInfoFromOpaqueModule);
+      clangRemapper, getModuleInfoFromOpaqueModule);
 
   // Module dependencies.
   SmallVector<ImportedModule, 8> imports;
@@ -630,7 +640,7 @@ recordSourceFileUnit(SourceFile *primarySourceFile, StringRef indexUnitToken,
   StringScratchSpace moduleNameScratch;
   addModuleDependencies(imports, indexStorePath, indexSystemModules, skipStdlib,
                         targetTriple, clangCI, diags, unitWriter,
-                        moduleNameScratch, primarySourceFile);
+                        moduleNameScratch, pathRemapper, primarySourceFile);
 
   // File dependencies.
   for (auto *F : fileDependencies)
@@ -684,7 +694,8 @@ bool index::indexAndRecord(SourceFile *primarySourceFile,
                            bool skipStdlib,
                            bool isDebugCompilation,
                            StringRef targetTriple,
-                           const DependencyTracker &dependencyTracker) {
+                           const DependencyTracker &dependencyTracker,
+                           const PathRemapper &pathRemapper) {
   auto &astContext = primarySourceFile->getASTContext();
   auto &clangCI = astContext.getClangModuleLoader()->getClangInstance();
   auto &diags = astContext.Diags;
@@ -712,7 +723,7 @@ bool index::indexAndRecord(SourceFile *primarySourceFile,
                               indexStorePath, indexSystemModules, skipStdlib,
                               isDebugCompilation, targetTriple,
                               fileDependencies.getArrayRef(),
-                              clangCI, diags);
+                              clangCI, pathRemapper, diags);
 }
 
 bool index::indexAndRecord(ModuleDecl *module,
@@ -723,7 +734,8 @@ bool index::indexAndRecord(ModuleDecl *module,
                            bool skipStdlib,
                            bool isDebugCompilation,
                            StringRef targetTriple,
-                           const DependencyTracker &dependencyTracker) {
+                           const DependencyTracker &dependencyTracker,
+                           const PathRemapper &pathRemapper) {
   auto &astContext = module->getASTContext();
   auto &clangCI = astContext.getClangModuleLoader()->getClangInstance();
   auto &diags = astContext.Diags;
@@ -759,7 +771,7 @@ bool index::indexAndRecord(ModuleDecl *module,
                                indexStorePath, indexSystemModules, skipStdlib,
                                isDebugCompilation, targetTriple,
                                fileDependencies.getArrayRef(),
-                               clangCI, diags))
+                               clangCI, pathRemapper, diags))
         return true;
       unitIndex += 1;
     }
diff --git a/lib/Serialization/Serialization.cpp b/lib/Serialization/Serialization.cpp
@@ -1088,8 +1088,11 @@ void Serializer::writeHeader(const SerializationOptions &options) {
               ++Arg;
               continue;
             }
-          } else if (arg.startswith("-fdebug-prefix-map=")) {
-            // We don't serialize the debug prefix map flags as these
+          } else if (arg.startswith("-fdebug-prefix-map=") ||
+              arg.startswith("-ffile-prefix-map=") ||
+              arg.startswith("-fcoverage-prefix-map=") ||
+              arg.startswith("-fmacro-prefix-map=")) {
+            // We don't serialize any of the prefix map flags as these flags
             // contain absolute paths that are not usable on different
             // machines. These flags are not necessary to compile the
             // clang modules again so are safe to remove.
diff --git a/test/Index/Store/unit-multiple-sourcefiles-remapped.swift b/test/Index/Store/unit-multiple-sourcefiles-remapped.swift
@@ -0,0 +1,40 @@
+//===--- Building source files separately with a module merge at the end
+
+// RUN: %empty-directory(%t)
+// RUN: touch %t/s1.swift %t/s2.swift
+// RUN: %target-swift-frontend -index-store-path %t/idx -file-prefix-map %t=REMAPPED_OUT_DIR -primary-file %t/s1.swift %t/s2.swift -o %t/s1.o -c -module-name main -emit-module -emit-module-path %t/s1.swiftmodule
+// RUN: %target-swift-frontend -index-store-path %t/idx -file-prefix-map %t=REMAPPED_OUT_DIR %t/s1.swift -primary-file %t/s2.swift -o %t/s2.o -c -module-name main -emit-module -emit-module-path %t/s2.swiftmodule
+// RUN: %target-swift-frontend -index-store-path %t/idx -file-prefix-map %t=REMAPPED_OUT_DIR %t/s1.swiftmodule %t/s2.swiftmodule -emit-module -o %t/main.swiftmodule -module-name main
+// RUN: c-index-test core -print-unit %t/idx | %FileCheck %s
+
+//===--- Building source files together (e.g. WMO)
+
+// RUN: %empty-directory(%t)
+// RUN: touch %t/s1.swift %t/s2.swift
+// RUN: %target-swift-frontend -index-store-path %t/idx -file-prefix-map %t=REMAPPED_OUT_DIR %t/s1.swift %t/s2.swift -o %t/s1.o -o %t/s2.o -c -module-name main -emit-module -emit-module-path %t/main.swiftmodule
+// RUN: c-index-test core -print-unit %t/idx | %FileCheck %s
+
+//===--- Building separately but with relative paths for the source file inputs
+
+// RUN: %empty-directory(%t)
+// RUN: cd %t
+// RUN: touch %t/s1.swift %t/s2.swift
+// RUN: %target-swift-frontend -index-store-path idx -file-prefix-map %t=REMAPPED_OUT_DIR -primary-file s1.swift s2.swift -o s1.o -c -module-name main -emit-module -emit-module-path s1.swiftmodule
+// RUN: %target-swift-frontend -index-store-path idx -file-prefix-map %t=REMAPPED_OUT_DIR s1.swift -primary-file s2.swift -o s2.o -c -module-name main -emit-module -emit-module-path s2.swiftmodule
+// RUN: %target-swift-frontend -index-store-path idx -file-prefix-map %t=REMAPPED_OUT_DIR s1.swiftmodule s2.swiftmodule -emit-module -o main.swiftmodule -module-name main
+// RUN: c-index-test core -print-unit idx | %FileCheck %s
+// CHECK-NOT: main.swiftmodule-{{[A-Z0-9]*}}
+
+// CHECK: s1.o-{{2LQAU7D8TZHZ8|2RHC8ZJFDYDW4}}
+// CHECK: --------
+// CHECK: out-file: REMAPPED_OUT_DIR{{/|\\}}s1.o
+// CHECK: DEPEND START
+// CHECK: Unit | system | {{.*}}Swift.swiftmodule
+// CHECK: DEPEND END
+
+// CHECK: s2.o-{{2OIL2LG8UULK6|15MCL6ZLKZKNL}}
+// CHECK: --------
+// CHECK: out-file: REMAPPED_OUT_DIR{{/|\\}}s2.o
+// CHECK: DEPEND START
+// CHECK: Unit | system | {{.*}}Swift.swiftmodule
+// CHECK: DEPEND END
diff --git a/test/Index/Store/unit-one-sourcefile-remapped.swift b/test/Index/Store/unit-one-sourcefile-remapped.swift
diff --git a/test/Index/Store/unit-pcm-dependency-remapped.swift b/test/Index/Store/unit-pcm-dependency-remapped.swift
