Merge pull request #61649 from xymus/index-swiftinterfaces

xymus · web-flow · commit 47b29b68db7b · 2022-10-31T14:18:45.000-07:00
[Index] Force indexing of system modules to read only from swiftinterfaces
diff --git a/include/swift/AST/ASTContext.h b/include/swift/AST/ASTContext.h
@@ -309,6 +309,10 @@ class ASTContext final {
   /// The name of the SwiftShims module "SwiftShims".
   Identifier SwiftShimsModuleName;
 
+  /// Should we globally ignore swiftmodule files adjacent to swiftinterface
+  /// files?
+  bool IgnoreAdjacentModules = false;
+
   // Define the set of known identifiers.
 #define IDENTIFIER_WITH_NAME(Name, IdStr) Identifier Id_##Name;
 #include "swift/AST/KnownIdentifiers.def"
@@ -1144,9 +1148,12 @@ class ASTContext final {
   ///
   /// \param ModulePath The module's \c ImportPath which describes
   /// the name of the module being loaded, possibly including submodules.
-
+  /// \param AllowMemoryCached Should we allow reuse of an already loaded
+  /// module or force reloading from disk, defaults to true.
+  ///
   /// \returns The requested module, or NULL if the module cannot be found.
-  ModuleDecl *getModule(ImportPath::Module ModulePath);
+  ModuleDecl *
+  getModule(ImportPath::Module ModulePath, bool AllowMemoryCached = true);
 
   /// Attempts to load the matching overlay module for the given clang
   /// module into this ASTContext.
@@ -1173,7 +1180,10 @@ class ASTContext final {
   /// in this context.
   void addLoadedModule(ModuleDecl *M);
 
-public:
+  /// Change the behavior of all loaders to ignore swiftmodules next to
+  /// swiftinterfaces.
+  void setIgnoreAdjacentModules(bool value);
+
   /// Retrieve the current generation number, which reflects the
   /// number of times a module import has caused mass invalidation of
   /// lookup tables.
diff --git a/include/swift/AST/DiagnosticsFrontend.def b/include/swift/AST/DiagnosticsFrontend.def
@@ -255,6 +255,10 @@ ERROR(error_index_failed_status_check,none,
       "failed file status check: %0", (StringRef))
 ERROR(error_index_inputs_more_than_outputs,none,
       "index output filenames do not match input source files", ())
+REMARK(remark_indexing_system_module,none,
+       "indexing system module at %0"
+       "%select{|; skipping because of a broken swiftinterface}1",
+       (StringRef, bool))
 
 ERROR(error_wrong_number_of_arguments,none,
       "wrong number of '%0' arguments (expected %1, got %2)",
diff --git a/include/swift/AST/Module.h b/include/swift/AST/Module.h
@@ -545,7 +545,7 @@ class ModuleDecl
   }
 
   /// Returns true if the module was rebuilt from a module interface instead
-  /// of being build from the full source.
+  /// of being built from the full source.
   bool isBuiltFromInterface() const {
     return Bits.ModuleDecl.IsBuiltFromInterface;
   }
diff --git a/include/swift/AST/ModuleLoader.h b/include/swift/AST/ModuleLoader.h
@@ -249,10 +249,14 @@ class ModuleLoader {
   /// \param path A sequence of (identifier, location) pairs that denote
   /// the dotted module name to load, e.g., AppKit.NSWindow.
   ///
+  /// \param AllowMemoryCache Enables preserving the loaded module in the
+  /// in-memory cache for the next loading attempt.
+  ///
   /// \returns the module referenced, if it could be loaded. Otherwise,
   /// emits a diagnostic and returns NULL.
   virtual
-  ModuleDecl *loadModule(SourceLoc importLoc, ImportPath::Module path) = 0;
+  ModuleDecl *loadModule(SourceLoc importLoc, ImportPath::Module path,
+                         bool AllowMemoryCache = true) = 0;
 
   /// Load extensions to the given nominal type.
   ///
diff --git a/include/swift/Basic/LangOptions.h b/include/swift/Basic/LangOptions.h
@@ -216,6 +216,9 @@ namespace swift {
 
     /// Emit a remark after loading a module.
     bool EnableModuleLoadingRemarks = false;
+
+    /// Emit a remark when indexing a system module.
+    bool EnableIndexingSystemModuleRemarks = false;
     
     /// Emit a remark on early exit in explicit interface build
     bool EnableSkipExplicitInterfaceModuleBuildRemarks = false;
diff --git a/include/swift/ClangImporter/ClangImporter.h b/include/swift/ClangImporter/ClangImporter.h
@@ -222,11 +222,15 @@ class ClangImporter final : public ClangModuleLoader {
   /// \param path A sequence of (identifier, location) pairs that denote
   /// the dotted module name to load, e.g., AppKit.NSWindow.
   ///
+  /// \param AllowMemoryCache Affects only loading serialized Swift modules,
+  /// this parameter has no effect in the ClangImporter.
+  ///
   /// \returns the module referenced, if it could be loaded. Otherwise,
   /// emits a diagnostic and returns NULL.
   virtual ModuleDecl *loadModule(
                         SourceLoc importLoc,
-                        ImportPath::Module path)
+                        ImportPath::Module path,
+                        bool AllowMemoryCache = true)
                       override;
 
   /// Determine whether \c overlayDC is within an overlay module for the
diff --git a/include/swift/Frontend/ModuleInterfaceLoader.h b/include/swift/Frontend/ModuleInterfaceLoader.h
@@ -439,7 +439,8 @@ class ModuleInterfaceLoader : public SerializedModuleLoaderBase {
       StringRef OutPath, StringRef ABIOutputPath,
       bool SerializeDependencyHashes,
       bool TrackSystemDependencies, ModuleInterfaceLoaderOptions Opts,
-      RequireOSSAModules_t RequireOSSAModules);
+      RequireOSSAModules_t RequireOSSAModules,
+      bool silenceInterfaceDiagnostics);
 
   /// Unconditionally build \p InPath (a swiftinterface file) to \p OutPath (as
   /// a swiftmodule file).
diff --git a/include/swift/Option/Options.td b/include/swift/Option/Options.td
@@ -343,6 +343,10 @@ def remark_loading_module : Flag<["-"], "Rmodule-loading">,
   Flags<[FrontendOption, DoesNotAffectIncrementalBuild]>,
   HelpText<"Emit a remark and file path of each loaded module">;
 
+def remark_indexing_system_module : Flag<["-"], "Rindexing-system-module">,
+  Flags<[FrontendOption, DoesNotAffectIncrementalBuild]>,
+  HelpText<"Emit a remark when indexing a system module">;
+
 def remark_skip_explicit_interface_build : Flag<["-"], "Rskip-explicit-interface-build">,
   Flags<[FrontendOption, DoesNotAffectIncrementalBuild]>,
   HelpText<"Emit a remark if an explicit module interface invocation has an early exit because the expected output is up-to-date">;
diff --git a/include/swift/Sema/SourceLoader.h b/include/swift/Sema/SourceLoader.h
@@ -73,7 +73,8 @@ class SourceLoader : public ModuleLoader {
   /// returns NULL.
   virtual ModuleDecl *
   loadModule(SourceLoc importLoc,
-             ImportPath::Module path) override;
+             ImportPath::Module path,
+             bool AllowMemoryCache) override;
 
   /// Load extensions to the given nominal type.
   ///
diff --git a/include/swift/Serialization/SerializedModuleLoader.h b/include/swift/Serialization/SerializedModuleLoader.h
@@ -185,7 +185,8 @@ class SerializedModuleLoaderBase : public ModuleLoader {
   /// emits a diagnostic and returns a FailedImportModule object.
   virtual ModuleDecl *
   loadModule(SourceLoc importLoc,
-             ImportPath::Module path) override;
+             ImportPath::Module path,
+             bool AllowMemoryCache) override;
 
 
   virtual void loadExtensions(NominalTypeDecl *nominal,
@@ -294,7 +295,8 @@ class MemoryBufferSerializedModuleLoader : public SerializedModuleLoaderBase {
 
   ModuleDecl *
   loadModule(SourceLoc importLoc,
-             ImportPath::Module path) override;
+             ImportPath::Module path,
+             bool AllowMemoryCache = true) override;
 
   /// Register a memory buffer that contains the serialized module for the given
   /// access path. This API is intended to be used by LLDB to add swiftmodules
diff --git a/lib/AST/ASTContext.cpp b/lib/AST/ASTContext.cpp
@@ -2201,6 +2201,10 @@ void ASTContext::addLoadedModule(ModuleDecl *M) {
   getImpl().LoadedModules[M->getRealName()] = M;
 }
 
+void ASTContext::setIgnoreAdjacentModules(bool value) {
+  IgnoreAdjacentModules = value;
+}
+
 rewriting::RewriteContext &
 ASTContext::getRewriteContext() {
   auto &rewriteCtx = getImpl().TheRewriteContext;
@@ -2451,17 +2455,19 @@ bool ASTContext::canImportModule(ImportPath::Module ModuleName,
 }
 
 ModuleDecl *
-ASTContext::getModule(ImportPath::Module ModulePath) {
+ASTContext::getModule(ImportPath::Module ModulePath, bool AllowMemoryCached) {
   assert(!ModulePath.empty());
 
-  if (auto *M = getLoadedModule(ModulePath))
-    return M;
+  if (AllowMemoryCached)
+    if (auto *M = getLoadedModule(ModulePath))
+      return M;
 
   auto moduleID = ModulePath[0];
   if (PreModuleImportCallback)
     PreModuleImportCallback(moduleID.Item.str(), false /*=IsOverlay*/);
   for (auto &importer : getImpl().ModuleLoaders) {
-    if (ModuleDecl *M = importer->loadModule(moduleID.Loc, ModulePath)) {
+    if (ModuleDecl *M = importer->loadModule(moduleID.Loc, ModulePath,
+                                             AllowMemoryCached)) {
       if (LangOpts.EnableModuleLoadingRemarks) {
         Diags.diagnose(ModulePath.getSourceRange().Start,
                        diag::module_loaded,
diff --git a/lib/ClangImporter/ClangImporter.cpp b/lib/ClangImporter/ClangImporter.cpp
@@ -2023,7 +2023,8 @@ ModuleDecl *ClangImporter::Implementation::loadModuleClang(
 
 ModuleDecl *
 ClangImporter::loadModule(SourceLoc importLoc,
-                          ImportPath::Module path) {
+                          ImportPath::Module path,
+                          bool AllowMemoryCache) {
   return Impl.loadModule(importLoc, path);
 }
 
diff --git a/lib/Frontend/CompilerInvocation.cpp b/lib/Frontend/CompilerInvocation.cpp
@@ -822,6 +822,8 @@ static bool ParseLangArgs(LangOptions &Opts, ArgList &Args,
 
   Opts.EnableModuleLoadingRemarks = Args.hasArg(OPT_remark_loading_module);
 
+  Opts.EnableIndexingSystemModuleRemarks = Args.hasArg(OPT_remark_indexing_system_module);
+
   Opts.EnableSkipExplicitInterfaceModuleBuildRemarks = Args.hasArg(OPT_remark_skip_explicit_interface_build);
   
   llvm::Triple Target = Opts.Target;
diff --git a/lib/Frontend/ModuleInterfaceBuilder.cpp b/lib/Frontend/ModuleInterfaceBuilder.cpp
@@ -320,11 +320,19 @@ bool ImplicitModuleInterfaceBuilder::buildSwiftModuleInternal(
       llvm::RestorePrettyStackState(savedInnerPrettyStackState);
     };
 
+    Optional<DiagnosticEngine> localDiags;
+    DiagnosticEngine *rebuildDiags = diags;
+    if (silenceInterfaceDiagnostics) {
+      // To silence diagnostics, use a local temporary engine.
+      localDiags.emplace(sourceMgr);
+      rebuildDiags = &*localDiags;
+    }
+
     SubError = (bool)subASTDelegate.runInSubCompilerInstance(
         moduleName, interfacePath, OutPath, diagnosticLoc,
         [&](SubCompilerInstanceInfo &info) {
           auto EBuilder = ExplicitModuleInterfaceBuilder(
-              *info.Instance, diags, sourceMgr, moduleCachePath, backupInterfaceDir,
+              *info.Instance, rebuildDiags, sourceMgr, moduleCachePath, backupInterfaceDir,
               prebuiltCachePath, ABIDescriptorPath, extraDependencies, diagnosticLoc,
               dependencyTracker);
           return EBuilder.buildSwiftModuleFromInterface(
diff --git a/lib/Frontend/ModuleInterfaceBuilder.h b/lib/Frontend/ModuleInterfaceBuilder.h
@@ -47,6 +47,7 @@ class ImplicitModuleInterfaceBuilder {
   const StringRef backupInterfaceDir;
   const StringRef ABIDescriptorPath;
   const bool disableInterfaceFileLock;
+  const bool silenceInterfaceDiagnostics;
   const SourceLoc diagnosticLoc;
   DependencyTracker *const dependencyTracker;
   SmallVector<StringRef, 3> extraDependencies;
@@ -87,6 +88,7 @@ class ImplicitModuleInterfaceBuilder {
       StringRef moduleName, StringRef moduleCachePath,
       StringRef backupInterfaceDir, StringRef prebuiltCachePath,
       StringRef ABIDescriptorPath, bool disableInterfaceFileLock = false,
+      bool silenceInterfaceDiagnostics = false,
       SourceLoc diagnosticLoc = SourceLoc(),
       DependencyTracker *tracker = nullptr)
       : sourceMgr(sourceMgr), diags(diags), subASTDelegate(subASTDelegate),
@@ -95,6 +97,7 @@ class ImplicitModuleInterfaceBuilder {
         backupInterfaceDir(backupInterfaceDir),
         ABIDescriptorPath(ABIDescriptorPath),
         disableInterfaceFileLock(disableInterfaceFileLock),
+        silenceInterfaceDiagnostics(silenceInterfaceDiagnostics),
         diagnosticLoc(diagnosticLoc), dependencyTracker(tracker) {}
 
   /// Ensures the requested file name is added as a dependency of the resulting
diff --git a/lib/Frontend/ModuleInterfaceLoader.cpp b/lib/Frontend/ModuleInterfaceLoader.cpp
@@ -677,7 +677,7 @@ class ModuleInterfaceLoaderImpl {
   std::pair<std::string, std::string> getCompiledModuleCandidates() {
     std::pair<std::string, std::string> result;
     // Should we attempt to load a swiftmodule adjacent to the swiftinterface?
-    bool shouldLoadAdjacentModule = true;
+    bool shouldLoadAdjacentModule = !ctx.IgnoreAdjacentModules;
 
     // Don't use the adjacent swiftmodule for frameworks from the public
     // Frameworks folder of the SDK.
@@ -1031,7 +1031,8 @@ class ModuleInterfaceLoaderImpl {
         ctx.SourceMgr, diagsToUse,
         astDelegate, interfacePath, moduleName, cacheDir,
         prebuiltCacheDir, backupInterfaceDir, StringRef(),
-        Opts.disableInterfaceLock, diagnosticLoc,
+        Opts.disableInterfaceLock,
+        ctx.IgnoreAdjacentModules, diagnosticLoc,
         dependencyTracker);
       // If we found an out-of-date .swiftmodule, we still want to add it as
       // a dependency of the .swiftinterface. That way if it's updated, but
@@ -1063,7 +1064,8 @@ class ModuleInterfaceLoaderImpl {
       ImplicitModuleInterfaceBuilder fallbackBuilder(
         ctx.SourceMgr, &ctx.Diags, astDelegate, backupPath, moduleName, cacheDir,
         prebuiltCacheDir, backupInterfaceDir, StringRef(),
-        Opts.disableInterfaceLock, diagnosticLoc,
+        Opts.disableInterfaceLock,
+        ctx.IgnoreAdjacentModules, diagnosticLoc,
         dependencyTracker);
       if (rebuildInfo.sawOutOfDateModule(modulePath))
         fallbackBuilder.addExtraDependency(modulePath);
@@ -1247,7 +1249,8 @@ bool ModuleInterfaceLoader::buildSwiftModuleFromSwiftInterface(
     StringRef OutPath, StringRef ABIOutputPath,
     bool SerializeDependencyHashes,
     bool TrackSystemDependencies, ModuleInterfaceLoaderOptions LoaderOpts,
-    RequireOSSAModules_t RequireOSSAModules) {
+    RequireOSSAModules_t RequireOSSAModules,
+    bool silenceInterfaceDiagnostics) {
   InterfaceSubContextDelegateImpl astDelegate(
       SourceMgr, &Diags, SearchPathOpts, LangOpts, ClangOpts, LoaderOpts,
       /*CreateCacheDirIfAbsent*/ true, CacheDir, PrebuiltCacheDir,
@@ -1256,7 +1259,8 @@ bool ModuleInterfaceLoader::buildSwiftModuleFromSwiftInterface(
   ImplicitModuleInterfaceBuilder builder(SourceMgr, &Diags, astDelegate, InPath,
                                          ModuleName, CacheDir, PrebuiltCacheDir,
                                          BackupInterfaceDir, ABIOutputPath,
-                                         LoaderOpts.disableInterfaceLock);
+                                         LoaderOpts.disableInterfaceLock,
+                                         silenceInterfaceDiagnostics);
   // FIXME: We really only want to serialize 'important' dependencies here, if
   //        we want to ship the built swiftmodules to another machine.
   auto failed = builder.buildSwiftModule(OutPath, /*shouldSerializeDeps*/true,
@@ -1274,7 +1278,8 @@ bool ModuleInterfaceLoader::buildSwiftModuleFromSwiftInterface(
   ImplicitModuleInterfaceBuilder backupBuilder(SourceMgr, &Diags, astDelegate, backInPath,
                                                ModuleName, CacheDir, PrebuiltCacheDir,
                                                BackupInterfaceDir, ABIOutputPath,
-                                               LoaderOpts.disableInterfaceLock);
+                                               LoaderOpts.disableInterfaceLock,
+                                               silenceInterfaceDiagnostics);
   // Ensure we can rebuild module after user changed the original interface file.
   backupBuilder.addExtraDependency(InPath);
   // FIXME: We really only want to serialize 'important' dependencies here, if
diff --git a/lib/FrontendTool/FrontendTool.cpp b/lib/FrontendTool/FrontendTool.cpp
@@ -422,6 +422,8 @@ static bool buildModuleFromInterface(CompilerInstance &Instance) {
   ModuleInterfaceLoaderOptions LoaderOpts(FEOpts);
   StringRef ABIPath = Instance.getPrimarySpecificPathsForAtMostOnePrimary()
                           .SupplementaryOutputs.ABIDescriptorOutputPath;
+  bool IgnoreAdjacentModules = Instance.hasASTContext() &&
+                               Instance.getASTContext().IgnoreAdjacentModules;
 
   // If an explicit interface build was requested, bypass the creation of a new
   // sub-instance from the interface which will build it in a separate thread,
@@ -443,7 +445,8 @@ static bool buildModuleFromInterface(CompilerInstance &Instance) {
       Invocation.getOutputFilename(), ABIPath,
       FEOpts.SerializeModuleInterfaceDependencyHashes,
       FEOpts.shouldTrackSystemDependencies(), LoaderOpts,
-      RequireOSSAModules_t(Invocation.getSILOptions()));
+      RequireOSSAModules_t(Invocation.getSILOptions()),
+      IgnoreAdjacentModules);
 }
 
 static bool compileLLVMIR(CompilerInstance &Instance) {
diff --git a/lib/Index/IndexRecord.cpp b/lib/Index/IndexRecord.cpp
@@ -513,14 +513,40 @@ emitDataForSwiftSerializedModule(ModuleDecl *module,
   if (*isUptodateOpt)
     return false;
 
-  // FIXME: Would be useful for testing if swift had clang's -Rremark system so
-  // we could output a remark here that we are going to create index data for
-  // a module file.
+  // Reload resilient modules from swiftinterface to avoid indexing
+  // internal details.
+  bool skipIndexingModule = false;
+  if (module->getResilienceStrategy() == ResilienceStrategy::Resilient &&
+      !module->isBuiltFromInterface() &&
+      !module->isStdlibModule()) {
+    module->getASTContext().setIgnoreAdjacentModules(true);
+
+    ImportPath::Module::Builder builder(module->getName());
+    ASTContext &ctx = module->getASTContext();
+    auto reloadedModule = ctx.getModule(builder.get(),
+                                        /*AllowMemoryCached=*/false);
+
+    if (reloadedModule) {
+      module = reloadedModule;
+    } else {
+      // If we can't rebuild from the swiftinterface, don't index this module.
+      skipIndexingModule = true;
+    }
+  }
+
+  if (module->getASTContext().LangOpts.EnableIndexingSystemModuleRemarks) {
+    diags.diagnose(SourceLoc(),
+                   diag::remark_indexing_system_module,
+                   filename, skipIndexingModule);
+  }
 
   // Pairs of (recordFile, groupName).
   std::vector<std::pair<std::string, std::string>> records;
 
-  if (!module->isStdlibModule()) {
+  if (skipIndexingModule) {
+    // Don't add anything to records but keep going so we still mark the module
+    // as indexed to avoid rebuilds of broken swiftinterfaces.
+  } else if (!module->isStdlibModule()) {
     std::string recordFile;
     bool failed = false;
     auto consumer = makeRecordingConsumer(filename.str(), indexStorePath.str(),
diff --git a/lib/Sema/SourceLoader.cpp b/lib/Sema/SourceLoader.cpp
@@ -94,7 +94,8 @@ bool SourceLoader::canImportModule(ImportPath::Module path,
 }
 
 ModuleDecl *SourceLoader::loadModule(SourceLoc importLoc,
-                                     ImportPath::Module path) {
+                                     ImportPath::Module path,
+                                     bool AllowMemoryCache) {
   // FIXME: Swift submodules?
   if (path.size() > 1)
     return nullptr;
diff --git a/lib/Serialization/SerializedModuleLoader.cpp b/lib/Serialization/SerializedModuleLoader.cpp
diff --git a/test/Driver/loaded_module_trace.swift b/test/Driver/loaded_module_trace.swift
diff --git a/test/Driver/loaded_module_trace_env.swift b/test/Driver/loaded_module_trace_env.swift
diff --git a/test/Driver/loaded_module_trace_foundation.swift b/test/Driver/loaded_module_trace_foundation.swift
diff --git a/test/Driver/loaded_module_trace_header.swift b/test/Driver/loaded_module_trace_header.swift
diff --git a/test/Driver/loaded_module_trace_multifile.swift b/test/Driver/loaded_module_trace_multifile.swift
diff --git a/test/Index/index_system_modules_swiftinterfaces.swift b/test/Index/index_system_modules_swiftinterfaces.swift
diff --git a/test/Index/skip-loaded-internal.swift b/test/Index/skip-loaded-internal.swift
diff --git a/test/ModuleInterface/loading-remarks.swift b/test/ModuleInterface/loading-remarks.swift

Original file line number	Diff line number	Diff line change
`@@ -545,7 +545,7 @@ class ModuleDecl`
`545`	`545`	`}`
`546`	`546`
`547`	`547`	`/// Returns true if the module was rebuilt from a module interface instead`
`548`		`- /// of being build from the full source.`
	`548`	`+ /// of being built from the full source.`
`549`	`549`	`bool isBuiltFromInterface() const {`
`550`	`550`	`return Bits.ModuleDecl.IsBuiltFromInterface;`
`551`	`551`	`}`