Merge pull request swiftlang#81244 from artemcm/IncrementalScanUnlimitedGrowth

[Dependency Scanning] Fix search path context hashing hole and avoid serializing unnecessary field

Author: artemcm

include/swift/AST/ASTContext.h

@@ -1097,9 +1097,6 @@ class ASTContext final {
   /// $SDKROOT/System/Library/Frameworks/ and $SDKROOT/Library/Frameworks/.
   std::vector<std::string> getDarwinImplicitFrameworkSearchPaths() const;
 
-  /// Return a set of all possible filesystem locations where modules can be found.
-  llvm::StringSet<> getAllModuleSearchPathsSet() const;
-
   /// Load extensions to the given nominal type from the external
   /// module loaders.
   ///

include/swift/AST/ModuleDependencies.h

@@ -787,11 +787,6 @@ class ModuleDependencyInfo {
     return storage->auxiliaryFiles;
   }
 
-  void
-  setAuxiliaryFiles(const ArrayRef<std::string> auxiliaryFiles) {
-    storage->auxiliaryFiles.assign(auxiliaryFiles.begin(), auxiliaryFiles.end());
-  }
-
   bool isStaticLibrary() const {
     if (auto *detail = getAsSwiftInterfaceModule())
       return detail->isStatic;

include/swift/AST/SearchPathOptions.h

@@ -323,6 +323,10 @@ class SearchPathOptions {
     friend bool operator!=(const SearchPath &LHS, const SearchPath &RHS) {
       return !(LHS == RHS);
     }
+    friend llvm::hash_code
+    hash_value(const SearchPath &searchPath) {
+      return llvm::hash_combine(searchPath.Path, searchPath.IsSystem);
+    }
   };
 
 private:
@@ -599,30 +603,17 @@ class SearchPathOptions {
   makeOverlayFileSystem(
       llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> BaseFS) const;
 
-private:
-  static StringRef pathStringFromSearchPath(const SearchPath &next) {
-    return next.Path;
-  };
-
 public:
   /// Return a hash code of any components from these options that should
   /// contribute to a Swift Bridging PCH hash.
   llvm::hash_code getPCHHashComponents() const {
     using llvm::hash_combine;
     using llvm::hash_combine_range;
-
-    using SearchPathView =
-        ArrayRefView<SearchPath, StringRef, pathStringFromSearchPath>;
-    SearchPathView importPathsOnly{ImportSearchPaths};
-    SearchPathView frameworkPathsOnly{FrameworkSearchPaths};
-
     return hash_combine(SDKPath,
-                        // FIXME: Should we include the system-ness of
-                        // search paths too?
-                        hash_combine_range(importPathsOnly.begin(), importPathsOnly.end()),
+                        hash_combine_range(ImportSearchPaths.begin(), ImportSearchPaths.end()),
                         hash_combine_range(VFSOverlayFiles.begin(), VFSOverlayFiles.end()),
-                        hash_combine_range(frameworkPathsOnly.begin(),
-                                           frameworkPathsOnly.end()),
+                        hash_combine_range(FrameworkSearchPaths.begin(),
+                                           FrameworkSearchPaths.end()),
                         hash_combine_range(LibrarySearchPaths.begin(),
                                            LibrarySearchPaths.end()),
                         RuntimeResourcePath,

include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h

@@ -41,7 +41,7 @@ using llvm::BCVBR;
 const unsigned char MODULE_DEPENDENCY_CACHE_FORMAT_SIGNATURE[] = {'I', 'M', 'D','C'};
 const unsigned MODULE_DEPENDENCY_CACHE_FORMAT_VERSION_MAJOR = 9;
 /// Increment this on every change.
-const unsigned MODULE_DEPENDENCY_CACHE_FORMAT_VERSION_MINOR = 1;
+const unsigned MODULE_DEPENDENCY_CACHE_FORMAT_VERSION_MINOR = 2;
 
 /// Various identifiers in this format will rely on having their strings mapped
 /// using this ID.
@@ -80,7 +80,6 @@ using LinkLibrariesArrayIDField = IdentifierIDField;
 using MacroDependenciesArrayIDField = IdentifierIDField;
 using FlagIDArrayIDField = IdentifierIDField;
 using DependencyIDArrayIDField = IdentifierIDField;
-using AuxiliaryFilesArrayIDField = IdentifierIDField;
 using SourceLocationIDArrayIDField = IdentifierIDField;
 
 /// The ID of the top-level block containing the dependency graph
@@ -204,8 +203,7 @@ using ModuleInfoLayout =
                    DependencyIDArrayIDField,       // importedClangModules
                    DependencyIDArrayIDField,       // crossImportOverlayModules
                    DependencyIDArrayIDField,       // swiftOverlayDependencies
-                   ModuleCacheKeyIDField,          // moduleCacheKey
-                   AuxiliaryFilesArrayIDField      // auxiliaryFiles
+                   ModuleCacheKeyIDField           // moduleCacheKey
                    >;
 
 using SwiftInterfaceModuleDetailsLayout =

lib/AST/ASTContext.cpp

@@ -2214,46 +2214,6 @@ const {
   return SearchPathOpts.getDarwinImplicitFrameworkSearchPaths();
 }
 
-llvm::StringSet<> ASTContext::getAllModuleSearchPathsSet()
-const {
-  llvm::StringSet<> result;
-
-  // Import and framework paths are "special", they contain more than path
-  // strings, but path strings are all we care about here.
-  using SearchPathView = ArrayRefView<SearchPathOptions::SearchPath, StringRef,
-                                      pathStringFromSearchPath>;
-
-  SearchPathView importPathsOnly{SearchPathOpts.getImportSearchPaths()};
-  result.insert(importPathsOnly.begin(), importPathsOnly.end());
-
-  SearchPathView frameworkPathsOnly{SearchPathOpts.getFrameworkSearchPaths()};
-  result.insert(frameworkPathsOnly.begin(), frameworkPathsOnly.end());
-
-  if (LangOpts.Target.isOSDarwin()) {
-    auto implicitFrameworkSearchPaths = getDarwinImplicitFrameworkSearchPaths();
-    result.insert(implicitFrameworkSearchPaths.begin(),
-                  implicitFrameworkSearchPaths.end());
-  }
-  result.insert(SearchPathOpts.RuntimeLibraryImportPaths.begin(),
-                SearchPathOpts.RuntimeLibraryImportPaths.end());
-
-  // ClangImporter special-cases the path for SwiftShims, so do the same here
-  // If there are no shims in the resource dir, add a search path in the SDK.
-  SmallString<128> shimsPath(SearchPathOpts.RuntimeResourcePath);
-  llvm::sys::path::append(shimsPath, "shims");
-  if (!llvm::sys::fs::exists(shimsPath)) {
-    shimsPath = SearchPathOpts.getSDKPath();
-    llvm::sys::path::append(shimsPath, "usr", "lib", "swift", "shims");
-  }
-  result.insert(shimsPath.str());
-
-  // Clang system modules are found in the SDK root
-  SmallString<128> clangSysRootPath(SearchPathOpts.getSDKPath());
-  llvm::sys::path::append(clangSysRootPath, "usr", "include");
-  result.insert(clangSysRootPath.str());
-  return result;
-}
-
 void ASTContext::loadExtensions(NominalTypeDecl *nominal,
                                 unsigned previousGeneration) {
   PrettyStackTraceDecl stackTrace("loading extensions for", nominal);

lib/DependencyScan/ModuleDependencyCacheSerialization.cpp

@@ -229,17 +229,12 @@ bool ModuleDependenciesCacheDeserializer::readGraph(
   std::vector<ScannerImportStatementInfo> importStatements;
   std::vector<ScannerImportStatementInfo> optionalImportStatements;
 
-  std::vector<std::string> auxiliaryFiles;
-
   auto addCommonDependencyInfo =
-      [&importedClangDependenciesIDs, &auxiliaryFiles,
-       &macroDependencies](ModuleDependencyInfo &moduleDep) {
+      [&importedClangDependenciesIDs, &macroDependencies]
+      (ModuleDependencyInfo &moduleDep) {
         // Add qualified dependencies of this module
         moduleDep.setImportedClangDependencies(importedClangDependenciesIDs);
 
-        // Add any auxiliary files
-        moduleDep.setAuxiliaryFiles(auxiliaryFiles);
-
         // Add macro dependencies
         for (const auto &md : macroDependencies)
           moduleDep.addMacroDependency(md.first, md.second.LibraryPath,
@@ -441,8 +436,7 @@ bool ModuleDependenciesCacheDeserializer::readGraph(
           importedSwiftDependenciesIDsArrayID,
           importedClangDependenciesIDsArrayID,
           crossImportOverlayDependenciesIDsArrayID,
-          swiftOverlayDependenciesIDsArrayID, moduleCacheKeyID,
-          AuxiliaryFilesArrayID;
+          swiftOverlayDependenciesIDsArrayID, moduleCacheKeyID;
 
       ModuleInfoLayout::readRecord(Scratch, moduleNameID, moduleImportsArrayID,
                                    optionalImportsArrayID, linkLibraryArrayID,
@@ -451,7 +445,7 @@ bool ModuleDependenciesCacheDeserializer::readGraph(
                                    importedClangDependenciesIDsArrayID,
                                    crossImportOverlayDependenciesIDsArrayID,
                                    swiftOverlayDependenciesIDsArrayID,
-                                   moduleCacheKeyID, AuxiliaryFilesArrayID);
+                                   moduleCacheKeyID);
       auto moduleName = getIdentifier(moduleNameID);
       if (!moduleName)
         llvm::report_fatal_error("Bad module name");
@@ -468,11 +462,6 @@ bool ModuleDependenciesCacheDeserializer::readGraph(
       if (optionalOptionalImportStatementInfos)
         optionalImportStatements = *optionalOptionalImportStatementInfos;
 
-      auto optionalAuxiliaryFiles = getStringArray(AuxiliaryFilesArrayID);
-      if (optionalAuxiliaryFiles)
-        for (const auto &af : *optionalAuxiliaryFiles)
-          auxiliaryFiles.push_back(af);
-
       auto optionalImportedSwiftDependenciesIDs =
           getModuleDependencyIDArray(importedSwiftDependenciesIDsArrayID);
       if (!optionalImportedSwiftDependenciesIDs)
@@ -1042,7 +1031,6 @@ enum ModuleIdentifierArrayKind : uint8_t {
   ImportedClangDependenciesIDs,
   CrossImportOverlayDependenciesIDs,
   SwiftOverlayDependenciesIDs,
-  AuxiliaryFileIDs,
   CompiledModuleCandidates,
   BuildCommandLine,
   SourceFiles,
@@ -1490,9 +1478,7 @@ void ModuleDependenciesCacheSerializer::writeModuleInfo(
           ModuleIdentifierArrayKind::CrossImportOverlayDependenciesIDs),
       getIdentifierArrayID(
           moduleID, ModuleIdentifierArrayKind::SwiftOverlayDependenciesIDs),
-      getIdentifier(dependencyInfo.getModuleCacheKey()),
-      getIdentifierArrayID(moduleID,
-                           ModuleIdentifierArrayKind::AuxiliaryFileIDs));
+      getIdentifier(dependencyInfo.getModuleCacheKey()));
 
   switch (dependencyInfo.getKind()) {
   case swift::ModuleDependencyKind::SwiftInterface: {
@@ -1779,9 +1765,6 @@ void ModuleDependenciesCacheSerializer::collectStringsAndArrays(
           moduleID, ModuleIdentifierArrayKind::SwiftOverlayDependenciesIDs,
           dependencyInfo->getSwiftOverlayDependencies());
 
-      addStringArray(moduleID, ModuleIdentifierArrayKind::AuxiliaryFileIDs,
-                     dependencyInfo->getAuxiliaryFiles());
-
       std::vector<std::string> clangHeaderDependencyNames;
       for (const auto &headerDepID :
            dependencyInfo->getHeaderClangDependencies())

test/ScanDependencies/Incremental/system_search_path_invalidate.swift

@@ -0,0 +1,17 @@
+// RUN: %empty-directory(%t)
+// RUN: %empty-directory(%t/module-cache)
+// RUN: %empty-directory(%t/Frameworks)
+
+// RUN: %target-swift-frontend -scan-dependencies -scanner-module-validation -module-load-mode prefer-interface -Rdependency-scan-cache -load-dependency-scan-cache -serialize-dependency-scan-cache -dependency-scan-cache-path %t/cache.moddepcache -module-cache-path %t/module-cache %s -o %t/deps.json -I %t/ExtraCModules -I %S/../Inputs/CHeaders -F %t/Frameworks &> %t/first_scan_output.txt
+// RUN: cat %t/first_scan_output.txt | %FileCheck %s
+
+// RUN: %target-swift-frontend -scan-dependencies -scanner-module-validation -module-load-mode prefer-interface -Rdependency-scan-cache -load-dependency-scan-cache -serialize-dependency-scan-cache -dependency-scan-cache-path %t/cache.moddepcache -module-cache-path %t/module-cache %s -o %t/deps.json -I %t/ExtraCModules -I %S/../Inputs/CHeaders -F %t/Frameworks &> %t/second_scan_output.txt
+// RUN: cat %t/second_scan_output.txt | %FileCheck %s -check-prefix=INCREMENTAL-CHECK
+
+// Change an '-F' to a '-Fsystem' and ensure that the serialized cache does not get reused
+// RUN: %target-swift-frontend -scan-dependencies -scanner-module-validation -module-load-mode prefer-interface -Rdependency-scan-cache -load-dependency-scan-cache -serialize-dependency-scan-cache -dependency-scan-cache-path %t/cache.moddepcache -module-cache-path %t/module-cache %s -o %t/deps.json -I %t/ExtraCModules -I %S/../Inputs/CHeaders -Fsystem %t/Frameworks &> %t/second_system_scan_output.txt
+// RUN: cat %t/second_system_scan_output.txt | %FileCheck %s
+
+// CHECK: remark: Incremental module scan: Failed to load module scanning dependency cache from:
+// INCREMENTAL-CHECK-NOT: remark: Incremental module scan: Failed to load module scanning dependency cache from:
+