[Dependency Scanning] Record header dependencies of Binary Swift module dependencies

These are meant to capture paths to the PCH files that a given module was built with.

Author: artemcm

include/swift/AST/ModuleDependencies.h

@@ -98,9 +98,14 @@ class ModuleDependencyInfoStorageBase {
 public:
   const ModuleDependencyKind dependencyKind;
 
+  ModuleDependencyInfoStorageBase(ModuleDependencyKind dependencyKind)
+      : dependencyKind(dependencyKind),
+        resolved(false) { }
+
   ModuleDependencyInfoStorageBase(ModuleDependencyKind dependencyKind,
-                                  bool resolved = false)
-      : dependencyKind(dependencyKind), resolved(resolved) { }
+                                  const std::vector<std::string> &moduleImports)
+      : dependencyKind(dependencyKind), moduleImports(moduleImports),
+        resolved(false)  {}
 
   virtual ModuleDependencyInfoStorageBase *clone() const = 0;
 
@@ -245,11 +250,13 @@ class SwiftBinaryModuleDependencyStorage : public ModuleDependencyInfoStorageBas
   SwiftBinaryModuleDependencyStorage(const std::string &compiledModulePath,
                                      const std::string &moduleDocPath,
                                      const std::string &sourceInfoPath,
+                                     const std::vector<std::string> &moduleImports,
+                                     const std::vector<std::string> &headerImports,
                                      const bool isFramework)
-      : ModuleDependencyInfoStorageBase(ModuleDependencyKind::SwiftBinary),
-        compiledModulePath(compiledModulePath),
-        moduleDocPath(moduleDocPath),
-        sourceInfoPath(sourceInfoPath),
+      : ModuleDependencyInfoStorageBase(ModuleDependencyKind::SwiftBinary,
+                                        moduleImports),
+        compiledModulePath(compiledModulePath), moduleDocPath(moduleDocPath),
+        sourceInfoPath(sourceInfoPath), preCompiledBridgingHeaderPaths(headerImports),
         isFramework(isFramework) {}
 
   ModuleDependencyInfoStorageBase *clone() const override {
@@ -265,6 +272,9 @@ class SwiftBinaryModuleDependencyStorage : public ModuleDependencyInfoStorageBas
   /// The path to the .swiftSourceInfo file.
   const std::string sourceInfoPath;
 
+  /// The paths of all the .pch dependencies of this module.
+  const std::vector<std::string> preCompiledBridgingHeaderPaths;
+
   /// A flag that indicates this dependency is a framework
   const bool isFramework;
 
@@ -400,10 +410,13 @@ class ModuleDependencyInfo {
       const std::string &compiledModulePath,
       const std::string &moduleDocPath,
       const std::string &sourceInfoPath,
+      const std::vector<std::string> &moduleImports,
+      const std::vector<std::string> &headerImports,
       bool isFramework) {
     return ModuleDependencyInfo(
         std::make_unique<SwiftBinaryModuleDependencyStorage>(
-          compiledModulePath, moduleDocPath, sourceInfoPath, isFramework));
+          compiledModulePath, moduleDocPath, sourceInfoPath,
+          moduleImports, headerImports, isFramework));
   }
 
   /// Describe the main Swift module.

include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h

@@ -155,6 +155,7 @@ using SwiftBinaryModuleDetailsLayout =
                    FileIDField,                      // compiledModulePath
                    FileIDField,                      // moduleDocPath
                    FileIDField,                      // moduleSourceInfoPath
+                   ImportArrayIDField,               // headerImports
                    IsFrameworkField                  // isFramework
                    >;
 

include/swift/Serialization/SerializedModuleLoader.h

@@ -150,15 +150,20 @@ class SerializedModuleLoaderBase : public ModuleLoader {
   /// Scan the given serialized module file to determine dependencies.
   llvm::ErrorOr<ModuleDependencyInfo> scanModuleFile(Twine modulePath, bool isFramework);
 
-  static llvm::ErrorOr<llvm::StringSet<>>
-  getModuleImportsOfModule(Twine modulePath,
-                           ModuleLoadingBehavior transitiveBehavior,
-                           bool isFramework,
-                           bool isRequiredOSSAModules,
-                           StringRef SDKName,
-                           StringRef packageName,
-                           llvm::vfs::FileSystem *fileSystem,
-                           PathObfuscator &recoverer);
+  struct BinaryModuleImports {
+    llvm::StringSet<> moduleImports;
+    llvm::StringSet<> headerImports;
+  };
+
+  static llvm::ErrorOr<BinaryModuleImports>
+  getImportsOfModule(Twine modulePath,
+                     ModuleLoadingBehavior transitiveBehavior,
+                     bool isFramework,
+                     bool isRequiredOSSAModules,
+                     StringRef SDKName,
+                     StringRef packageName,
+                     llvm::vfs::FileSystem *fileSystem,
+                     PathObfuscator &recoverer);
 
   /// Load the module file into a buffer and also collect its module name.
   static std::unique_ptr<llvm::MemoryBuffer>

lib/DependencyScan/ModuleDependencyCacheSerialization.cpp

@@ -413,10 +413,10 @@ bool ModuleDependenciesCacheDeserializer::readGraph(SwiftDependencyScanningServi
             "Unexpected SWIFT_BINARY_MODULE_DETAILS_NODE record");
       cache.configureForContextHash(getContextHash());
       unsigned compiledModulePathID, moduleDocPathID, moduleSourceInfoPathID,
-          isFramework;
+               headerImportsArrayID, isFramework;
       SwiftBinaryModuleDetailsLayout::readRecord(
           Scratch, compiledModulePathID, moduleDocPathID,
-          moduleSourceInfoPathID, isFramework);
+          moduleSourceInfoPathID, headerImportsArrayID, isFramework);
 
       auto compiledModulePath = getIdentifier(compiledModulePathID);
       if (!compiledModulePath)
@@ -428,13 +428,14 @@ bool ModuleDependenciesCacheDeserializer::readGraph(SwiftDependencyScanningServi
       if (!moduleSourceInfoPath)
         llvm::report_fatal_error("Bad module source info path");
 
+      auto headerImports = getStringArray(headerImportsArrayID);
+      if (!headerImports)
+        llvm::report_fatal_error("Bad binary direct dependencies: no header imports");
+
       // Form the dependencies storage object
       auto moduleDep = ModuleDependencyInfo::forSwiftBinaryModule(
-          *compiledModulePath, *moduleDocPath, *moduleSourceInfoPath,
-          isFramework);
-      // Add dependencies of this module
-      for (const auto &moduleName : *currentModuleImports)
-        moduleDep.addModuleImport(moduleName);
+           *compiledModulePath, *moduleDocPath, *moduleSourceInfoPath,
+           *currentModuleImports, *headerImports, isFramework);
 
       cache.recordDependency(currentModuleName, std::move(moduleDep),
                              getContextHash());
@@ -638,6 +639,7 @@ bool swift::dependencies::module_dependency_cache_serialization::
 enum ModuleIdentifierArrayKind : uint8_t {
   Empty = 0,
   DependencyImports,
+  DependencyHeaders,
   QualifiedModuleDependencyIDs,
   CompiledModuleCandidates,
   BuildCommandLine,
@@ -894,6 +896,7 @@ void ModuleDependenciesCacheSerializer::writeModuleInfo(ModuleDependencyID modul
         getIdentifier(swiftBinDeps->compiledModulePath),
         getIdentifier(swiftBinDeps->moduleDocPath),
         getIdentifier(swiftBinDeps->sourceInfoPath),
+        getArrayID(moduleID, ModuleIdentifierArrayKind::DependencyHeaders),
         swiftBinDeps->isFramework);
 
     break;
@@ -1062,6 +1065,8 @@ void ModuleDependenciesCacheSerializer::collectStringsAndArrays(
         addIdentifier(swiftBinDeps->compiledModulePath);
         addIdentifier(swiftBinDeps->moduleDocPath);
         addIdentifier(swiftBinDeps->sourceInfoPath);
+        addStringArray(moduleID, ModuleIdentifierArrayKind::DependencyHeaders,
+                       swiftBinDeps->preCompiledBridgingHeaderPaths);
         break;
       }
       case swift::ModuleDependencyKind::SwiftPlaceholder: {

lib/Serialization/ModuleDependencyScanner.cpp

@@ -194,38 +194,41 @@ ErrorOr<ModuleDependencyInfo> ModuleDependencyScanner::scanInterfaceFile(
           });
       if (adjacentBinaryModule != compiledCandidates.end()) {
         // Required modules.
-        auto adjacentBinaryModuleRequiredImports = getModuleImportsOfModule(
+        auto adjacentBinaryModuleRequiredImports = getImportsOfModule(
             *adjacentBinaryModule, ModuleLoadingBehavior::Required, isFramework,
             isRequiredOSSAModules(), Ctx.LangOpts.SDKName,
             Ctx.LangOpts.PackageName, Ctx.SourceMgr.getFileSystem().get(),
             Ctx.SearchPathOpts.DeserializedPathRecoverer);
         if (!adjacentBinaryModuleRequiredImports)
           return adjacentBinaryModuleRequiredImports.getError();
-
+        auto adjacentBinaryModuleRequiredModuleImports =
+          (*adjacentBinaryModuleRequiredImports).moduleImports;
 #ifndef NDEBUG
         //  Verify that the set of required modules read out from the binary
         //  module is a super-set of module imports identified in the
         //  textual interface.
         for (const auto &requiredImport : Result->getModuleImports()) {
-          assert(adjacentBinaryModuleRequiredImports->contains(requiredImport) &&
+          assert(adjacentBinaryModuleRequiredModuleImports.contains(requiredImport) &&
                  "Expected adjacent binary module's import set to contain all "
                  "textual interface imports.");
         }
 #endif
 
-        for (const auto &requiredImport : *adjacentBinaryModuleRequiredImports)
+        for (const auto &requiredImport : adjacentBinaryModuleRequiredModuleImports)
           Result->addModuleImport(requiredImport.getKey(),
                                   &alreadyAddedModules);
 
         // Optional modules. Will be looked-up on a best-effort basis
-        auto adjacentBinaryModuleOptionalImports = getModuleImportsOfModule(
+        auto adjacentBinaryModuleOptionalImports = getImportsOfModule(
             *adjacentBinaryModule, ModuleLoadingBehavior::Optional, isFramework,
             isRequiredOSSAModules(), Ctx.LangOpts.SDKName,
             Ctx.LangOpts.PackageName, Ctx.SourceMgr.getFileSystem().get(),
             Ctx.SearchPathOpts.DeserializedPathRecoverer);
         if (!adjacentBinaryModuleOptionalImports)
           return adjacentBinaryModuleOptionalImports.getError();
-        for (const auto &optionalImport : *adjacentBinaryModuleOptionalImports)
+        auto adjacentBinaryModuleOptionalModuleImports =
+          (*adjacentBinaryModuleOptionalImports).moduleImports;
+        for (const auto &optionalImport : adjacentBinaryModuleOptionalModuleImports)
           Result->addOptionalModuleImport(optionalImport.getKey(),
                                           &alreadyAddedModules);
       }

lib/Serialization/SerializedModuleLoader.cpp

@@ -391,8 +391,8 @@ std::error_code SerializedModuleLoaderBase::openModuleFile(
   return std::error_code();
 }
 
-llvm::ErrorOr<llvm::StringSet<>>
-SerializedModuleLoaderBase::getModuleImportsOfModule(
+llvm::ErrorOr<SerializedModuleLoaderBase::BinaryModuleImports>
+SerializedModuleLoaderBase::getImportsOfModule(
     Twine modulePath, ModuleLoadingBehavior transitiveBehavior,
     bool isFramework, bool isRequiredOSSAModules, StringRef SDKName,
     StringRef packageName, llvm::vfs::FileSystem *fileSystem,
@@ -402,16 +402,18 @@ SerializedModuleLoaderBase::getModuleImportsOfModule(
     return moduleBuf.getError();
 
   llvm::StringSet<> importedModuleNames;
+  llvm::StringSet<> importedHeaders;
   // Load the module file without validation.
   std::shared_ptr<const ModuleFileSharedCore> loadedModuleFile;
   serialization::ValidationInfo loadInfo = ModuleFileSharedCore::load(
       "", "", std::move(moduleBuf.get()), nullptr, nullptr, isFramework,
       isRequiredOSSAModules, SDKName, recoverer, loadedModuleFile);
 
   for (const auto &dependency : loadedModuleFile->getDependencies()) {
-    // FIXME: Record header dependency?
-    if (dependency.isHeader())
+    if (dependency.isHeader()) {
+      importedHeaders.insert(dependency.RawPath);
       continue;
+    }
 
     ModuleLoadingBehavior dependencyTransitiveBehavior =
         loadedModuleFile->getTransitiveLoadingBehavior(
@@ -432,16 +434,13 @@ SerializedModuleLoaderBase::getModuleImportsOfModule(
     importedModuleNames.insert(moduleName);
   }
 
-  return importedModuleNames;
+  return SerializedModuleLoaderBase::BinaryModuleImports{importedModuleNames, importedHeaders};
 }
 
 llvm::ErrorOr<ModuleDependencyInfo>
 SerializedModuleLoaderBase::scanModuleFile(Twine modulePath, bool isFramework) {
   const std::string moduleDocPath;
   const std::string sourceInfoPath;
-  // Map the set of dependencies over to the "module dependencies".
-  auto dependencies = ModuleDependencyInfo::forSwiftBinaryModule(
-      modulePath.str(), moduleDocPath, sourceInfoPath, isFramework);
   // Some transitive dependencies of binary modules are not required to be
   // imported during normal builds.
   // TODO: This is worth revisiting for debugger purposes where
@@ -450,18 +449,37 @@ SerializedModuleLoaderBase::scanModuleFile(Twine modulePath, bool isFramework) {
   //       optional.
   ModuleLoadingBehavior transitiveLoadingBehavior =
       ModuleLoadingBehavior::Required;
-  auto importedModuleNames = getModuleImportsOfModule(
+  auto binaryModuleImports = getImportsOfModule(
       modulePath, transitiveLoadingBehavior, isFramework,
       isRequiredOSSAModules(), Ctx.LangOpts.SDKName, Ctx.LangOpts.PackageName,
       Ctx.SourceMgr.getFileSystem().get(),
       Ctx.SearchPathOpts.DeserializedPathRecoverer);
-  if (!importedModuleNames)
-    return importedModuleNames.getError();
+  if (!binaryModuleImports)
+    return binaryModuleImports.getError();
+
+  auto importedModuleSet = binaryModuleImports.get().moduleImports;
+  std::vector<std::string> importedModuleNames;
+  importedModuleNames.reserve(importedModuleSet.size());
+  llvm::transform(importedModuleSet.keys(),
+                  std::back_inserter(importedModuleNames),
+                  [](llvm::StringRef N) {
+                     return N.str();
+                  });
+
+  auto importedHeaderSet = binaryModuleImports.get().headerImports;
+  std::vector<std::string> importedHeaders;
+  importedHeaders.reserve(importedHeaderSet.size());
+  llvm::transform(importedHeaderSet.keys(),
+                  std::back_inserter(importedHeaders),
+                  [](llvm::StringRef N) {
+                     return N.str();
+                  });
 
-  llvm::StringSet<> addedModuleNames;
-  for (const auto &importedModuleName : *importedModuleNames)
-    dependencies.addModuleImport(importedModuleName.getKey(),
-                                 &addedModuleNames);
+  // Map the set of dependencies over to the "module dependencies".
+  auto dependencies = ModuleDependencyInfo::forSwiftBinaryModule(
+       modulePath.str(), moduleDocPath, sourceInfoPath,
+       importedModuleNames, importedHeaders, isFramework,
+       /*module-cache-key*/ "");
 
   return std::move(dependencies);
 }