[Dependency Scanning] Pull optional dependencies from the adjacent binary module for direct interface dependencies

artemcm · artemcm · commit 6fcd8be072d1 · 2023-04-17T14:47:46.000-07:00
For a `@Testable` import in program source, if a Swift interface dependency is discovered, and has an adjacent binary `.swiftmodule`, open up the module, and pull in its optional dependencies. If an optional dependency cannot be resolved on the filesystem, fail silently without raising a diagnostic.
diff --git a/include/swift/AST/ASTContext.h b/include/swift/AST/ASTContext.h
@@ -1033,6 +1033,7 @@ class ASTContext final {
       ModuleDependenciesCache &cache,
       InterfaceSubContextDelegate &delegate,
       bool optionalDependencyLookup = false,
+      bool isTestableImport = false,
       llvm::Optional<std::pair<std::string, swift::ModuleDependencyKind>> dependencyOf = None);
 
   /// Retrieve the module dependencies for the Clang module with the given name.
diff --git a/include/swift/AST/ModuleLoader.h b/include/swift/AST/ModuleLoader.h
@@ -245,7 +245,8 @@ class ModuleLoader {
   /// If a non-null \p versionInfo is provided, the module version will be
   /// parsed and populated.
   virtual bool canImportModule(ImportPath::Module named,
-                               ModuleVersionInfo *versionInfo) = 0;
+                               ModuleVersionInfo *versionInfo,
+                               bool isTestableImport = false) = 0;
 
   /// Import a module with the given module path.
   ///
@@ -328,7 +329,8 @@ class ModuleLoader {
   virtual Optional<const ModuleDependencyInfo*> getModuleDependencies(
       StringRef moduleName,
       ModuleDependenciesCache &cache,
-      InterfaceSubContextDelegate &delegate) = 0;
+      InterfaceSubContextDelegate &delegate,
+      bool isTestableImport = false) = 0;
 };
 
 } // namespace swift
diff --git a/include/swift/ClangImporter/ClangImporter.h b/include/swift/ClangImporter/ClangImporter.h
@@ -213,7 +213,8 @@ class ClangImporter final : public ClangModuleLoader {
   /// If a non-null \p versionInfo is provided, the module version will be
   /// parsed and populated.
   virtual bool canImportModule(ImportPath::Module named,
-                               ModuleVersionInfo *versionInfo) override;
+                               ModuleVersionInfo *versionInfo,
+                               bool isTestableImport = false) override;
 
   /// Import a module with the given module path.
   ///
@@ -427,7 +428,8 @@ class ClangImporter final : public ClangModuleLoader {
 
   Optional<const ModuleDependencyInfo*> getModuleDependencies(
       StringRef moduleName, ModuleDependenciesCache &cache,
-      InterfaceSubContextDelegate &delegate) override;
+      InterfaceSubContextDelegate &delegate,
+      bool isTestableImport = false) override;
 
   /// Add dependency information for the bridging header.
   ///
diff --git a/include/swift/Frontend/ModuleInterfaceLoader.h b/include/swift/Frontend/ModuleInterfaceLoader.h
@@ -142,8 +142,8 @@ class ExplicitSwiftModuleLoader: public SerializedModuleLoaderBase {
                   std::unique_ptr<llvm::MemoryBuffer> *moduleBuffer,
                   std::unique_ptr<llvm::MemoryBuffer> *moduleDocBuffer,
                   std::unique_ptr<llvm::MemoryBuffer> *moduleSourceInfoBuffer,
-                  bool skipBuildingInterface, bool &isFramework,
-                  bool &isSystemModule) override;
+                  bool skipBuildingInterface, bool isTestableDependencyLookup,
+                  bool &isFramework, bool &isSystemModule) override;
 
   std::error_code findModuleFilesInDirectory(
       ImportPath::Element ModuleID, const SerializedModuleBaseName &BaseName,
@@ -152,10 +152,12 @@ class ExplicitSwiftModuleLoader: public SerializedModuleLoaderBase {
       std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
       std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer,
       std::unique_ptr<llvm::MemoryBuffer> *ModuleSourceInfoBuffer,
-      bool skipBuildingInterface, bool IsFramework) override;
+      bool SkipBuildingInterface, bool IsFramework,
+      bool IsTestableDependencyLookup = false) override;
 
   bool canImportModule(ImportPath::Module named,
-                       ModuleVersionInfo *versionInfo) override;
+                       ModuleVersionInfo *versionInfo,
+                       bool isTestableDependencyLookup = false) override;
 
   bool isCached(StringRef DepPath) override { return false; };
 
@@ -477,7 +479,8 @@ class ModuleInterfaceLoader : public SerializedModuleLoaderBase {
       std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
       std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer,
       std::unique_ptr<llvm::MemoryBuffer> *ModuleSourceInfoBuffer,
-      bool skipBuildingInterface, bool IsFramework) override;
+      bool SkipBuildingInterface, bool IsFramework,
+      bool IsTestableDependencyLookup = false) override;
 
   bool isCached(StringRef DepPath) override;
 public:
diff --git a/include/swift/Sema/SourceLoader.h b/include/swift/Sema/SourceLoader.h
@@ -60,7 +60,8 @@ class SourceLoader : public ModuleLoader {
   /// If a non-null \p versionInfo is provided, the module version will be
   /// parsed and populated.
   virtual bool canImportModule(ImportPath::Module named,
-                               ModuleVersionInfo *versionInfo) override;
+                               ModuleVersionInfo *versionInfo,
+                               bool isTestableDependencyLookup = false) override;
 
   /// Import a module with the given module path.
   ///
@@ -98,7 +99,8 @@ class SourceLoader : public ModuleLoader {
 
   Optional<const ModuleDependencyInfo*>
   getModuleDependencies(StringRef moduleName, ModuleDependenciesCache &cache,
-                        InterfaceSubContextDelegate &delegate) override;
+                        InterfaceSubContextDelegate &delegate,
+                        bool isTestableImport) override;
 };
 }
 
diff --git a/include/swift/Serialization/ModuleDependencyScanner.h b/include/swift/Serialization/ModuleDependencyScanner.h
@@ -34,7 +34,8 @@ namespace swift {
 
       /// Scan the given interface file to determine dependencies.
       llvm::ErrorOr<ModuleDependencyInfo> scanInterfaceFile(
-          Twine moduleInterfacePath, bool isFramework);
+          Twine moduleInterfacePath, bool isFramework,
+          bool isTestableImport);
 
       InterfaceSubContextDelegate &astDelegate;
 
@@ -61,7 +62,8 @@ namespace swift {
           std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
           std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer,
           std::unique_ptr<llvm::MemoryBuffer> *ModuleSourceInfoBuffer,
-          bool skipBuildingInterface, bool IsFramework) override;
+          bool SkipBuildingInterface, bool IsFramework,
+          bool IsTestableDependencyLookup) override;
 
       virtual void collectVisibleTopLevelModuleNames(
           SmallVectorImpl<Identifier> &names) const override {
@@ -125,8 +127,8 @@ namespace swift {
                  std::unique_ptr<llvm::MemoryBuffer> *moduleBuffer,
                  std::unique_ptr<llvm::MemoryBuffer> *moduleDocBuffer,
                  std::unique_ptr<llvm::MemoryBuffer> *moduleSourceInfoBuffer,
-                 bool skipBuildingInterface, bool &isFramework,
-                 bool &isSystemModule) override;
+                 bool skipBuildingInterface, bool isTestableDependencyLookup,
+                 bool &isFramework, bool &isSystemModule) override;
 
       static bool classof(const ModuleDependencyScanner *MDS) {
         return MDS->getKind() == MDS_placeholder;
diff --git a/include/swift/Serialization/SerializedModuleLoader.h b/include/swift/Serialization/SerializedModuleLoader.h
@@ -86,7 +86,8 @@ class SerializedModuleLoaderBase : public ModuleLoader {
       std::unique_ptr<llvm::MemoryBuffer> *moduleBuffer,
       std::unique_ptr<llvm::MemoryBuffer> *moduleDocBuffer,
       std::unique_ptr<llvm::MemoryBuffer> *moduleSourceInfoBuffer,
-      bool skipBuildingInterface, bool &isFramework, bool &isSystemModule);
+      bool skipBuildingInterface, bool isTestableDependencyLookup,
+      bool &isFramework, bool &isSystemModule);
 
   /// Attempts to search the provided directory for a loadable serialized
   /// .swiftmodule with the provided `ModuleFilename`. Subclasses must
@@ -107,7 +108,8 @@ class SerializedModuleLoaderBase : public ModuleLoader {
       std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
       std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer,
       std::unique_ptr<llvm::MemoryBuffer> *ModuleSourceInfoBuffer,
-      bool skipBuildingInterface, bool IsFramework) = 0;
+      bool SkipBuildingInterface, bool IsFramework,
+      bool isTestableDependencyLookup = false) = 0;
 
   std::error_code
   openModuleFile(
@@ -191,7 +193,8 @@ class SerializedModuleLoaderBase : public ModuleLoader {
   /// If a non-null \p versionInfo is provided, the module version will be
   /// parsed and populated.
   virtual bool canImportModule(ImportPath::Module named,
-                               ModuleVersionInfo *versionInfo) override;
+                               ModuleVersionInfo *versionInfo,
+                               bool isTestableDependencyLookup = false) override;
 
   /// Import a module with the given module path.
   ///
@@ -226,7 +229,8 @@ class SerializedModuleLoaderBase : public ModuleLoader {
 
   virtual Optional<const ModuleDependencyInfo*> getModuleDependencies(
       StringRef moduleName, ModuleDependenciesCache &cache,
-      InterfaceSubContextDelegate &delegate) override;
+      InterfaceSubContextDelegate &delegate,
+      bool isTestableImport) override;
 };
 
 /// Imports serialized Swift modules into an ASTContext.
@@ -244,7 +248,8 @@ class ImplicitSerializedModuleLoader : public SerializedModuleLoaderBase {
       std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
       std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer,
       std::unique_ptr<llvm::MemoryBuffer> *ModuleSourceInfoBuffer,
-      bool skipBuildingInterface, bool IsFramework) override;
+      bool SkipBuildingInterface, bool IsFramework,
+      bool isTestableDependencyLookup = false) override;
 
   bool maybeDiagnoseTargetMismatch(
       SourceLoc sourceLocation,
@@ -298,7 +303,8 @@ class MemoryBufferSerializedModuleLoader : public SerializedModuleLoaderBase {
       std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
       std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer,
       std::unique_ptr<llvm::MemoryBuffer> *ModuleSourceInfoBuffer,
-      bool skipBuildingInterface, bool IsFramework) override;
+      bool SkipBuildingInterface, bool IsFramework,
+      bool IsTestableDependencyLookup = false) override;
 
   bool maybeDiagnoseTargetMismatch(
       SourceLoc sourceLocation,
@@ -310,7 +316,8 @@ class MemoryBufferSerializedModuleLoader : public SerializedModuleLoaderBase {
   virtual ~MemoryBufferSerializedModuleLoader();
 
   bool canImportModule(ImportPath::Module named,
-                       ModuleVersionInfo *versionInfo) override;
+                       ModuleVersionInfo *versionInfo,
+                       bool isTestableDependencyLookup = false) override;
 
   ModuleDecl *
   loadModule(SourceLoc importLoc,
diff --git a/lib/AST/ASTContext.cpp b/lib/AST/ASTContext.cpp
@@ -2024,6 +2024,7 @@ Optional<const ModuleDependencyInfo*> ASTContext::getModuleDependencies(
     StringRef moduleName, ModuleDependenciesCache &cache,
     InterfaceSubContextDelegate &delegate,
     bool optionalDependencyLookup,
+    bool isTestableImport,
     llvm::Optional<ModuleDependencyID> dependencyOf) {
   // Retrieve the dependencies for this module.
   // Check whether we've cached this result.
@@ -2045,7 +2046,8 @@ Optional<const ModuleDependencyInfo*> ASTContext::getModuleDependencies(
 
   for (auto &loader : getImpl().ModuleLoaders) {
     if (auto dependencies =
-        loader->getModuleDependencies(moduleName, cache, delegate))
+        loader->getModuleDependencies(moduleName, cache, delegate,
+                                      isTestableImport))
       return dependencies;
   }
 
diff --git a/lib/AST/ModuleDependencies.cpp b/lib/AST/ModuleDependencies.cpp
@@ -127,6 +127,8 @@ void ModuleDependencyInfo::addModuleImport(
     auto realPath = importDecl->getRealModulePath(scratch);
     addModuleImport(realPath, &alreadyAddedModules);
 
+    // Additionally, keep track of which dependencies of a Source
+    // module are `@Testable`.
     if (getKind() == swift::ModuleDependencyKind::SwiftSource &&
         importDecl->isTestable())
       addTestableImport(realPath);
@@ -140,9 +142,8 @@ void ModuleDependencyInfo::addModuleImport(
   case swift::ModuleDependencyKind::SwiftInterface: {
     // If the storage is for an interface file, the only source file we
     // should see is that interface file.
-    auto swiftInterfaceStorage =
-        cast<SwiftInterfaceModuleDependenciesStorage>(storage.get());
-    assert(fileName == swiftInterfaceStorage->swiftInterfaceFile);
+    assert(fileName ==
+           cast<SwiftInterfaceModuleDependenciesStorage>(storage.get())->swiftInterfaceFile);
     break;
   }
   case swift::ModuleDependencyKind::SwiftSource: {
diff --git a/lib/ClangImporter/ClangImporter.cpp b/lib/ClangImporter/ClangImporter.cpp
@@ -1855,7 +1855,8 @@ static std::string getScalaNodeText(llvm::yaml::Node *N) {
 }
 
 bool ClangImporter::canImportModule(ImportPath::Module modulePath,
-                                    ModuleVersionInfo *versionInfo) {
+                                    ModuleVersionInfo *versionInfo,
+                                    bool isTestableDependencyLookup) {
   // Look up the top-level module to see if it exists.
   auto &clangHeaderSearch = Impl.getClangPreprocessor().getHeaderSearchInfo();
   auto topModule = modulePath.front();
diff --git a/lib/ClangImporter/ClangModuleDependencyScanner.cpp b/lib/ClangImporter/ClangModuleDependencyScanner.cpp
@@ -202,7 +202,7 @@ void ClangImporter::recordModuleDependencies(
 
 Optional<const ModuleDependencyInfo*> ClangImporter::getModuleDependencies(
     StringRef moduleName, ModuleDependenciesCache &cache,
-    InterfaceSubContextDelegate &delegate) {
+    InterfaceSubContextDelegate &delegate, bool isTestableImport) {
   auto &ctx = Impl.SwiftContext;
   // Determine the command-line arguments for dependency scanning.
   std::vector<std::string> commandLineArgs =
diff --git a/lib/DependencyScan/ScanDependencies.cpp b/lib/DependencyScan/ScanDependencies.cpp
@@ -330,6 +330,8 @@ resolveDirectDependencies(CompilerInstance &instance, ModuleDependencyID moduleI
   for (auto dependsOn : knownDependencies->getModuleImports()) {
     // Figure out what kind of module we need.
     bool onlyClangModule = !isSwift || moduleID.first == dependsOn;
+    bool isTestable = knownDependencies->isTestableImport(dependsOn);
+
     if (onlyClangModule) {
       if (auto found =
               ctx.getClangModuleDependencies(dependsOn, cache, ASTDelegate))
@@ -338,6 +340,7 @@ resolveDirectDependencies(CompilerInstance &instance, ModuleDependencyID moduleI
       if (auto found =
               ctx.getModuleDependencies(dependsOn, cache, ASTDelegate,
                                         /* optionalDependencyLookup */ false,
+                                        isTestable,
                                         moduleID))
         result.insert({dependsOn, found.value()->getKind()});
     }
@@ -350,6 +353,7 @@ resolveDirectDependencies(CompilerInstance &instance, ModuleDependencyID moduleI
     if (auto found =
             ctx.getModuleDependencies(optionallyDependsOn, cache, ASTDelegate,
                                       /* optionalDependencyLookup */ true,
+                                      /* isTestableDependency */ false,
                                       moduleID))
       result.insert({optionallyDependsOn, found.value()->getKind()});
   }
diff --git a/lib/Frontend/ModuleInterfaceLoader.cpp b/lib/Frontend/ModuleInterfaceLoader.cpp
@@ -1143,7 +1143,8 @@ std::error_code ModuleInterfaceLoader::findModuleFilesInDirectory(
     std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
     std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer,
     std::unique_ptr<llvm::MemoryBuffer> *ModuleSourceInfoBuffer,
-    bool skipBuildingInterface, bool IsFramework) {
+    bool skipBuildingInterface, bool IsFramework,
+    bool isTestableImport) {
 
   // If running in OnlySerialized mode, ModuleInterfaceLoader
   // should not have been constructed at all.
@@ -1991,7 +1992,8 @@ bool ExplicitSwiftModuleLoader::findModule(
     std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
     std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer,
     std::unique_ptr<llvm::MemoryBuffer> *ModuleSourceInfoBuffer,
-    bool skipBuildingInterface, bool &IsFramework, bool &IsSystemModule) {
+    bool skipBuildingInterface, bool isTestableDependencyLookup,
+    bool &IsFramework, bool &IsSystemModule) {
   // Find a module with an actual, physical name on disk, in case
   // -module-alias is used (otherwise same).
   //
@@ -2069,13 +2071,15 @@ std::error_code ExplicitSwiftModuleLoader::findModuleFilesInDirectory(
     std::unique_ptr<llvm::MemoryBuffer> *ModuleBuffer,
     std::unique_ptr<llvm::MemoryBuffer> *ModuleDocBuffer,
     std::unique_ptr<llvm::MemoryBuffer> *ModuleSourceInfoBuffer,
-    bool skipBuildingInterface, bool IsFramework) {
+    bool skipBuildingInterface, bool IsFramework,
+    bool IsTestableDependencyLookup) {
   llvm_unreachable("Not supported in the Explicit Swift Module Loader.");
   return std::make_error_code(std::errc::not_supported);
 }
 
 bool ExplicitSwiftModuleLoader::canImportModule(
-    ImportPath::Module path, ModuleVersionInfo *versionInfo) {
+    ImportPath::Module path, ModuleVersionInfo *versionInfo,
+    bool isTestableDependencyLookup) {
   // FIXME: Swift submodules?
   if (path.hasSubmodule())
     return false;
diff --git a/lib/Sema/SourceLoader.cpp b/lib/Sema/SourceLoader.cpp
@@ -71,7 +71,8 @@ void SourceLoader::collectVisibleTopLevelModuleNames(
 }
 
 bool SourceLoader::canImportModule(ImportPath::Module path,
-                                   ModuleVersionInfo *versionInfo) {
+                                   ModuleVersionInfo *versionInfo,
+                                   bool isTestableDependencyLookup) {
   // FIXME: Swift submodules?
   if (path.hasSubmodule())
     return false;
@@ -155,7 +156,8 @@ void SourceLoader::loadExtensions(NominalTypeDecl *nominal,
 Optional<const ModuleDependencyInfo*>
 SourceLoader::getModuleDependencies(StringRef moduleName,
                                     ModuleDependenciesCache &cache,
-                                    InterfaceSubContextDelegate &delegate) {
+                                    InterfaceSubContextDelegate &delegate,
+                                    bool isTestableImport) {
   // FIXME: Implement?
   return None;
 }
diff --git a/lib/Serialization/ModuleDependencyScanner.cpp b/lib/Serialization/ModuleDependencyScanner.cpp
diff --git a/lib/Serialization/SerializedModuleLoader.cpp b/lib/Serialization/SerializedModuleLoader.cpp
diff --git a/test/ScanDependencies/optional_deps_of_testable_imports.swift b/test/ScanDependencies/optional_deps_of_testable_imports.swift

Original file line number	Diff line number	Diff line change
`@@ -1855,7 +1855,8 @@ static std::string getScalaNodeText(llvm::yaml::Node *N) {`
`1855`	`1855`	`}`
`1856`	`1856`
`1857`	`1857`	`bool ClangImporter::canImportModule(ImportPath::Module modulePath,`
`1858`		`- ModuleVersionInfo *versionInfo) {`
	`1858`	`+ ModuleVersionInfo *versionInfo,`
	`1859`	`+ bool isTestableDependencyLookup) {`
`1859`	`1860`	`// Look up the top-level module to see if it exists.`
`1860`	`1861`	`auto &clangHeaderSearch = Impl.getClangPreprocessor().getHeaderSearchInfo();`
`1861`	`1862`	`auto topModule = modulePath.front();`