Merge pull request swiftlang#33032 from artemcm/ExplicitPackageBuilds

[Dependency Scanner] Add a scanner for explicit placeholder module dependencies

Author: artemcm

include/swift/AST/DiagnosticsDriver.def

@@ -2,7 +2,7 @@
 //
 // This source file is part of the Swift.org open source project
 //
-// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
+// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
 // Licensed under Apache License v2.0 with Runtime Library Exception
 //
 // See https://swift.org/LICENSE.txt for license information
@@ -122,6 +122,9 @@ ERROR(error_conflicting_options, none,
 ERROR(error_option_not_supported, none,
       "'%0' is not supported with '%1'",
       (StringRef, StringRef))
+ERROR(error_requirement_not_met, none,
+      "'%0' requires '%1'",
+      (StringRef, StringRef))
 
 WARNING(warn_ignore_embed_bitcode, none,
         "ignoring -embed-bitcode since no object file is being generated", ())

include/swift/AST/DiagnosticsFrontend.def

@@ -266,6 +266,14 @@ ERROR(explicit_swift_module_map_corrupted,none,
       "explicit Swift module map from %0 is malformed",
       (StringRef))
 
+ERROR(placeholder_dependency_module_map_missing,none,
+      "cannot open Swift placeholder dependency module map from %0",
+      (StringRef))
+
+ERROR(placeholder_dependency_module_map_corrupted,none,
+      "Swift placeholder dependency module map from %0 is malformed",
+      (StringRef))
+
 REMARK(default_previous_install_name, none,
       "default previous install name for %0 is %1", (StringRef, StringRef))
 

include/swift/AST/ModuleDependencies.h

@@ -35,6 +35,29 @@ class Identifier;
 /// Which kind of module dependencies we are looking for.
 enum class ModuleDependenciesKind : int8_t {
   Swift,
+  // Placeholder dependencies are a kind of dependencies used only by the
+  // dependency scanner. They are swift modules that the scanner will not be
+  // able to locate in its search paths and which are the responsibility of the
+  // scanner's client to ensure are provided.
+  //
+  // Placeholder dependencies will be specified in the scanner's output
+  // dependency graph where it is the responsibility of the scanner's client to
+  // ensure required post-processing takes place to "resolve" them. In order to
+  // do so, the client (swift driver, or any other client build system) is
+  // expected to have access to a full dependency graph of all placeholder
+  // dependencies and be able to replace placeholder nodes in the dependency
+  // graph with their full dependency trees, `uniquing` common dependency module
+  // nodes in the process.
+  //
+  // One example where placeholder dependencies are employed is when using
+  // SwiftPM in Explicit Module Build mode. SwiftPM constructs a build plan for
+  // all targets ahead-of-time. When planning a build for a target that depends
+  // on other targets, the dependency scanning action is not able to locate
+  // dependency target modules, because they have not yet been built. Instead,
+  // the build system treats them as placeholder dependencies and resolves them
+  // with `actual` dependencies in a post-processing step once dependency graphs
+  // of all targets, individually, have been computed.
+  SwiftPlaceholder,
   Clang,
 };
 
@@ -43,11 +66,11 @@ enum class ModuleDependenciesKind : int8_t {
 /// This class is mostly an implementation detail for \c ModuleDependencies.
 class ModuleDependenciesStorageBase {
 public:
-  const bool isSwiftModule;
+  const ModuleDependenciesKind dependencyKind;
 
-  ModuleDependenciesStorageBase(bool isSwiftModule,
+  ModuleDependenciesStorageBase(ModuleDependenciesKind dependencyKind,
                                 const std::string &compiledModulePath)
-      : isSwiftModule(isSwiftModule),
+      : dependencyKind(dependencyKind),
         compiledModulePath(compiledModulePath) { }
 
   virtual ModuleDependenciesStorageBase *clone() const = 0;
@@ -103,7 +126,8 @@ class SwiftModuleDependenciesStorage : public ModuleDependenciesStorageBase {
       ArrayRef<StringRef> buildCommandLine,
       ArrayRef<StringRef> extraPCMArgs,
       StringRef contextHash
-  ) : ModuleDependenciesStorageBase(/*isSwiftModule=*/true, compiledModulePath),
+  ) : ModuleDependenciesStorageBase(ModuleDependenciesKind::Swift,
+                                    compiledModulePath),
       swiftInterfaceFile(swiftInterfaceFile),
       compiledModuleCandidates(compiledModuleCandidates.begin(),
                                compiledModuleCandidates.end()),
@@ -116,7 +140,7 @@ class SwiftModuleDependenciesStorage : public ModuleDependenciesStorageBase {
   }
 
   static bool classof(const ModuleDependenciesStorageBase *base) {
-    return base->isSwiftModule;
+    return base->dependencyKind == ModuleDependenciesKind::Swift;
   }
 };
 
@@ -143,7 +167,7 @@ class ClangModuleDependenciesStorage : public ModuleDependenciesStorageBase {
       const std::string &contextHash,
       const std::vector<std::string> &nonPathCommandLine,
       const std::vector<std::string> &fileDependencies
-  ) : ModuleDependenciesStorageBase(/*isSwiftModule=*/false,
+  ) : ModuleDependenciesStorageBase(ModuleDependenciesKind::Clang,
                                     compiledModulePath),
       moduleMapFile(moduleMapFile),
       contextHash(contextHash),
@@ -155,7 +179,35 @@ class ClangModuleDependenciesStorage : public ModuleDependenciesStorageBase {
   }
 
   static bool classof(const ModuleDependenciesStorageBase *base) {
-    return !base->isSwiftModule;
+    return base->dependencyKind == ModuleDependenciesKind::Clang;
+  }
+};
+
+/// Describes an placeholder Swift module dependency module stub.
+///
+/// This class is mostly an implementation detail for \c ModuleDependencies.
+class PlaceholderSwiftModuleDependencyStorage : public ModuleDependenciesStorageBase {
+public:
+  PlaceholderSwiftModuleDependencyStorage(const std::string &compiledModulePath,
+                                       const std::string &moduleDocPath,
+                                       const std::string &sourceInfoPath)
+      : ModuleDependenciesStorageBase(ModuleDependenciesKind::SwiftPlaceholder,
+                                      compiledModulePath),
+        moduleDocPath(moduleDocPath),
+        sourceInfoPath(sourceInfoPath) {}
+
+  ModuleDependenciesStorageBase *clone() const override {
+    return new PlaceholderSwiftModuleDependencyStorage(*this);
+  }
+
+  /// The path to the .swiftModuleDoc file.
+  const std::string moduleDocPath;
+
+  /// The path to the .swiftSourceInfo file.
+  const std::string sourceInfoPath;
+
+  static bool classof(const ModuleDependenciesStorageBase *base) {
+    return base->dependencyKind == ModuleDependenciesKind::SwiftPlaceholder;
   }
 };
 
@@ -230,6 +282,16 @@ class ModuleDependencies {
           fileDependencies));
   }
 
+  /// Describe a placeholder dependency swift module.
+  static ModuleDependencies forPlaceholderSwiftModuleStub(
+      const std::string &compiledModulePath,
+      const std::string &moduleDocPath,
+      const std::string &sourceInfoPath) {
+    return ModuleDependencies(
+        std::make_unique<PlaceholderSwiftModuleDependencyStorage>(
+          compiledModulePath, moduleDocPath, sourceInfoPath));
+  }
+
   /// Retrieve the path to the compiled module.
   const std::string getCompiledModulePath() const {
     return storage->compiledModulePath;
@@ -243,16 +305,22 @@ class ModuleDependencies {
   /// Whether the dependencies are for a Swift module.
   bool isSwiftModule() const;
 
+  /// Whether this represents a placeholder module stub
+  bool isPlaceholderSwiftModule() const;
+
   ModuleDependenciesKind getKind() const {
-    return isSwiftModule() ? ModuleDependenciesKind::Swift
-                           : ModuleDependenciesKind::Clang;
+    return storage->dependencyKind;
   }
   /// Retrieve the dependencies for a Swift module.
   const SwiftModuleDependenciesStorage *getAsSwiftModule() const;
 
   /// Retrieve the dependencies for a Clang module.
   const ClangModuleDependenciesStorage *getAsClangModule() const;
 
+  /// Retrieve the dependencies for a placeholder dependency module stub.
+  const PlaceholderSwiftModuleDependencyStorage *
+  getAsPlaceholderDependencyModule() const;
+
   /// Add a dependency on the given module, if it was not already in the set.
   void addModuleDependency(StringRef module,
                            llvm::StringSet<> *alreadyAddedModules = nullptr);
@@ -293,6 +361,9 @@ class ModuleDependenciesCache {
   /// Dependencies for Swift modules that have already been computed.
   llvm::StringMap<ModuleDependencies> SwiftModuleDependencies;
 
+  /// Dependencies for Swift placeholder dependency modules.
+  llvm::StringMap<ModuleDependencies> PlaceholderSwiftModuleDependencies;
+
   /// Dependencies for Clang modules that have already been computed.
   llvm::StringMap<ModuleDependencies> ClangModuleDependencies;
 

include/swift/AST/SearchPathOptions.h

@@ -90,6 +90,9 @@ class SearchPathOptions {
 
   /// A map of explict Swift module information.
   std::string ExplicitSwiftModuleMap;
+
+  /// A map of placeholder Swift module dependency information.
+  std::string PlaceholderDependencyModuleMap;
 private:
   static StringRef
   pathStringFromFrameworkSearchPath(const FrameworkSearchPath &next) {

include/swift/Frontend/ModuleInterfaceLoader.h

@@ -162,6 +162,106 @@ class ExplicitSwiftModuleLoader: public SerializedModuleLoaderBase {
   ~ExplicitSwiftModuleLoader();
 };
 
+/// Information about explicitly specified Swift module files.
+struct ExplicitModuleInfo {
+  // Path of the .swiftmodule file.
+  StringRef modulePath;
+  // Path of the .swiftmoduledoc file.
+  StringRef moduleDocPath;
+  // Path of the .swiftsourceinfo file.
+  StringRef moduleSourceInfoPath;
+  // Opened buffer for the .swiftmodule file.
+  std::unique_ptr<llvm::MemoryBuffer> moduleBuffer;
+};
+
+/// Parser of explicit module maps passed into the compiler.
+//  [
+//    {
+//      "moduleName": "A",
+//      "modulePath": "A.swiftmodule",
+//      "docPath": "A.swiftdoc",
+//      "sourceInfoPath": "A.swiftsourceinfo"
+//    },
+//    {
+//      "moduleName": "B",
+//      "modulePath": "B.swiftmodule",
+//      "docPath": "B.swiftdoc",
+//      "sourceInfoPath": "B.swiftsourceinfo"
+//    }
+//  ]
+class ExplicitModuleMapParser {
+public:
+  ExplicitModuleMapParser(llvm::BumpPtrAllocator &Allocator) : Saver(Allocator) {}
+
+  std::error_code
+  parseSwiftExplicitModuleMap(llvm::StringRef fileName,
+                              llvm::StringMap<ExplicitModuleInfo> &moduleMap) {
+    using namespace llvm::yaml;
+    // Load the input file.
+    llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> fileBufOrErr =
+        llvm::MemoryBuffer::getFile(fileName);
+    if (!fileBufOrErr) {
+      return std::make_error_code(std::errc::no_such_file_or_directory);
+    }
+    StringRef Buffer = fileBufOrErr->get()->getBuffer();
+    // Use a new source manager instead of the one from ASTContext because we
+    // don't want the JSON file to be persistent.
+    llvm::SourceMgr SM;
+    Stream Stream(llvm::MemoryBufferRef(Buffer, fileName), SM);
+    for (auto DI = Stream.begin(); DI != Stream.end(); ++DI) {
+      assert(DI != Stream.end() && "Failed to read a document");
+      if (auto *MN = dyn_cast_or_null<SequenceNode>(DI->getRoot())) {
+        for (auto &entry : *MN) {
+          if (parseSingleModuleEntry(entry, moduleMap)) {
+            return std::make_error_code(std::errc::invalid_argument);
+          }
+        }
+      } else {
+        return std::make_error_code(std::errc::invalid_argument);
+      }
+    }
+    return std::error_code{}; // success
+  }
+
+private:
+  StringRef getScalaNodeText(llvm::yaml::Node *N) {
+    SmallString<32> Buffer;
+    return Saver.save(cast<llvm::yaml::ScalarNode>(N)->getValue(Buffer));
+  }
+  
+  bool parseSingleModuleEntry(llvm::yaml::Node &node,
+                              llvm::StringMap<ExplicitModuleInfo> &moduleMap) {
+    using namespace llvm::yaml;
+    auto *mapNode = dyn_cast<MappingNode>(&node);
+    if (!mapNode)
+      return true;
+    StringRef moduleName;
+    ExplicitModuleInfo result;
+    for (auto &entry : *mapNode) {
+      auto key = getScalaNodeText(entry.getKey());
+      auto val = getScalaNodeText(entry.getValue());
+      if (key == "moduleName") {
+        moduleName = val;
+      } else if (key == "modulePath") {
+        result.modulePath = val;
+      } else if (key == "docPath") {
+        result.moduleDocPath = val;
+      } else if (key == "sourceInfoPath") {
+        result.moduleSourceInfoPath = val;
+      } else {
+        // Being forgiving for future fields.
+        continue;
+      }
+    }
+    if (moduleName.empty())
+      return true;
+    moduleMap[moduleName] = std::move(result);
+    return false;
+  }
+
+  llvm::StringSaver Saver;
+};
+
 struct ModuleInterfaceLoaderOptions {
   bool remarkOnRebuildFromInterface = false;
   bool disableInterfaceLock = false;

include/swift/Option/FrontendOptions.td

@@ -224,6 +224,10 @@ def swift_module_file
 def explict_swift_module_map
   : Separate<["-"], "explicit-swift-module-map-file">, MetaVarName<"<path>">,
     HelpText<"Specify a JSON file containing information of explict Swift modules">;
+
+def placeholder_dependency_module_map
+  : Separate<["-"], "placeholder-dependency-module-map-file">, MetaVarName<"<path>">,
+    HelpText<"Specify a JSON file containing information of external Swift module dependencies">;
 }
 
 

lib/AST/ModuleDependencies.cpp

@@ -24,6 +24,10 @@ bool ModuleDependencies::isSwiftModule() const {
   return isa<SwiftModuleDependenciesStorage>(storage.get());
 }
 
+bool ModuleDependencies::isPlaceholderSwiftModule() const {
+  return isa<PlaceholderSwiftModuleDependencyStorage>(storage.get());
+}
+
 /// Retrieve the dependencies for a Swift module.
 const SwiftModuleDependenciesStorage *
 ModuleDependencies::getAsSwiftModule() const {
@@ -36,6 +40,12 @@ ModuleDependencies::getAsClangModule() const {
   return dyn_cast<ClangModuleDependenciesStorage>(storage.get());
 }
 
+/// Retrieve the dependencies for a placeholder dependency module stub.
+const PlaceholderSwiftModuleDependencyStorage *
+ModuleDependencies::getAsPlaceholderDependencyModule() const {
+  return dyn_cast<PlaceholderSwiftModuleDependencyStorage>(storage.get());
+}
+
 void ModuleDependencies::addModuleDependency(
     StringRef module, llvm::StringSet<> *alreadyAddedModules) {
   if (!alreadyAddedModules || alreadyAddedModules->insert(module).second)
@@ -102,7 +112,8 @@ ModuleDependenciesCache::getDependenciesMap(ModuleDependenciesKind kind) {
   switch (kind) {
   case ModuleDependenciesKind::Swift:
     return SwiftModuleDependencies;
-
+  case ModuleDependenciesKind::SwiftPlaceholder:
+    return PlaceholderSwiftModuleDependencies;
   case ModuleDependenciesKind::Clang:
     return ClangModuleDependencies;
   }
@@ -114,7 +125,8 @@ ModuleDependenciesCache::getDependenciesMap(ModuleDependenciesKind kind) const {
   switch (kind) {
   case ModuleDependenciesKind::Swift:
     return SwiftModuleDependencies;
-
+  case ModuleDependenciesKind::SwiftPlaceholder:
+    return PlaceholderSwiftModuleDependencies;
   case ModuleDependenciesKind::Clang:
     return ClangModuleDependencies;
   }
@@ -126,6 +138,7 @@ bool ModuleDependenciesCache::hasDependencies(
     Optional<ModuleDependenciesKind> kind) const {
   if (!kind) {
     return hasDependencies(moduleName, ModuleDependenciesKind::Swift) ||
+        hasDependencies(moduleName, ModuleDependenciesKind::SwiftPlaceholder) ||
         hasDependencies(moduleName, ModuleDependenciesKind::Clang);
   }
 
@@ -140,8 +153,11 @@ Optional<ModuleDependencies> ModuleDependenciesCache::findDependencies(
     if (auto swiftDep = findDependencies(
             moduleName, ModuleDependenciesKind::Swift))
       return swiftDep;
-
-    return findDependencies(moduleName, ModuleDependenciesKind::Clang);
+    else if (auto swiftPlaceholderDep = findDependencies(
+            moduleName, ModuleDependenciesKind::SwiftPlaceholder))
+      return swiftPlaceholderDep;
+    else
+      return findDependencies(moduleName, ModuleDependenciesKind::Clang);
   }
 
   const auto &map = getDependenciesMap(*kind);