Fast dependency scanning for Swift

Implement a new "fast" dependency scanning option,
`-scan-dependencies`, in the Swift frontend that determines all
of the source file and module dependencies for a given set of
Swift sources. It covers four forms of modules:

1) Swift (serialized) module files, by reading the module header
2) Swift interface files, by parsing the source code to find imports
3) Swift source modules, by parsing the source code to find imports
4) Clang modules, using Clang's fast dependency scanning tool

A single `-scan-dependencies` operation maps out the full
dependency graph for the given Swift source files, including all
of the Swift and Clang modules that may need to be built, such
that all of the work can be scheduled up front by the Swift
driver or any other build system that understands this
option. The dependency graph is emitted as JSON, which can be
consumed by these other tools.

Author: DougGregor

include/swift/AST/ASTContext.h

@@ -78,6 +78,7 @@ namespace swift {
   class LazyContextData;
   class LazyIterableDeclContextData;
   class LazyMemberLoader;
+  class ModuleDependencies;
   class PatternBindingDecl;
   class PatternBindingInitializer;
   class SourceFile;
@@ -91,6 +92,7 @@ namespace swift {
   class Identifier;
   class InheritedNameSet;
   class ModuleDecl;
+  class ModuleDependenciesCache;
   class ModuleLoader;
   class NominalTypeDecl;
   class NormalProtocolConformance;
@@ -710,6 +712,15 @@ class ASTContext final {
   void addModuleLoader(std::unique_ptr<ModuleLoader> loader,
                        bool isClang = false, bool isDWARF = false);
 
+  /// Retrieve the module dependencies for the module with the given name.
+  ///
+  /// \param isUnderlyingClangModule When true, only look for a Clang module
+  /// with the given name, ignoring any Swift modules.
+  Optional<ModuleDependencies> getModuleDependencies(
+      StringRef moduleName,
+      bool isUnderlyingClangModule,
+      ModuleDependenciesCache &cache);
+
   /// Load extensions to the given nominal type from the external
   /// module loaders.
   ///

include/swift/AST/ModuleDependencies.h

@@ -0,0 +1,284 @@
+//===--- ModuleDependencies.h - Module Dependencies -------------*- C++ -*-===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines data structures for capturing module dependencies.
+//
+//===----------------------------------------------------------------------===//
+#ifndef SWIFT_AST_MODULE_DEPENDENCIES_H
+#define SWIFT_AST_MODULE_DEPENDENCIES_H
+
+#include "swift/Basic/LLVM.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/Optional.h"
+#include "llvm/ADT/StringSet.h"
+#include <string>
+#include <vector>
+
+namespace swift {
+
+class ClangModuleDependenciesCacheImpl;
+class SourceFile;
+
+/// Which kind of module dependencies we are looking for.
+enum class ModuleDependenciesKind : int8_t {
+  Swift,
+  Clang,
+};
+
+/// Base class for the variant storage of ModuleDependencies.
+class ModuleDependenciesStorageBase {
+public:
+  const bool isSwiftModule;
+
+  ModuleDependenciesStorageBase(bool isSwiftModule,
+                                const std::string &compiledModulePath)
+      : isSwiftModule(isSwiftModule),
+        compiledModulePath(compiledModulePath) { }
+
+  virtual ModuleDependenciesStorageBase *clone() const = 0;
+
+  virtual ~ModuleDependenciesStorageBase();
+
+  /// The path to the compiled module file.
+  const std::string compiledModulePath;
+
+  /// The set of modules on which this module depends.
+  std::vector<std::string> moduleDependencies;
+};
+
+/// Describes the dependencies of a Swift module.
+class SwiftModuleDependenciesStorage : public ModuleDependenciesStorageBase {
+public:
+  /// The Swift interface file, if it can be used to generate the module file.
+  const Optional<std::string> swiftInterfaceFile;
+
+  /// Bridging header file, if there is one.
+  Optional<std::string> bridgingHeaderFile;
+
+  /// Swift source files that are part of the Swift module, when known.
+  std::vector<std::string> sourceFiles;
+
+  /// Source files on which the bridging header depends.
+  std::vector<std::string> bridgingSourceFiles;
+
+  SwiftModuleDependenciesStorage(
+      const std::string &compiledModulePath,
+      const Optional<std::string> &swiftInterfaceFile
+  ) : ModuleDependenciesStorageBase(/*isSwiftModule=*/true, compiledModulePath),
+      swiftInterfaceFile(swiftInterfaceFile) { }
+
+  ModuleDependenciesStorageBase *clone() const override {
+    return new SwiftModuleDependenciesStorage(*this);
+  }
+
+  static bool classof(const ModuleDependenciesStorageBase *base) {
+    return base->isSwiftModule;
+  }
+};
+
+class ClangModuleDependenciesStorage : public ModuleDependenciesStorageBase {
+public:
+  /// The module map file used to generate the Clang module.
+  const std::string moduleMapFile;
+
+  /// The file dependencies
+  const std::vector<std::string> fileDependencies;
+
+  ClangModuleDependenciesStorage(
+      const std::string &compiledModulePath,
+      const std::string &moduleMapFile,
+      const std::vector<std::string> &fileDependencies
+  ) : ModuleDependenciesStorageBase(/*isSwiftModule=*/false,
+                                    compiledModulePath),
+      moduleMapFile(moduleMapFile),
+      fileDependencies(fileDependencies) { }
+
+  ModuleDependenciesStorageBase *clone() const override {
+    return new ClangModuleDependenciesStorage(*this);
+  }
+
+  static bool classof(const ModuleDependenciesStorageBase *base) {
+    return !base->isSwiftModule;
+  }
+};
+
+/// Describes the dependencies of a given module.
+class ModuleDependencies {
+private:
+  std::unique_ptr<ModuleDependenciesStorageBase> storage;
+
+  ModuleDependencies(std::unique_ptr<ModuleDependenciesStorageBase> &&storage)
+    : storage(std::move(storage)) { }
+
+public:
+  ModuleDependencies(const ModuleDependencies &other)
+    : storage(other.storage->clone()) { }
+  ModuleDependencies(ModuleDependencies &&other) = default;
+
+  ModuleDependencies &operator=(const ModuleDependencies &other) {
+    storage.reset(other.storage->clone());
+    return *this;
+  }
+
+  ModuleDependencies &operator=(ModuleDependencies &&other) = default;
+
+  /// Describe the module dependencies for a Swift module that can be
+  /// built from a Swift interface file (\c .swiftinterface).
+  static ModuleDependencies forSwiftInterface(
+      const std::string &compiledModulePath,
+      const std::string &swiftInterfaceFile) {
+    return ModuleDependencies(
+        std::make_unique<SwiftModuleDependenciesStorage>(
+          compiledModulePath, swiftInterfaceFile));
+  }
+
+  /// Describe the module dependencies for a serialized or parsed Swift module.
+  static ModuleDependencies forSwiftModule(
+      const std::string &compiledModulePath) {
+    return ModuleDependencies(
+        std::make_unique<SwiftModuleDependenciesStorage>(
+          compiledModulePath, None));
+  }
+
+  /// Describe the module dependencies for a Clang module that can be
+  /// built from a module map and headers.
+  static ModuleDependencies forClangModule(
+      const std::string &compiledModulePath,
+      const std::string &moduleMapFile,
+      const std::vector<std::string> &fileDependencies) {
+    return ModuleDependencies(
+        std::make_unique<ClangModuleDependenciesStorage>(compiledModulePath,
+                                                         moduleMapFile,
+                                                         fileDependencies));
+  }
+
+  /// Retrieve the path to the compiled module.
+  const std::string getCompiledModulePath() const {
+    return storage->compiledModulePath;
+  }
+
+  /// Retrieve the module-level dependencies.
+  ArrayRef<std::string> getModuleDependencies() const {
+    return storage->moduleDependencies;
+  }
+
+  /// Whether the dependencies are for a Swift module.
+  bool isSwiftModule() const;
+
+  ModuleDependenciesKind getKind() const {
+    return isSwiftModule() ? ModuleDependenciesKind::Swift
+                           : ModuleDependenciesKind::Clang;
+  }
+  /// Retrieve the dependencies for a Swift module.
+  const SwiftModuleDependenciesStorage *getAsSwiftModule() const;
+
+  /// Retrieve the dependencies for a Clang module.
+  const ClangModuleDependenciesStorage *getAsClangModule() const;
+
+  /// Add a dependency on the given module, if it was not already in the set.
+  void addModuleDependency(StringRef module,
+                           llvm::StringSet<> &alreadyAddedModules);
+
+  /// Add all of the module dependencies for the imports in the given source
+  /// file to the set of module dependencies.
+  void addModuleDependencies(const SourceFile &sf,
+                             llvm::StringSet<> &alreadyAddedModules);
+
+  /// Add a bridging header to a Swift module's dependencies.
+  void addBridgingHeader(StringRef bridgingHeader);
+};
+
+using ModuleDependencyID = std::pair<std::string, ModuleDependenciesKind>;
+
+/// A cache describing the set of module dependencies that has been queried
+/// thus far.
+class ModuleDependenciesCache {
+  /// All cached module dependencies, in the order in which they were
+  /// encountered.
+  std::vector<ModuleDependencyID> AllModules;
+
+  /// Dependencies for Swift modules that have already been computed.
+  llvm::StringMap<ModuleDependencies> SwiftModuleDependencies;
+
+  /// Dependencies for Clang modules that have already been computed.
+  llvm::StringMap<ModuleDependencies> ClangModuleDependencies;
+
+  /// Additional information needed for Clang dependency scanning.
+  ClangModuleDependenciesCacheImpl *clangImpl = nullptr;
+
+  /// Function that will delete \c clangImpl properly.
+  void (*clangImplDeleter)(ClangModuleDependenciesCacheImpl *) = nullptr;
+
+  /// Free up the storage associated with the Clang implementation.
+  void destroyClangImpl() {
+    if (this->clangImplDeleter)
+      this->clangImplDeleter(this->clangImpl);
+  }
+
+  /// Retrieve the dependencies map that corresponds to the given dependency
+  /// kind.
+  llvm::StringMap<ModuleDependencies> &getDependenciesMap(
+      ModuleDependenciesKind kind);
+  const llvm::StringMap<ModuleDependencies> &getDependenciesMap(
+      ModuleDependenciesKind kind) const;
+
+public:
+  ModuleDependenciesCache() { }
+
+  ModuleDependenciesCache(const ModuleDependenciesCache &) = delete;
+  ModuleDependenciesCache &operator=(const ModuleDependenciesCache &) = delete;
+
+  ~ModuleDependenciesCache() {
+    destroyClangImpl();
+  }
+
+  /// Set the Clang-specific implementation data.
+  void setClangImpl(
+      ClangModuleDependenciesCacheImpl *clangImpl,
+      void (*clangImplDeleter)(ClangModuleDependenciesCacheImpl *)) {
+    destroyClangImpl();
+
+    this->clangImpl = clangImpl;
+    this->clangImplDeleter = clangImplDeleter;
+  }
+
+  /// Retrieve the Clang-specific implementation data;
+  ClangModuleDependenciesCacheImpl *getClangImpl() const {
+    return clangImpl;
+  }
+
+  /// Whether we have cached dependency information for the given module.
+  bool hasDependencies(StringRef moduleName,
+                       Optional<ModuleDependenciesKind> kind) const;
+
+  /// Look for module dependencies for a module with the given name.
+  ///
+  /// \returns the cached result, or \c None if there is no cached entry.
+  Optional<ModuleDependencies> findDependencies(
+      StringRef moduleName,
+      Optional<ModuleDependenciesKind> kind) const;
+
+  /// Record dependencies for the given module.
+  void recordDependencies(StringRef moduleName,
+                          ModuleDependencies dependencies,
+                          ModuleDependenciesKind kind);
+
+  /// Reference the list of all module dependencies.
+  const std::vector<ModuleDependencyID> &getAllModules() const {
+    return AllModules;
+  }
+};
+
+}
+
+#endif /* SWIFT_AST_MODULE_DEPENDENCIES_H */

include/swift/AST/ModuleLoader.h

@@ -23,6 +23,7 @@
 #include "swift/Basic/SourceLoc.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallSet.h"
+#include "llvm/ADT/StringSet.h"
 #include "llvm/ADT/TinyPtrVector.h"
 
 namespace llvm {
@@ -41,7 +42,10 @@ class ClangImporterOptions;
 class ClassDecl;
 class FileUnit;
 class ModuleDecl;
+class ModuleDependencies;
+class ModuleDependenciesCache;
 class NominalTypeDecl;
+class SourceFile;
 class TypeDecl;
 
 enum class KnownProtocolKind : uint8_t;
@@ -177,6 +181,12 @@ class ModuleLoader {
   /// Discover overlays declared alongside this file and add infomation about
   /// them to it.
   void findOverlayFiles(SourceLoc diagLoc, ModuleDecl *module, FileUnit *file);
+
+  /// Retrieve the dependencies for the given, named module, or \c None
+  /// if no such module exists.
+  virtual Optional<ModuleDependencies> getModuleDependencies(
+      StringRef moduleName,
+      ModuleDependenciesCache &cache) = 0;
 };
 
 } // namespace swift

include/swift/Basic/FileTypes.def

@@ -71,6 +71,9 @@ TYPE("tbd",                 TBD,                       "tbd",             "")
 // Swift section of the internal wiki.
 TYPE("module-trace",        ModuleTrace,               "trace.json",      "")
 
+// Complete dependency information for the given Swift files as JSON.
+TYPE("json-dependencies",   JSONDependencies,          "dependencies.json",      "")
+
 TYPE("index-data",          IndexData,                 "",                "")
 TYPE("yaml-opt-record",     YAMLOptRecord,             "opt.yaml",        "")
 TYPE("bitstream-opt-record",BitstreamOptRecord,        "opt.bitstream",   "")

include/swift/ClangImporter/ClangImporter.h

@@ -369,6 +369,9 @@ class ClangImporter final : public ClangModuleLoader {
 
   void verifyAllModules() override;
 
+  Optional<ModuleDependencies> getModuleDependencies(
+      StringRef moduleName, ModuleDependenciesCache &cache) override;
+
   clang::TargetInfo &getTargetInfo() const override;
   clang::ASTContext &getClangASTContext() const override;
   clang::Preprocessor &getClangPreprocessor() const override;

include/swift/Frontend/FrontendOptions.h

@@ -132,6 +132,8 @@ class FrontendOptions {
 
     EmitPCM, ///< Emit precompiled Clang module from a module map
     DumpPCM, ///< Dump information about a precompiled Clang module
+
+    ScanDependencies,   ///< Scan dependencies of Swift source files
   };
 
   /// Indicates the action the user requested that the frontend perform.

include/swift/Option/Options.td

@@ -1031,6 +1031,10 @@ def sanitize_coverage_EQ : CommaJoined<["-"], "sanitize-coverage=">,
   HelpText<"Specify the type of coverage instrumentation for Sanitizers and"
   " additional options separated by commas">;
 
+def scan_dependencies : Flag<["-"], "scan-dependencies">,
+  HelpText<"Scan dependencies of the given Swift sources">, ModeOpt,
+  Flags<[FrontendOption, NoInteractiveOption, DoesNotAffectIncrementalBuild]>;
+
 def enable_astscope_lookup : Flag<["-"], "enable-astscope-lookup">,
   Flags<[FrontendOption]>,
   HelpText<"Enable ASTScope-based unqualified name lookup">;