[Dependency Scanning] Refactor dependency scanner to return an in-memory format as a result

This commit refactors ScanDependencies.cpp to split the functionality into two functional groups:
- Scan execution code that performs the mechanics of the scan and produces an in-memory result
- Dependency scanner entry-points used when the scanning action is invoked as a `swift-frontend` execution mode
This commit also adds the aforementioned in-memory dependency scanning result in `FullDependencies.h`, modeled after the InterModuleDependencyGraph representation in swift-driver

Author: artemcm

include/swift/DependencyScan/DependencyScanningTool.h

@@ -16,8 +16,6 @@
 #include "swift/AST/ModuleDependencies.h"
 #include "swift/Frontend/PrintingDiagnosticConsumer.h"
 #include "swift/DependencyScan/ScanDependencies.h"
-#include "llvm/ADT/StringMap.h"
-#include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/StringSet.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/StringSaver.h"
@@ -37,7 +35,7 @@ class DependencyScanningTool {
   ///
   /// \returns a \c StringError with the diagnostic output if errors
   /// occurred, \c FullDependencies otherwise.
-  llvm::ErrorOr<std::string>
+  llvm::ErrorOr<FullDependencies>
   getDependencies(ArrayRef<const char *> Command,
                   const llvm::StringSet<> &PlaceholderModules);
 

include/swift/DependencyScan/FullDependencies.h

@@ -0,0 +1,221 @@
+//===------------------ FullDependencies.h - Swift Compiler ---------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SWIFT_DEPENDENCY_FULL_DEPENDENCIES_H
+#define SWIFT_DEPENDENCY_FULL_DEPENDENCIES_H
+
+#include "swift/AST/ModuleDependencies.h"
+#include <set>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+// This file describes the format for in-memory dependency scanning result
+// produced by the Swift dependency scanner. 
+
+namespace swift {
+namespace dependencies {
+
+/// Swift modules to be built from a module interface, may have a bridging
+/// header.
+struct SwiftTextualModuleDetails {
+  /// The module interface from which this module was built, if any.
+  std::string ModuleInterfacePath;
+
+  /// The paths of potentially ready-to-use compiled modules for the interface.
+  std::vector<std::string> CompiledModuleCandidates;
+
+  /// The bridging header, if any.
+  std::string BridgingHeaderPath;
+
+  /// The source files referenced by the bridging header.
+  std::vector<std::string> BridgingSourceFiles;
+
+  /// (Clang) modules on which the bridging header depends.
+  std::vector<std::string> BridgingModuleDependencies;
+
+  /// Options to the compile command required to build this module interface
+  std::vector<std::string> CommandLine;
+
+  /// To build a PCM to be used by this Swift module, we need to append these
+  /// arguments to the generic PCM build arguments reported from the dependency
+  /// graph.
+  std::vector<std::string> ExtraPcmArgs;
+
+  /// The hash value that will be used for the generated module
+  std::string ContextHash;
+
+  /// A flag to indicate whether or not this module is a framework.
+  bool IsFramework = false;
+};
+
+/// Swift modules with only a binary module file.
+struct SwiftBinaryModuleDetails {
+  /// The path to the pre-compiled binary module
+  std::string CompiledModulePath;
+
+  /// The path to the .swiftModuleDoc file.
+  std::string ModuleDocPath;
+
+  /// The path to the .swiftSourceInfo file.
+  std::string ModuleSourceInfoPath;
+};
+
+/// Swift placeholder modules carry additional details that specify their
+/// module doc path and source info paths.
+struct SwiftPlaceholderModuleDetails {
+  /// The path to the pre-compiled binary module
+  std::string CompiledModulePath;
+
+  /// The path to the .swiftModuleDoc file.
+  std::string ModuleDocPath;
+
+  /// The path to the .swiftSourceInfo file.
+  std::string ModuleSourceInfoPath;
+};
+
+/// Clang modules are built from a module map file.
+struct ClangModuleDetails {
+  /// The path to the module map used to build this module.
+  std::string ModuleMapPath;
+
+  /// Set of PCM Arguments of depending modules which
+  /// are covered by the directDependencies info of this module
+  // TODO: Should this be modeled here or in the clients?
+  // const std::set<std::string> DependenciesCapturedPCMArgs;
+
+  /// clang-generated context hash
+  std::string ContextHash;
+
+  /// Options to the compile command required to build this clang modulemap
+  std::vector<std::string> CommandLine;
+};
+
+struct ModuleDetails {
+  ModuleDetails(SwiftTextualModuleDetails Details)
+      : DetailsKind(ModuleDependenciesKind::SwiftTextual), Value(Details) {}
+  ModuleDetails(SwiftBinaryModuleDetails Details)
+      : DetailsKind(ModuleDependenciesKind::SwiftBinary), Value(Details) {}
+  ModuleDetails(SwiftPlaceholderModuleDetails Details)
+      : DetailsKind(ModuleDependenciesKind::SwiftPlaceholder), Value(Details) {}
+  ModuleDetails(ClangModuleDetails Details)
+      : DetailsKind(ModuleDependenciesKind::Clang), Value(Details) {}
+
+  ~ModuleDetails() {
+    switch (DetailsKind) {
+    case ModuleDependenciesKind::SwiftTextual:
+      (&Value.TextualDetails)->~SwiftTextualModuleDetails();
+      break;
+    case ModuleDependenciesKind::SwiftBinary:
+      (&Value.BinaryDetails)->~SwiftBinaryModuleDetails();
+      break;
+    case ModuleDependenciesKind::SwiftPlaceholder:
+      (&Value.PlaceholderDetails)->~SwiftPlaceholderModuleDetails();
+      break;
+    case ModuleDependenciesKind::Clang:
+      (&Value.ClangDetails)->~ClangModuleDetails();
+      break;
+    }
+  }
+
+  ModuleDetails(const ModuleDetails &src) :
+    DetailsKind(src.DetailsKind) {
+    switch (src.DetailsKind) {
+    case ModuleDependenciesKind::SwiftBinary:
+        new (&Value.BinaryDetails) auto(src.Value.BinaryDetails);
+        break;
+    case ModuleDependenciesKind::SwiftTextual:
+      new (&Value.TextualDetails) auto(src.Value.TextualDetails);
+      break;
+    case ModuleDependenciesKind::SwiftPlaceholder:
+      new (&Value.PlaceholderDetails) auto(src.Value.PlaceholderDetails);
+      break;
+    case ModuleDependenciesKind::Clang:
+      new (&Value.ClangDetails) auto(src.Value.ClangDetails);
+      break;
+    }
+  }
+
+  const SwiftTextualModuleDetails *getAsSwiftTextualModule() const {
+    return DetailsKind == ModuleDependenciesKind::SwiftTextual
+               ? &Value.TextualDetails
+               : nullptr;
+  }
+  const SwiftPlaceholderModuleDetails *
+  getAsPlaceholderDependencyModule() const {
+    return DetailsKind == ModuleDependenciesKind::SwiftPlaceholder
+               ? &Value.PlaceholderDetails
+               : nullptr;
+  }
+  const SwiftBinaryModuleDetails *getAsSwiftBinaryModule() const {
+    return DetailsKind == ModuleDependenciesKind::SwiftBinary
+               ? &Value.BinaryDetails
+               : nullptr;
+  }
+  const ClangModuleDetails *getAsClangModule() const {
+    return DetailsKind == ModuleDependenciesKind::Clang ? &Value.ClangDetails
+                                                        : nullptr;
+  }
+
+private:
+  ModuleDependenciesKind DetailsKind;
+
+  union ModuleDetailsValue {
+    SwiftTextualModuleDetails TextualDetails;
+    SwiftBinaryModuleDetails BinaryDetails;
+    SwiftPlaceholderModuleDetails PlaceholderDetails;
+    ClangModuleDetails ClangDetails;
+
+    ModuleDetailsValue(SwiftTextualModuleDetails Details)
+        : TextualDetails(Details) {}
+    ModuleDetailsValue(SwiftBinaryModuleDetails Details)
+        : BinaryDetails(Details) {}
+    ModuleDetailsValue(SwiftPlaceholderModuleDetails Details)
+        : PlaceholderDetails(Details) {}
+    ModuleDetailsValue(ClangModuleDetails Details) : ClangDetails(Details) {}
+
+    ModuleDetailsValue() {}
+    ~ModuleDetailsValue() {}
+  } Value;
+};
+
+struct ModuleInfo {
+  /// The ID of the module this info describes
+  ModuleDependencyID ID;
+
+  /// The path for the module.
+  std::string ModulePath;
+
+  /// The source files used to build this module.
+  std::vector<std::string> SourceFiles;
+
+  /// The set of direct module dependencies of this module.
+  std::vector<ModuleDependencyID> DirectDependencies;
+
+  /// Specific details of a particular kind of module.
+  ModuleDetails Details;
+};
+
+struct FullDependencies {
+  /// The name of the main module for this dependency graph (root node)
+  const std::string MainModuleName;
+
+  /// The complete set of modules discovered
+  std::vector<ModuleInfo> Modules;
+
+  FullDependencies(std::string MainModuleName)
+      : MainModuleName(MainModuleName), Modules() {}
+};
+} // namespace dependencies
+} // namespace swift
+
+#endif // SWIFT_DEPENDENCY_FULL_DEPENDENCIES_H

include/swift/DependencyScan/ScanDependencies.h

@@ -10,10 +10,11 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef SWIFT_FRONTENDTOOL_SCANDEPENDENCIES_H
-#define SWIFT_FRONTENDTOOL_SCANDEPENDENCIES_H
+#ifndef SWIFT_DEPENDENCY_SCANDEPENDENCIES_H
+#define SWIFT_DEPENDENCY_SCANDEPENDENCIES_H
 
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Error.h"
 
 namespace llvm {
 class StringSaver;
@@ -27,47 +28,48 @@ class ModuleDependenciesCache;
 
 namespace dependencies {
 
+struct FullDependencies;
 struct BatchScanInput {
   llvm::StringRef moduleName;
   llvm::StringRef arguments;
   llvm::StringRef outputPath;
   bool isSwift;
 };
 
-// MARK: swift-frontend -scan-dependencies entry points
+// MARK: FrontendTool dependency scanner entry points
 /// Scans the dependencies of the main module of \c instance and writes out
 /// the resulting JSON according to the instance's output parameters.
-/// This method is used for swift-frontend invocations in dependency scanning mode
-/// (-scan-dependencies), where the module dependency cache is not shared.
 bool scanDependencies(CompilerInstance &instance);
 
+/// Identify all imports in the translation unit's module.
+bool prescanDependencies(CompilerInstance &instance);
+
 /// Batch scan the dependencies for modules specified in \c batchInputFile.
 bool batchScanDependencies(CompilerInstance &instance,
                            llvm::StringRef batchInputFile);
 
-/// Identify all imports in the translation unit's module.
-bool prescanMainModuleDependencies(CompilerInstance &instance);
-
+/// Batch prescan the imports of modules specified in \c batchInputFile.
+bool batchPrescanDependencies(CompilerInstance &instance,
+                              llvm::StringRef batchInputFile);
 
-// MARK: dependency scanning execution
+// MARK: Dependency scanning execution
 /// Scans the dependencies of the main module of \c instance.
-bool performModuleScan(CompilerInstance &instance,
-                       ModuleDependenciesCache &cache,
-                       llvm::raw_ostream &out);
+llvm::ErrorOr<FullDependencies>
+performModuleScan(CompilerInstance &instance,
+                  ModuleDependenciesCache &cache);
 
 /// Batch scan the dependencies for modules specified in \c batchInputFile.
-bool performBatchModuleScan(CompilerInstance &instance,
-                            ModuleDependenciesCache &cache,
-                            llvm::StringSaver &saver,
-                            const std::vector<BatchScanInput> &BatchInput);
-
-/// Scan for dependencies of a module with a specified name, producing the resulting output
-/// at the specified output path.
-bool scanBatchModuleEntry(CompilerInstance &instance,
+std::vector<llvm::ErrorOr<FullDependencies>>
+performBatchModuleScan(CompilerInstance &instance,
+                       ModuleDependenciesCache &cache, llvm::StringSaver &saver,
+                       const std::vector<BatchScanInput> &BatchInput);
+
+/// Batch prescan the imports of modules specified in \c batchInputFile.
+std::vector<llvm::ErrorOr<std::vector<std::string>>>
+performBatchModulePrescan(CompilerInstance &instance,
                           ModuleDependenciesCache &cache,
-                          llvm::StringRef moduleName,
-                          bool isClang,
-                          llvm::StringRef outputPath);
+                          llvm::StringSaver &saver,
+                          const std::vector<BatchScanInput> &BatchInput);
 
 } // end namespace dependencies
 } // end namespace swift

lib/DependencyScan/DependencyScanningTool.cpp

@@ -11,7 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "swift/DependencyScan/DependencyScanningTool.h"
-#include "swift/DependencyScan/ModuleInfo.h"
+#include "swift/DependencyScan/FullDependencies.h"
 #include "swift/AST/DiagnosticEngine.h"
 #include "swift/AST/DiagnosticsFrontend.h"
 #include "swift/Basic/LLVMInitialize.h"
@@ -28,7 +28,7 @@ DependencyScanningTool::DependencyScanningTool()
     : SharedCache(std::make_unique<ModuleDependenciesCache>()), PDC(), Alloc(),
       Saver(Alloc) {}
 
-llvm::ErrorOr<std::string> DependencyScanningTool::getDependencies(
+llvm::ErrorOr<FullDependencies> DependencyScanningTool::getDependencies(
     ArrayRef<const char *> Command,
     const llvm::StringSet<> &PlaceholderModules) {
   // The primary instance used to scan the query Swift source-code
@@ -37,13 +37,13 @@ llvm::ErrorOr<std::string> DependencyScanningTool::getDependencies(
     return EC;
   auto Instance = std::move(*InstanceOrErr);
 
-  std::string JSONOutput;
-  llvm::raw_string_ostream OSS(JSONOutput);
-  performModuleScan(*Instance.get(), *SharedCache, OSS);
-  OSS.flush();
+  // Execute the scanning action, retreiving the in-memory result
+  auto DependenciesOrErr = performModuleScan(*Instance.get(), *SharedCache);
+  if (DependenciesOrErr.getError())
+    return std::make_error_code(std::errc::not_supported);
+  auto Dependencies = std::move(*DependenciesOrErr);
 
-  // TODO: swiftch to an in-memory representation
-  return JSONOutput;
+  return Dependencies;
 }
 
 std::error_code DependencyScanningTool::getDependencies(