Fine-grained and driver fixes.

Restructure fine-grained-dependencies to enable unit testing

Get frontend to emit correct swiftdeps file (fine-grained when needed) and only emit dot file for -emit-fine-grained-dependency-sourcefile-dot-files

Use deterministic order for more information outputs.

Set EnableFineGrainedDependencies consistently in frontend.

Tolerate errors that result in null getExtendedNominal()

Fix memory issue by removing node everywhere.

Break up print routine

Be more verbose so it will compile on Linux.

Sort batchable jobs, too.

Author: David Ungar

include/swift/AST/FineGrainedDependencies.h

@@ -325,7 +325,7 @@ class BiIndexedTwoStageMap {
 /// Write out the .swiftdeps file for a frontend compilation of a primary file.
 bool emitReferenceDependencies(DiagnosticEngine &diags, SourceFile *SF,
                                const DependencyTracker &depTracker,
-                               StringRef outputPath);
+                               StringRef outputPath, bool alsoEmitDotFile);
 //==============================================================================
 // MARK: Enums
 //==============================================================================
@@ -438,8 +438,8 @@ class DependencyKey {
         name() {}
 
   /// For constructing a key in the frontend.
-  DependencyKey(NodeKind kind, DeclAspect aspect, std::string context,
-                std::string name)
+  DependencyKey(NodeKind kind, DeclAspect aspect, const std::string &context,
+                const std::string &name)
       : kind(kind), aspect(aspect), context(context), name(name) {
     assert(verify());
   }
@@ -449,7 +449,7 @@ class DependencyKey {
   StringRef getContext() const { return context; }
   StringRef getName() const { return name; }
 
-  StringRef getSwiftDepsFromSourceFileProvide() const {
+  StringRef getSwiftDepsFromASourceFileProvideNodeKey() const {
     assert(getKind() == NodeKind::sourceFileProvide &&
            "Receiver must be sourceFileProvide.");
     return getName();
@@ -494,11 +494,19 @@ class DependencyKey {
   static std::string computeNameForProvidedEntity(Entity);
 
   /// Given some type of depended-upon entity create the key.
-  template <NodeKind kind, typename Entity>
-  static DependencyKey createDependedUponKey(const Entity &);
+  static DependencyKey createDependedUponKey(StringRef mangledHolderName,
+                                             StringRef memberBaseName);
+
+  template <NodeKind kind>
+  static DependencyKey createDependedUponKey(StringRef);
+
+  static DependencyKey
+  createTransitiveKeyForWholeSourceFile(StringRef swiftDeps);
 
   std::string humanReadableName() const;
 
+  StringRef aspectName() const { return DeclAspectNames[size_t(aspect)]; }
+
   void dump(llvm::raw_ostream &os) const { os << asString() << "\n"; }
   SWIFT_DEBUG_DUMP { dump(llvm::errs()); }
 
@@ -535,6 +543,13 @@ struct std::hash<typename swift::fine_grained_dependencies::DeclAspect> {
   }
 };
 
+namespace swift {
+namespace fine_grained_dependencies {
+using ContextNameFingerprint =
+    std::tuple<std::string, std::string, Optional<std::string>>;
+}
+} // namespace swift
+
 //==============================================================================
 // MARK: DepGraphNode
 //==============================================================================
@@ -684,6 +699,9 @@ class SourceFileDepGraphNode : public DepGraphNode {
   }
 
   /// Record the sequence number, \p n, of another use.
+  /// The relationship between an interface and its implementation is NOT
+  /// included here. See \c
+  /// SourceFileDepGraph::findExistingNodePairOrCreateAndAddIfNew.
   void addDefIDependUpon(size_t n) {
     if (n != getSequenceNumber())
       defsIDependUpon.insert(n);
@@ -727,6 +745,25 @@ class SourceFileDepGraph {
   SourceFileDepGraph(const SourceFileDepGraph &g) = delete;
   SourceFileDepGraph(SourceFileDepGraph &&g) = default;
 
+  /// Simulate loading for unit testing:
+  /// \param swiftDepsFileName The name of the swiftdeps file of the phony job
+  /// \param includePrivateDeps Whether the graph includes intra-file arcs
+  /// \param hadCompilationError Simulate a compilation error
+  /// \param interfaceHash The interface hash of the simulated graph
+  /// \param simpleNamesByRDK A map of vectors of names keyed by reference
+  /// dependency key \param compoundNamesByRDK A map of (mangledHolder,
+  /// baseName) pairs keyed by reference dependency key. For single-name
+  /// dependencies, an initial underscore indicates that the name does not
+  /// cascade. For compound names, it is the first name, the holder which
+  /// indicates non-cascading. For member names, an initial underscore indicates
+  /// file-privacy.
+  static SourceFileDepGraph
+  simulateLoad(std::string swiftDepsFileName, const bool includePrivateDeps,
+               const bool hadCompilationError, std::string interfaceHash,
+               llvm::StringMap<std::vector<std::string>> simpleNamesByRDK,
+               llvm::StringMap<std::vector<std::pair<std::string, std::string>>>
+                   compoundNamesByRDK);
+
   /// Nodes are owned by the graph.
   ~SourceFileDepGraph() {
     forEachNode([&](SourceFileDepGraphNode *n) { delete n; });
@@ -746,7 +783,7 @@ class SourceFileDepGraph {
   InterfaceAndImplementationPair<SourceFileDepGraphNode>
   getSourceFileNodePair() const;
 
-  StringRef getSwiftDepsFromSourceFileProvide() const;
+  StringRef getSwiftDepsOfJobThatProducedThisGraph() const;
 
   std::string getGraphID() const {
     return getSourceFileNodePair().getInterface()->getKey().humanReadableName();
@@ -770,12 +807,13 @@ class SourceFileDepGraph {
   /// The frontend creates a pair of nodes for every tracked Decl and the source
   /// file itself.
   InterfaceAndImplementationPair<SourceFileDepGraphNode>
-  findExistingNodePairOrCreateAndAddIfNew(NodeKind k, StringRef context,
-                                          StringRef name,
-                                          Optional<std::string> fingerprint);
+  findExistingNodePairOrCreateAndAddIfNew(
+      NodeKind k, const ContextNameFingerprint &contextNameFingerprint);
 
-  SourceFileDepGraphNode *findExistingNodeOrCreateIfNew(
-      DependencyKey key, Optional<std::string> fingerprint, bool isProvides);
+  SourceFileDepGraphNode *
+  findExistingNodeOrCreateIfNew(DependencyKey key,
+                                const Optional<std::string> &fingerprint,
+                                bool isProvides);
 
   /// \p Use is the Node that must be rebuilt when \p def changes.
   /// Record that fact in the graph.
@@ -892,7 +930,7 @@ template <typename GraphT> class DotFileEmitter {
   }
   void emitArcs() {
     g.forEachArc([&](const NodeT *def, const NodeT *use) {
-      if (includeGraphArc(use, def))
+      if (includeGraphArc(def, use))
         emitGraphArc(def, use);
     });
   }

include/swift/Basic/LangOptions.h

@@ -285,10 +285,14 @@ namespace swift {
     /// Whether to verify the parsed syntax tree and emit related diagnostics.
     bool VerifySyntaxTree = false;
 
-    /// Scaffolding to permit experimentation with finer-grained dependencies
-    /// and faster rebuilds.
+    /// Emit the newer, finer-grained swiftdeps file. Eventually will support
+    /// faster rebuilds.
     bool EnableFineGrainedDependencies = false;
-    
+
+    /// When using fine-grained dependencies, emit dot files for every swiftdeps
+    /// file.
+    bool EmitFineGrainedDependencySourcefileDotFiles = false;
+
     /// To mimic existing system, set to false.
     /// To experiment with including file-private and private dependency info,
     /// set to true.

include/swift/Driver/Action.h

@@ -168,6 +168,23 @@ class CompileJobAction : public JobAction {
   static bool classof(const Action *A) {
     return A->getKind() == Action::Kind::CompileJob;
   }
+
+  /// Return a _single_ TY_Swift InputAction, if one exists;
+  /// if 0 or >1 such inputs exist, return nullptr.
+  const InputAction *findSingleSwiftInput() const {
+    auto Inputs = getInputs();
+    auto isSwiftInput = [](const Action *A) -> const InputAction* {
+      if (auto const *S = dyn_cast<InputAction>(A))
+        return S->getType() == file_types::TY_Swift ? S : nullptr;
+      return nullptr;
+    };
+    const auto loc1 = std::find_if(Inputs.begin(), Inputs.end(), isSwiftInput);
+    if (loc1 == Inputs.end())
+      return nullptr; // none found
+    // Ensure uniqueness
+    const auto loc2 = std::find_if(loc1 + 1, Inputs.end(), isSwiftInput);
+    return loc2 == Inputs.end() ? dyn_cast<InputAction>(*loc1) : nullptr;
+  }
 };
 
 class InterpretJobAction : public JobAction {

include/swift/Driver/Compilation.h

@@ -521,6 +521,18 @@ class Compilation {
   /// How many .swift input files?
   unsigned countSwiftInputs() const;
 
+  /// Unfortunately the success or failure of a Swift compilation is currently
+  /// sensitive to the order in which files are processed, at least in terms of
+  /// the order of processing extensions (and likely other ways we haven't
+  /// discovered yet). So long as this is true, we need to make sure any batch
+  /// job we build names its inputs in an order that's a subsequence of the
+  /// sequence of inputs the driver was initially invoked with.
+  ///
+  /// Also use to write out information in a consistent order.
+  void sortJobsToMatchCompilationInputs(
+      ArrayRef<const Job *> unsortedJobs,
+      SmallVectorImpl<const Job *> &sortedJobs) const;
+
 private:
   /// Perform all jobs.
   ///