Reorganization of SourceFileDepGraph building and mocking.

Author: David Ungar

include/swift/AST/FineGrainedDependencies.h

@@ -93,14 +93,20 @@ template <typename KeyT, typename ValueT> class Memoizer {
 public:
   Memoizer() = default;
 
+  Optional<ValueT> findExisting(KeyT key) {
+    auto iter = memos.find(key);
+    if (iter != memos.end())
+      return iter->second;
+    return None;
+  }
+
   /// \p createFn must create a \ref ValueT that corresponds to the \ref KeyT
   /// passed into it.
   ValueT
   findExistingOrCreateIfNew(KeyT key,
                             function_ref<ValueT(const KeyT &)> createFn) {
-    auto iter = memos.find(key);
-    if (iter != memos.end())
-      return iter->second;
+    if (auto existing = findExisting(key))
+      return existing.getValue();
     ValueT v = createFn(key);
     (void)insert(key, v);
     return v;
@@ -499,10 +505,23 @@ class DependencyKey {
   }
   bool isInterface() const { return getAspect() == DeclAspect::interface; }
 
+  /// Create just the interface half of the keys for a provided Decl or Decl
+  /// pair
+  template <NodeKind kind, typename Entity>
+  static DependencyKey createForProvidedEntityInterface(Entity);
+
   /// Given some type of provided entity compute the context field of the key.
   template <NodeKind kind, typename Entity>
   static std::string computeContextForProvidedEntity(Entity);
 
+  DependencyKey correspondingImplementation() const {
+    return withAspect(DeclAspect::implementation);
+  }
+
+  DependencyKey withAspect(DeclAspect aspect) const {
+    return DependencyKey(kind, aspect, context, name);
+  }
+
   /// Given some type of provided entity compute the name field of the key.
   template <NodeKind kind, typename Entity>
   static std::string computeNameForProvidedEntity(Entity);
@@ -514,7 +533,8 @@ class DependencyKey {
   template <NodeKind kind>
   static DependencyKey createDependedUponKey(StringRef);
 
-  static DependencyKey createKeyForWholeSourceFile(StringRef swiftDeps);
+  static DependencyKey createKeyForWholeSourceFile(DeclAspect,
+                                                   StringRef swiftDeps);
 
   std::string humanReadableName() const;
 
@@ -616,6 +636,10 @@ class DepGraphNode {
   /// See SourceFileDepGraphNode::SourceFileDepGraphNode(...) and
   /// ModuleDepGraphNode::ModuleDepGraphNode(...) Don't set swiftDeps on
   /// creation because this field can change if a node is moved.
+  DepGraphNode(DependencyKey key, Optional<StringRef> fingerprint)
+      : DepGraphNode(key, fingerprint ? fingerprint->str()
+                                      : Optional<std::string>()) {}
+
   DepGraphNode(DependencyKey key, Optional<std::string> fingerprint)
       : key(key), fingerprint(fingerprint) {}
   DepGraphNode(const DepGraphNode &other) = default;
@@ -627,8 +651,12 @@ class DepGraphNode {
 
   const DependencyKey &getKey() const { return key; }
 
-  const Optional<std::string> &getFingerprint() const { return fingerprint; }
-
+  const Optional<StringRef> getFingerprint() const {
+    if (fingerprint) {
+      return StringRef(fingerprint.getValue());
+    }
+    return None;
+  }
   /// When driver reads a SourceFileDepGraphNode, it may be a node that was
   /// created to represent a name-lookup (a.k.a a "depend") in the frontend. In
   /// that case, the node represents an entity that resides in some other file
@@ -637,7 +665,9 @@ class DepGraphNode {
   /// (someday) have a fingerprint. In order to preserve the
   /// ModuleDepGraphNode's identity but bring its fingerprint up to date, it
   /// needs to set the fingerprint *after* the node has been created.
-  void setFingerprint(Optional<std::string> fp) { fingerprint = fp; }
+  void setFingerprint(Optional<StringRef> fp) {
+    fingerprint = fp ? fp->str() : Optional<std::string>();
+  }
 
   SWIFT_DEBUG_DUMP;
   void dump(llvm::raw_ostream &os) const;
@@ -684,7 +714,7 @@ class SourceFileDepGraphNode : public DepGraphNode {
   SourceFileDepGraphNode() : DepGraphNode(), sequenceNumber(~0) {}
 
   /// Used by the frontend to build nodes.
-  SourceFileDepGraphNode(DependencyKey key, Optional<std::string> fingerprint,
+  SourceFileDepGraphNode(DependencyKey key, Optional<StringRef> fingerprint,
                          bool isProvides)
       : DepGraphNode(key, fingerprint), isProvides(isProvides) {
     assert(key.verify());
@@ -780,34 +810,6 @@ class SourceFileDepGraph {
   SourceFileDepGraph(const SourceFileDepGraph &g) = delete;
   SourceFileDepGraph(SourceFileDepGraph &&g) = default;
 
-  /// Simulate loading for unit testing:
-  /// \param swiftDepsFileName The name of the swiftdeps file of the phony job
-  /// \param includePrivateDeps Whether the graph includes intra-file arcs
-  /// \param hadCompilationError Simulate a compilation error
-  /// \param interfaceHash The interface hash of the simulated graph
-  /// \param simpleNamesByRDK A map of vectors of names keyed by reference
-  /// dependency key \param compoundNamesByRDK A map of (mangledHolder,
-  /// baseName) pairs keyed by reference dependency key. For single-name
-  /// dependencies, an initial underscore indicates that the name does not
-  /// cascade. For compound names, it is the first name, the holder which
-  /// indicates non-cascading. For member names, an initial underscore indicates
-  /// file-privacy.
-  static SourceFileDepGraph
-  simulateLoad(std::string swiftDepsFileName, const bool includePrivateDeps,
-               const bool hadCompilationError, std::string interfaceHash,
-               llvm::StringMap<std::vector<std::string>> simpleNamesByRDK,
-               llvm::StringMap<std::vector<std::pair<std::string, std::string>>>
-                   compoundNamesByRDK);
-
-  static constexpr char noncascadingOrPrivatePrefix = '#';
-  static constexpr char nameFingerprintSeparator = ',';
-
-  static std::string noncascading(std::string name);
-
-  LLVM_ATTRIBUTE_UNUSED
-  static std::string privatize(std::string name);
-
-
   /// Nodes are owned by the graph.
   ~SourceFileDepGraph() {
     forEachNode([&](SourceFileDepGraphNode *n) { delete n; });
@@ -851,12 +853,15 @@ class SourceFileDepGraph {
   /// The frontend creates a pair of nodes for every tracked Decl and the source
   /// file itself.
   InterfaceAndImplementationPair<SourceFileDepGraphNode>
-  findExistingNodePairOrCreateAndAddIfNew(
-      NodeKind k, const ContextNameFingerprint &contextNameFingerprint);
+  findExistingNodePairOrCreateAndAddIfNew(const DependencyKey &interfaceKey,
+                                          Optional<StringRef> fingerprint);
+
+  NullablePtr<SourceFileDepGraphNode>
+  findExistingNode(const DependencyKey &key);
 
   SourceFileDepGraphNode *
-  findExistingNodeOrCreateIfNew(DependencyKey key,
-                                const Optional<std::string> &fingerprint,
+  findExistingNodeOrCreateIfNew(const DependencyKey &key,
+                                const Optional<StringRef> fingerprint,
                                 bool isProvides);
 
   /// \p Use is the Node that must be rebuilt when \p def changes.

include/swift/AST/SourceFileDepGraphConstructor.h

@@ -0,0 +1,143 @@
+//===----- SourceFileDepGraphConstructor.h ----------------------*- C++ -*-===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2018 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SWIFT_AST_SOURCE_FILE_DEP_GRAPH_CONSTRUCTOR_H
+#define SWIFT_AST_SOURCE_FILE_DEP_GRAPH_CONSTRUCTOR_H
+
+#include <swift/AST/DeclContext.h>
+
+namespace swift {
+namespace fine_grained_dependencies {
+/// Reads the information provided by the frontend and builds the
+/// SourceFileDepGraph
+class SourceFileDepGraphConstructor {
+  /// To match the existing system, set this to false.
+  /// To include even private entities and get intra-file info, set to true.
+  const bool includePrivateDeps;
+
+  /// If there was an error, cannot get accurate info.
+  const bool hadCompilationError;
+
+public:
+  /// A lambda which invokes its parameter for every unconstrained \c Decl used
+  /// in the source file, passing in the base name, and whether this use
+  /// cascades..
+  using ForEachUnconstrainedDeclUsed =
+      function_ref<void(function_ref<void(StringRef, bool)>)>;
+
+  /// A lambda which invokes its parameter for every constrained \c Decl used in
+  /// the source file, passing in the manged type name of the holder, the member
+  /// base name, whether the holder is private to its source file, and whether
+  /// the use cascades. If the baseName is empty, this entry will be a \c
+  /// potentialMember dependency.
+  using ForEachConstrainedDeclUsed =
+      function_ref<void(function_ref<void(StringRef, StringRef, bool, bool)>)>;
+
+  /// Adds nodes to the graph corresponding to a defined \c Decl.
+  /// Parameters are an interface key (implementation node will also be added)
+  /// and the fingerprint if any.
+  using AddDefinedDecl =
+      function_ref<void(const DependencyKey &, Optional<StringRef>)>;
+
+  /// A lambda which invokes its parameter for every mock \c Decl "defined" in
+  /// the source file, passing in to its argument the \c NodeKind, \c context,
+  /// and \c name fields of resulting \c DependencyKey pair, and also the
+  /// fingerprint (if present) of the resulting node pair.
+  using ForEachDefinedDecl = function_ref<void(AddDefinedDecl)>;
+
+  /// A lambda which invokes its parameter for every  use of a \c Decl in the
+  /// source file, passing in to its argument the \c DependencyKey used, and the
+  /// \c DependencyKey doing the using. (A.k.a. a "def" and an a "use".
+  using ForEachUsedDecl = function_ref<void(
+      function_ref<void(const DependencyKey &, const DependencyKey &)>)>;
+
+private:
+  /// Graph under construction
+  SourceFileDepGraph g;
+
+public:
+  /// Expose this layer to enable faking up a constructor for testing.
+  /// See the instance variable comments for explanation.
+  // clang-format off
+  SourceFileDepGraphConstructor(
+    bool includePrivateDeps,
+    bool hadCompilationError
+    ) :
+    includePrivateDeps(includePrivateDeps),
+    hadCompilationError(hadCompilationError)
+    {} // clang-format on
+
+  /// Create a SourceFileDepGraph.
+  ///
+  /// \param swiftDeps The "output path name" and \c sourceFileProvide node
+  /// names \param interfaceHash The fingerprint for the \c sourceFileProvide
+  /// nodes. \param forEachMockDefinedDecl Iterator for "provides", i.e.
+  /// definitions in this file \param forEachMockUsedDecl Iterator for
+  /// "depends", i.e. used definitions in this file
+  SourceFileDepGraph construct(StringRef swiftDeps, StringRef interfaceHash,
+                               ForEachDefinedDecl forEachDefinedDecl,
+                               ForEachUsedDecl forEachUsedDecl) {
+
+    return addSourceFileNodesAndThen(swiftDeps, interfaceHash, [&] {
+      addAllDefinedDecls(forEachDefinedDecl);
+      addAllUsedDecls(forEachUsedDecl);
+    });
+  }
+
+  /// Centralize the invariant that the fingerprint of the whole file is the
+  /// interface hash
+  static std::string getFingerprint(SourceFile *SF);
+
+  void enumerateDefinedDecls(SourceFile *SF, AddDefinedDecl addDefinedDeclFn);
+
+private:
+  /// Add the first two nodes in the graph and then, if there is no compilation
+  /// error, do rest of the work to build the graph.
+  SourceFileDepGraph addSourceFileNodesAndThen(StringRef name,
+                                               StringRef fingerprint,
+                                               function_ref<void()> doTheRest);
+
+  /// Add the "provides" nodes when mocking up a graph
+  void addAllDefinedDecls(ForEachDefinedDecl forEachDefinedDecl);
+
+  /// Add the "depends" nodes and arcs when mocking a graph
+  void addAllUsedDecls(ForEachUsedDecl forEachUsedDecl);
+
+  /// At present, only nominals, protocols, and extensions have (body)
+  /// fingerprints
+  static Optional<std::string>
+  getFingerprintIfAny(std::pair<const NominalTypeDecl *, const ValueDecl *>);
+
+  static Optional<std::string> getFingerprintIfAny(const Decl *d);
+
+  static std::string getInterfaceHash(SourceFile *SF);
+
+  void addSourceFileNodesToGraph(StringRef swiftDeps, StringRef fingerprint);
+
+  /// Given an array of Decls or pairs of them in \p declsOrPairs
+  /// create node pairs for context and name
+  template <NodeKind kind, typename ContentsT>
+  void
+  enumerateAllProviderNodesOfAGivenType(std::vector<ContentsT> &contentsVec,
+                                        AddDefinedDecl addDefinedDeclFn);
+
+  /// Add an pair of interface, implementation nodes to the graph, which
+  /// represent some \c Decl defined in this source file. \param key the
+  /// interface key of the pair
+  void addDefinedDecl(const DependencyKey &key,
+                      Optional<StringRef> fingerprint);
+};
+
+} // namespace fine_grained_dependencies
+} // namespace swift
+
+#endif // SWIFT_AST_SOURCE_FILE_DEP_GRAPH_CONSTRUCTOR_H

include/swift/Driver/FineGrainedDependencyDriverGraph.h

@@ -57,8 +57,7 @@ class ModuleDepGraphNode : public DepGraphNode {
   bool hasBeenTracedAsADependent = false;
 
 public:
-  ModuleDepGraphNode(const DependencyKey &key,
-                     Optional<std::string> fingerprint,
+  ModuleDepGraphNode(const DependencyKey &key, Optional<StringRef> fingerprint,
                      Optional<std::string> swiftDeps)
       : DepGraphNode(key, fingerprint), swiftDeps(swiftDeps) {}
 
@@ -187,11 +186,13 @@ class ModuleDepGraph {
   /// files for the same name distinct, keep a sequence number for each name.
   std::unordered_map<std::string, unsigned> dotFileSequenceNumber;
 
+public:
   const bool verifyFineGrainedDependencyGraphAfterEveryImport;
   const bool emitFineGrainedDependencyDotFileAfterEveryImport;
 
   const bool EnableTypeFingerprints;
 
+private:
   /// If tracing dependencies, holds a vector used to hold the current path
   /// def - use/def - use/def - ...
   Optional<std::vector<const ModuleDepGraphNode *>> currentPathIfTracing;
@@ -336,15 +337,6 @@ class ModuleDepGraph {
                                      const SourceFileDepGraph &,
                                      DiagnosticEngine &);
 
-  /// Also for unit tests
-  Changes
-  simulateLoad(const driver::Job *cmd,
-               llvm::StringMap<std::vector<std::string>> simpleNames,
-               llvm::StringMap<std::vector<std::pair<std::string, std::string>>>
-                   compoundNames = {},
-               const bool includePrivateDeps = false,
-               const bool hadCompilationError = false);
-
 
 private:
   /// Read a SourceFileDepGraph belonging to \p job from \p buffer
@@ -515,6 +507,8 @@ class ModuleDepGraph {
   /// Record a new (to this graph) Job.
   void registerJob(const driver::Job *);
 
+  std::vector<const driver::Job *> getAllJobs() const;
+
   /// Find jobs that were previously not known to need compilation but that
   /// depend on \c externalDependency.
   std::vector<const driver::Job *>