Merge pull request swiftlang#35014 from CodaFi/whorlpool

Extend the Concept of "Interface Hash" To Entire Modules

Author: CodaFi

include/swift/AST/AbstractSourceFileDepGraphFactory.h

@@ -87,6 +87,12 @@ class AbstractSourceFileDepGraphFactory {
 
   void addAUsedDecl(const DependencyKey &def, const DependencyKey &use);
 
+  /// Add an external dependency node to the graph. If the provided fingerprint
+  /// is not \c None, it is added to the def key.
+  void addAnExternalDependency(const DependencyKey &def,
+                               const DependencyKey &use,
+                               Optional<Fingerprint> dependencyFingerprint);
+
   static Optional<Fingerprint>
   getFingerprintIfAny(std::pair<const NominalTypeDecl *, const ValueDecl *>) {
     return None;

include/swift/AST/Module.h

@@ -734,7 +734,20 @@ class ModuleDecl : public DeclContext, public TypeDecl {
 
   /// Calls \p callback for each source file of the module.
   void collectBasicSourceFileInfo(
-      llvm::function_ref<void(const BasicSourceFileInfo &)> callback);
+      llvm::function_ref<void(const BasicSourceFileInfo &)> callback) const;
+
+public:
+  /// Retrieve a fingerprint value that summarizes the contents of this module.
+  ///
+  /// This interface hash a of a module is guaranteed to change if the interface
+  /// hash of any of its (primary) source files changes. For example, when
+  /// building incrementally, the interface hash of this module will change when
+  /// the primaries contributing to its content changes. In contrast, when
+  /// a module is deserialized, the hash of every source file contributes to
+  /// the module's interface hash. It therefore serves as an effective, if
+  /// coarse-grained, way of determining when top-level changes to a module's
+  /// contents have been made.
+  Fingerprint getFingerprint() const;
 
   SourceRange getSourceRange() const { return SourceRange(); }
 

include/swift/AST/ModuleLoader.h

@@ -19,12 +19,14 @@
 
 #include "swift/AST/Identifier.h"
 #include "swift/AST/Import.h"
+#include "swift/AST/ModuleDependencies.h"
+#include "swift/Basic/ArrayRefView.h"
+#include "swift/Basic/Fingerprint.h"
 #include "swift/Basic/LLVM.h"
 #include "swift/Basic/Located.h"
 #include "swift/Basic/SourceLoc.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/TinyPtrVector.h"
-#include "swift/AST/ModuleDependencies.h"
 #include <system_error>
 
 namespace llvm {
@@ -74,8 +76,25 @@ enum class IntermoduleDepTrackingMode {
 /// Records dependencies on files outside of the current module;
 /// implemented in terms of a wrapped clang::DependencyCollector.
 class DependencyTracker {
+public:
+  /// A representation of a first-class incremental dependency known to the
+  /// Swift compiler.
+  struct IncrementalDependency {
+    std::string path;
+    Fingerprint fingerprint;
+
+    IncrementalDependency(std::string Path, Fingerprint FP)
+        : path(std::move(Path)), fingerprint(std::move(FP)) {}
+  };
+
+  inline static std::string getPath(const IncrementalDependency &id) {
+    return id.path;
+  }
+  typedef ArrayRefView<IncrementalDependency, std::string, getPath>
+      DependencyPathArrayRef;
+
   std::shared_ptr<clang::DependencyCollector> clangCollector;
-  SmallVector<std::string, 8> incrementalDeps;
+  SmallVector<IncrementalDependency, 8> incrementalDeps;
   llvm::StringSet<> incrementalDepsUniquer;
 
 public:
@@ -94,14 +113,20 @@ class DependencyTracker {
   ///
   /// No additional canonicalization or adulteration of the file path in
   /// \p File is performed.
-  void addIncrementalDependency(StringRef File);
+  void addIncrementalDependency(StringRef File, Fingerprint FP);
 
   /// Fetches the list of dependencies.
   ArrayRef<std::string> getDependencies() const;
 
   /// Fetches the list of dependencies that are known to have incremental swift
   /// dependency information embedded inside of them.
-  ArrayRef<std::string> getIncrementalDependencies() const;
+  ArrayRef<IncrementalDependency> getIncrementalDependencies() const;
+
+  /// A view of the paths of the dependencies known to have incremental swift
+  /// dependency information embedded inside of them.
+  DependencyPathArrayRef getIncrementalDependencyPaths() const {
+    return DependencyPathArrayRef(getIncrementalDependencies());
+  }
 
   /// Return the underlying clang::DependencyCollector that this
   /// class wraps.

include/swift/AST/RawComment.h

@@ -104,7 +104,7 @@ struct BasicSourceFileInfo {
 
   BasicSourceFileInfo() {}
 
-  bool populate(SourceFile *SF);
+  bool populate(const SourceFile *SF);
 };
 
 } // namespace swift

include/swift/AST/SourceFile.h

@@ -520,12 +520,24 @@ class SourceFile final : public FileUnit {
   /// Set the root refinement context for the file.
   void setTypeRefinementContext(TypeRefinementContext *TRC);
 
-  /// Whether this file has an interface hash available.
+  /// Whether this file can compute an interface hash.
   bool hasInterfaceHash() const {
     return ParsingOpts.contains(ParsingFlags::EnableInterfaceHash);
   }
 
-  /// Output this file's interface hash into the provided string buffer.
+  /// Retrieve a fingerprint value that summarizes the declarations in this
+  /// source file.
+  ///
+  /// Note that the interface hash merely summarizes the top-level declarations
+  /// in this file. Type body fingerprints are currently implemented such that
+  /// they divert tokens away from the hasher used for fingerprints. That is,
+  /// changes to the bodies of types and extensions will not result in a change
+  /// to the interface hash.
+  ///
+  /// In order for the interface hash to be enabled, this source file must be a
+  /// primary and the compiler must be set in incremental mode. If this is not
+  /// the case, this function will try to signal with an assert. It is useful
+  /// to guard requests for the interface hash with \c hasInterfaceHash().
   Fingerprint getInterfaceHash() const;
 
   void dumpInterfaceHash(llvm::raw_ostream &out) {

include/swift/Basic/Fingerprint.h

@@ -94,6 +94,11 @@ class Fingerprint final {
     return llvm::hash_value(fp.core);
   }
 
+public:
+  bool operator<(const Fingerprint &other) const {
+    return core < other.core;
+  }
+
 public:
   /// The fingerprint value consisting of 32 bytes of zeroes.
   ///

lib/AST/AbstractSourceFileDepGraphFactory.cpp

@@ -108,3 +108,16 @@ void AbstractSourceFileDepGraphFactory::addAUsedDecl(
   assert(useNode->getIsProvides() && "Use (using node) must be a provides");
   g.addArc(defNode, useNode);
 }
+
+void AbstractSourceFileDepGraphFactory::addAnExternalDependency(
+    const DependencyKey &defKey, const DependencyKey &useKey,
+    Optional<Fingerprint> maybeFP) {
+  auto *defNode = g.findExistingNodeOrCreateIfNew(defKey, maybeFP,
+                                                  false /* = !isProvides */);
+
+  auto nullableUse = g.findExistingNode(useKey);
+  assert(nullableUse.isNonNull() && "Use must be an already-added provides");
+  auto *useNode = nullableUse.get();
+  assert(useNode->getIsProvides() && "Use (using node) must be a provides");
+  g.addArc(defNode, useNode);
+}

lib/AST/FrontendSourceFileDepGraphFactory.cpp

@@ -402,16 +402,14 @@ namespace {
 /// Extracts uses out of a SourceFile
 class UsedDeclEnumerator {
   const SourceFile *SF;
-  const DependencyTracker &depTracker;
   StringRef swiftDeps;
 
   /// Cache these for efficiency
   const DependencyKey sourceFileImplementation;
 
 public:
-  UsedDeclEnumerator(const SourceFile *SF, const DependencyTracker &depTracker,
-                     StringRef swiftDeps)
-      : SF(SF), depTracker(depTracker), swiftDeps(swiftDeps),
+  UsedDeclEnumerator(const SourceFile *SF, StringRef swiftDeps)
+      : SF(SF), swiftDeps(swiftDeps),
         sourceFileImplementation(DependencyKey::createKeyForWholeSourceFile(
             DeclAspect::implementation, swiftDeps)) {}
 
@@ -446,7 +444,6 @@ class UsedDeclEnumerator {
       }
       }
     });
-    enumerateExternalUses(enumerator);
     enumerateNominalUses(enumerator);
   }
 
@@ -474,23 +471,59 @@ class UsedDeclEnumerator {
       enumerateUse<NodeKind::nominal>(context, "", enumerator);
     });
   }
+};
+} // end namespace
+
+namespace {
+class ExternalDependencyEnumerator {
+  const DependencyTracker &depTracker;
+  const DependencyKey sourceFileImplementation;
+
+public:
+  using UseEnumerator = llvm::function_ref<void(
+      const DependencyKey &, const DependencyKey &, Optional<Fingerprint>)>;
+
+  ExternalDependencyEnumerator(const DependencyTracker &depTracker,
+                               StringRef swiftDeps)
+      : depTracker(depTracker),
+        sourceFileImplementation(DependencyKey::createKeyForWholeSourceFile(
+            DeclAspect::implementation, swiftDeps)) {}
 
   void enumerateExternalUses(UseEnumerator enumerator) {
-    for (StringRef s : depTracker.getIncrementalDependencies())
-      enumerateUse<NodeKind::incrementalExternalDepend>("", s, enumerator);
+    for (const auto &id : depTracker.getIncrementalDependencies()) {
+      enumerateUse<NodeKind::incrementalExternalDepend>(enumerator, id.path,
+                                                        id.fingerprint);
+    }
+    for (StringRef s : depTracker.getDependencies()) {
+      enumerateUse<NodeKind::externalDepend>(enumerator, s, None);
+    }
+  }
 
-    for (StringRef s : depTracker.getDependencies())
-      enumerateUse<NodeKind::externalDepend>("", s, enumerator);
+private:
+  template <NodeKind kind>
+  void enumerateUse(UseEnumerator createDefUse, StringRef name,
+                    Optional<Fingerprint> maybeFP) {
+    static_assert(kind == NodeKind::incrementalExternalDepend ||
+                      kind == NodeKind::externalDepend,
+                  "Not a kind of external dependency!");
+    createDefUse(DependencyKey(kind, DeclAspect::interface, "", name.str()),
+                 sourceFileImplementation, maybeFP);
   }
 };
 } // end namespace
 
 void FrontendSourceFileDepGraphFactory::addAllUsedDecls() {
-  UsedDeclEnumerator(SF, depTracker, swiftDeps)
+  UsedDeclEnumerator(SF, swiftDeps)
       .enumerateAllUses(
           [&](const DependencyKey &def, const DependencyKey &use) {
             addAUsedDecl(def, use);
           });
+  ExternalDependencyEnumerator(depTracker, swiftDeps)
+      .enumerateExternalUses([&](const DependencyKey &def,
+                                 const DependencyKey &use,
+                                 Optional<Fingerprint> maybeFP) {
+        addAnExternalDependency(def, use, maybeFP);
+      });
 }
 
 //==============================================================================
@@ -499,19 +532,10 @@ void FrontendSourceFileDepGraphFactory::addAllUsedDecls() {
 
 ModuleDepGraphFactory::ModuleDepGraphFactory(const ModuleDecl *Mod,
                                              bool emitDot)
-    : AbstractSourceFileDepGraphFactory(Mod->getASTContext().hadError(),
-                                        Mod->getNameStr(), Fingerprint::ZERO(),
-                                        emitDot, Mod->getASTContext().Diags),
-
-      Mod(Mod) {
-  // Since a fingerprint only summarizes the state of the module but not
-  // the state of its fingerprinted sub-declarations, and since a module
-  // contains no state other than sub-declarations, its fingerprint does not
-  // matter and can just be some arbitrary value. Should it be the case that a
-  // change in a declaration that does not have a fingerprint must cause
-  // a rebuild of a file outside of the module, this assumption will need
-  // to be revisited.
-}
+    : AbstractSourceFileDepGraphFactory(
+          Mod->getASTContext().hadError(), Mod->getNameStr(),
+          Mod->getFingerprint(), emitDot, Mod->getASTContext().Diags),
+      Mod(Mod) {}
 
 void ModuleDepGraphFactory::addAllDefinedDecls() {
   // TODO: express the multiple provides and depends streams with variadic

lib/AST/Module.cpp

@@ -1536,9 +1536,9 @@ const clang::Module *ModuleDecl::findUnderlyingClangModule() const {
 }
 
 void ModuleDecl::collectBasicSourceFileInfo(
-    llvm::function_ref<void(const BasicSourceFileInfo &)> callback) {
-  for (FileUnit *fileUnit : getFiles()) {
-    if (SourceFile *SF = dyn_cast<SourceFile>(fileUnit)) {
+    llvm::function_ref<void(const BasicSourceFileInfo &)> callback) const {
+  for (const FileUnit *fileUnit : getFiles()) {
+    if (const auto *SF = dyn_cast<SourceFile>(fileUnit)) {
       BasicSourceFileInfo info;
       if (info.populate(SF))
         continue;
@@ -1549,6 +1549,25 @@ void ModuleDecl::collectBasicSourceFileInfo(
   }
 }
 
+Fingerprint ModuleDecl::getFingerprint() const {
+  StableHasher hasher = StableHasher::defaultHasher();
+  SmallVector<Fingerprint, 16> FPs;
+  collectBasicSourceFileInfo([&](const BasicSourceFileInfo &bsfi) {
+    FPs.emplace_back(bsfi.InterfaceHash);
+  });
+  
+  // Sort the fingerprints lexicographically so we have a stable hash despite
+  // an unstable ordering of files across rebuilds.
+  // FIXME: If we used a commutative hash combine (say, if we could take an
+  // XOR here) we could avoid this sort.
+  std::sort(FPs.begin(), FPs.end(), std::less<Fingerprint>());
+  for (const auto &FP : FPs) {
+    hasher.combine(FP);
+  }
+
+  return Fingerprint{std::move(hasher)};
+}
+
 //===----------------------------------------------------------------------===//
 // Cross-Import Overlays
 //===----------------------------------------------------------------------===//

lib/AST/ModuleLoader.cpp

@@ -50,9 +50,10 @@ DependencyTracker::addDependency(StringRef File, bool IsSystem) {
                                      /*IsMissing=*/false);
 }
 
-void DependencyTracker::addIncrementalDependency(StringRef File) {
+void DependencyTracker::addIncrementalDependency(StringRef File,
+                                                 Fingerprint FP) {
   if (incrementalDepsUniquer.insert(File).second) {
-    incrementalDeps.emplace_back(File.str());
+    incrementalDeps.emplace_back(File.str(), FP);
   }
 }
 
@@ -61,7 +62,7 @@ DependencyTracker::getDependencies() const {
   return clangCollector->getDependencies();
 }
 
-ArrayRef<std::string>
+ArrayRef<DependencyTracker::IncrementalDependency>
 DependencyTracker::getIncrementalDependencies() const {
   return incrementalDeps;
 }