[FineGrainedDeps] Fix the non-deterministic .swiftdeps output

Fix the non-deterministic .swiftdeps output. In contrary to the comments
in the `FineGrainedDependencies.h`, the non-determinism is not because
of the use of unordered_* data structure there. Those data structures
are not traversed so removing all the unused traversing methods to avoid
the confusion.

The true reason for the non-determinism is all the DenseSet in the
Evaluator dependency tracking, that causes the FineGrainedDependencies
to see arbitrary ordering. Use `SetVector` instead to keep track of the
insertion order to make dependency output to be deterministic.

rdar://104876331

Author: cachemeifyoucan

include/swift/AST/EvaluatorDependencies.h

@@ -24,6 +24,7 @@
 #include "swift/Basic/NullablePtr.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/SetVector.h"
 #include <vector>
 
 namespace swift {
@@ -57,9 +58,12 @@ class DependencyRecorder {
   /// References recorded while evaluating a dependency source request for each
   /// source file. This map is updated upon completion of a dependency source
   /// request, and includes all references from each downstream request as well.
-  llvm::DenseMap<SourceFile *,
-                 llvm::DenseSet<DependencyCollector::Reference,
-                                DependencyCollector::Reference::Info>>
+  llvm::DenseMap<
+      SourceFile *,
+      llvm::SetVector<DependencyCollector::Reference,
+                      llvm::SmallVector<DependencyCollector::Reference>,
+                      llvm::DenseSet<DependencyCollector::Reference,
+                                     DependencyCollector::Reference::Info>>>
       fileReferences;
 
   /// References recorded while evaluating each request. This map is populated
@@ -73,8 +77,11 @@ class DependencyRecorder {
   /// dependency sink request, we update the innermost set of references.
   /// Upon completion of a request, we union the completed request's references
   /// with the next innermost active request.
-  std::vector<llvm::SmallDenseSet<DependencyCollector::Reference, 2,
-                                  DependencyCollector::Reference::Info>>
+  std::vector<llvm::SetVector<
+      DependencyCollector::Reference,
+      std::vector<DependencyCollector::Reference>,
+      llvm::SmallDenseSet<DependencyCollector::Reference, 2,
+                          DependencyCollector::Reference::Info>>>
       activeRequestReferences;
 
 #ifndef NDEBUG
@@ -163,8 +170,7 @@ void evaluator::DependencyRecorder::endRequest(const Request &req) {
     return;
 
   // Convert the set of dependencies into a vector.
-  std::vector<DependencyCollector::Reference>
-      vec(recorded.begin(), recorded.end());
+  std::vector<DependencyCollector::Reference> vec = recorded.takeVector();
 
   // The recorded dependencies bubble up to the parent request.
   if (!activeRequestReferences.empty()) {

include/swift/AST/FineGrainedDependencies.h

@@ -137,10 +137,6 @@ template <typename KeyT, typename ValueT> class Memoizer {
 template <typename Key1, typename Key2, typename Value> class TwoStageMap {
 public:
   // Define this here so it can be changed easily.
-  // TODO: Use llvm structure such as DenseMap. However, DenseMap does not
-  // preserve pointers to elements, so be careful!
-  // TODO: Consider using an ordered structure to guarantee determinism
-  // when compilation order changes.
   template <typename Key, typename MapValue>
   using Map = std::unordered_map<Key, MapValue>;
 
@@ -188,32 +184,6 @@ template <typename Key1, typename Key2, typename Value> class TwoStageMap {
   /// Returns the submap at \p k1. May create one if not present.
   Map<Key2, Value> &operator[](const Key1 &k1) { return map[k1]; }
 
-  /// Invoke \p fn on each Key2 and Value matching (k, *)
-  void forEachValueMatching(
-      const Key1 &k1,
-      function_ref<void(const Key2 &, const Value &)> fn) const {
-    const auto &iter = map.find(k1);
-    if (iter == map.end())
-      return;
-    for (auto &p : iter->second)
-      fn(p.first, p.second);
-  }
-
-  /// Invoke \p fn for each entry
-  void forEachEntry(
-      function_ref<void(const Key1 &, const Key2 &, const Value &)> fn) const {
-    for (const auto &p : map)
-      for (const auto &p2 : p.second)
-        fn(p.first, p2.first, p2.second);
-  }
-
-  /// Invoke fn for each Key1 and submap
-  void
-  forEachKey1(function_ref<void(const Key1 &, const InnerMap &)> fn) const {
-    for (const auto &p : map)
-      fn(p.first, p.second);
-  }
-
   /// Check integrity and call \p verifyFn for each element, so that element can
   /// be verified.
   ///
@@ -292,36 +262,6 @@ class BiIndexedTwoStageMap {
     return findAndErase(k1, k2);
   }
 
-  /// Invoke \p fn on each Key2 and Value matching (\p k1, *)
-  void forEachValueMatching(
-      const Key1 &k1,
-      function_ref<void(const Key2 &, const Value &)> fn) const {
-    map1.forEachValueMatching(k1, fn);
-  }
-
-  /// Invoke \p fn on each Key1 and Value matching (*, \p k2)
-  void forEachValueMatching(
-      const Key2 &k2,
-      function_ref<void(const Key1 &, const Value &)> fn) const {
-    map2.forEachValueMatching(k2, fn);
-  }
-
-  /// Invoke \p fn for each entry
-  void forEachEntry(
-      function_ref<void(const Key1 &, const Key2 &, const Value &)> fn) const {
-    map1.forEachEntry(fn);
-  }
-
-  /// Invoke fn for each Key1 and submap
-  void forEachKey1(function_ref<void(const Key1 &, const Key2Map &)> fn) const {
-    map1.forEachKey1(fn);
-  }
-
-  /// Invoke fn for each Key2 and submap
-  void forEachKey2(function_ref<void(const Key1 &, const Key1Map &)> fn) const {
-    map2.forEachKey1(fn);
-  }
-
   /// Verify the integrity of each map and the cross-map consistency.
   /// Then call \p verifyFn for each entry found in each of the two maps,
   /// passing an index so that the verifyFn knows which map is being tested.

test/Frontend/output_determinism_check.swift

@@ -8,8 +8,7 @@
 /// object files should match when forcing object generation.
 // RUN: %target-swift-frontend -module-name test -emit-dependencies -c -o %t/test.o -primary-file %s -enable-deterministic-check -always-compile-output-files 2>&1 | %FileCheck %s --check-prefix=OBJECT_OUTPUT --check-prefix=DEPS_OUTPUT
 
-/// FIXME: Fine-grain dependencies graph is not deterministics.
-/// FAIL:  %target-swift-frontend -module-name test -emit-reference-dependencies-path %t/test.swiftdeps -c -o %t/test.o -primary-file %s -enable-deterministic-check -always-compile-output-files
+/// RUN:  %target-swift-frontend -module-name test -emit-reference-dependencies-path %t/test.swiftdeps -c -o %t/test.o -primary-file %s -enable-deterministic-check -always-compile-output-files
 
 /// Explicit module build. Check building swiftmodule from interface file.
 // RUN: %target-swift-frontend -scan-dependencies -module-name test -o %t/test.json %s -enable-deterministic-check 2>&1 | %FileCheck %s --check-prefix=DEPSCAN_OUTPUT