Initial plumbing for private dependencies

Add a mode bit to the dependency collector that respects the frontend flag in the previous commit.

Notably, we now write over the dependency files at the end of the compiler pipeline when this flag is on so that dependency from SILGen and IRGen are properly written to disk.

Author: CodaFi

include/swift/AST/Evaluator.h

@@ -245,7 +245,8 @@ class Evaluator {
   /// diagnostics through the given diagnostics engine.
   Evaluator(DiagnosticEngine &diags,
             bool debugDumpCycles,
-            bool buildDependencyGraph);
+            bool buildDependencyGraph,
+            bool enableExperimentalPrivateDeps);
 
   /// Emit GraphViz output visualizing the request graph.
   void emitRequestEvaluatorGraphViz(llvm::StringRef graphVizPath);

include/swift/AST/EvaluatorDependencies.h

@@ -113,7 +113,22 @@ struct DependencyCollector {
   llvm::SmallVector<evaluator::DependencySource, 8> dependencySources;
 
 public:
-  DependencyCollector() = default;
+  enum class Mode {
+    // Enables the current "status quo" behavior of the dependency collector.
+    //
+    // By default, the dependency collector moves to register dependencies in
+    // the referenced name trackers at the top of the active dependency stack.
+    StatusQuo,
+    // Enables an experimental mode to only register private dependencies.
+    //
+    // This mode restricts the dependency collector to ignore changes of
+    // scope. This has practical effect of charging all unqualified lookups to
+    // the primary file being acted upon instead of to the destination file.
+    ExperimentalPrivateDependencies,
+  };
+  Mode mode;
+
+  explicit DependencyCollector(Mode mode) : mode{mode} {};
 
 public:
   /// Registers a named reference from the current dependency scope to a member
@@ -206,19 +221,43 @@ struct DependencyCollector {
   };
 
 private:
+  /// Returns the first dependency source registered with the tracker, or
+  /// \c nullptr if no dependency sources have been registered.
+  SourceFile *getFirstDependencySourceOrNull() const {
+    if (dependencySources.empty())
+      return nullptr;
+    return dependencySources.front().getPointer();
+  }
+
   /// If there is an active dependency source, returns its
   /// \c ReferencedNameTracker. Else, returns \c nullptr.
   ReferencedNameTracker *getActiveDependencyTracker() const {
-    if (auto *source = getActiveDependencySourceOrNull())
-      return source->getRequestBasedReferencedNameTracker();
-    return nullptr;
+    SourceFile *source = nullptr;
+    switch (mode) {
+    case Mode::StatusQuo:
+      source = getActiveDependencySourceOrNull();
+      break;
+    case Mode::ExperimentalPrivateDependencies:
+      source = getFirstDependencySourceOrNull();
+      break;
+    }
+    
+    if (!source)
+      return nullptr;
+
+    return source->getRequestBasedReferencedNameTracker();
   }
 
   /// Returns \c true if the scope of the current active source cascades.
   ///
   /// If there is no active scope, the result always cascades.
   bool isActiveSourceCascading() const {
-    return getActiveSourceScope() == evaluator::DependencyScope::Cascading;
+    switch (mode) {
+    case Mode::StatusQuo:
+      return getActiveSourceScope() == evaluator::DependencyScope::Cascading;
+    case Mode::ExperimentalPrivateDependencies:
+      return false;
+    }
   }
 };
 } // end namespace evaluator

lib/AST/ASTContext.cpp

@@ -552,7 +552,8 @@ ASTContext::ASTContext(LangOptions &langOpts, TypeCheckerOptions &typeckOpts,
     SearchPathOpts(SearchPathOpts), SourceMgr(SourceMgr), Diags(Diags),
     evaluator(Diags,
               langOpts.DebugDumpCycles,
-              langOpts.BuildRequestDependencyGraph),
+              langOpts.BuildRequestDependencyGraph,
+              langOpts.EnableExperientalPrivateIntransitiveDependencies),
     TheBuiltinModule(createBuiltinModule(*this)),
     StdlibModuleName(getIdentifier(STDLIB_NAME)),
     SwiftShimsModuleName(getIdentifier(SWIFT_SHIMS_NAME)),

lib/AST/Evaluator.cpp

@@ -60,12 +60,22 @@ void Evaluator::registerRequestFunctions(
   requestFunctionsByZone.push_back({zoneID, functions});
 }
 
-Evaluator::Evaluator(DiagnosticEngine &diags,
-                     bool debugDumpCycles,
-                     bool buildDependencyGraph)
-  : diags(diags),
-    debugDumpCycles(debugDumpCycles),
-    buildDependencyGraph(buildDependencyGraph) { }
+static evaluator::DependencyCollector::Mode
+computeDependencyModeFromFlags(bool enableExperimentalPrivateDeps) {
+  using Mode = evaluator::DependencyCollector::Mode;
+  if (enableExperimentalPrivateDeps) {
+    return Mode::ExperimentalPrivateDependencies;
+  }
+  return Mode::StatusQuo;
+}
+
+Evaluator::Evaluator(DiagnosticEngine &diags, bool debugDumpCycles,
+                     bool buildDependencyGraph,
+                     bool enableExperimentalPrivateDeps)
+    : diags(diags), debugDumpCycles(debugDumpCycles),
+      buildDependencyGraph(buildDependencyGraph),
+      collector{computeDependencyModeFromFlags(enableExperimentalPrivateDeps)} {
+}
 
 void Evaluator::emitRequestEvaluatorGraphViz(llvm::StringRef graphVizPath) {
   std::error_code error;

lib/Driver/ToolChains.cpp

@@ -250,7 +250,9 @@ void ToolChain::addCommonFrontendArgs(const OutputInfo &OI,
                        options::OPT_enable_experimental_concise_pound_file);
   inputArgs.AddLastArg(arguments,
                        options::OPT_verify_incremental_dependencies);
-  
+  inputArgs.AddLastArg(arguments,
+                       options::OPT_experimental_private_intransitive_dependencies);
+
   // Pass on any build config options
   inputArgs.AddAllArgs(arguments, options::OPT_D);
 

lib/Frontend/CompilerInvocation.cpp

@@ -434,6 +434,9 @@ static bool ParseLangArgs(LangOptions &Opts, ArgList &Args,
   if (Args.hasArg(OPT_enable_experimental_additive_arithmetic_derivation))
     Opts.EnableExperimentalAdditiveArithmeticDerivedConformances = true;
 
+  if (Args.hasArg(OPT_experimental_private_intransitive_dependencies))
+    Opts.EnableExperientalPrivateIntransitiveDependencies = true;
+
   Opts.EnableExperimentalForwardModeDifferentiation |=
       Args.hasArg(OPT_enable_experimental_forward_mode_differentiation);
 

lib/FrontendTool/FrontendTool.cpp

@@ -2184,6 +2184,14 @@ int swift::performFrontend(ArrayRef<const char *> Args,
                        Invocation.getFrontendOptions().DumpAPIPath);
   }
 
+  // If we're asked to enable private intransitive dependencies, we need to
+  // write over the dependency files we just emitted because we need to
+  // get the dependencies written post-Sema down on disk.
+  // FIXME: Evaluate the impact turning this on universally has.
+  if (Invocation.getLangOptions().EnableExperientalPrivateIntransitiveDependencies) {
+    emitReferenceDependenciesForAllPrimaryInputsIfNeeded(*Instance);
+  }
+
   // Verify reference dependencies of the current compilation job *before*
   // verifying diagnostics so that the former can be tested via the latter.
   if (Invocation.getFrontendOptions().EnableIncrementalDependencyVerifier) {

test/Incremental/PrivateDependencies/Inputs/InterestingType.swift

@@ -0,0 +1,23 @@
+protocol IntMaker {}
+extension IntMaker {
+  func make() -> Int { return 0 }
+}
+
+protocol DoubleMaker {}
+extension DoubleMaker {
+  func make() -> Double { return 0 }
+}
+
+
+#if OLD
+typealias InterestingType = Int
+typealias InterestingProto = IntMaker
+
+#elseif NEW
+typealias InterestingType = Double
+typealias InterestingProto = DoubleMaker
+
+#else
+typealias InterestingType = ErrorMustSetOLDOrNew
+
+#endif

test/Incremental/PrivateDependencies/private-function-fine.swift

@@ -0,0 +1,19 @@
+// REQUIRES: shell
+// Also uses awk:
+// XFAIL OS=windows
+
+// RUN: %target-swift-frontend -emit-silgen -primary-file %s %S/Inputs/InterestingType.swift -DOLD -emit-reference-dependencies-path %t.swiftdeps -module-name main -experimental-private-intransitive-dependencies | %FileCheck %s -check-prefix=CHECK-OLD
+// RUN: %S/../../Inputs/process_fine_grained_swiftdeps.sh <%t.swiftdeps >%t-processed.swiftdeps
+// RUN: %FileCheck -check-prefix=CHECK-DEPS %s < %t-processed.swiftdeps
+
+// RUN: %target-swift-frontend -emit-silgen -primary-file %s %S/Inputs/InterestingType.swift -DNEW -emit-reference-dependencies-path %t.swiftdeps -module-name main -experimental-private-intransitive-dependencies | %FileCheck %s -check-prefix=CHECK-NEW
+// RUN: %S/../../Inputs/process_fine_grained_swiftdeps.sh <%t.swiftdeps >%t-processed.swiftdeps
+// RUN: %FileCheck -check-prefix=CHECK-DEPS %s < %t-processed.swiftdeps
+
+private func testParamType(_: InterestingType) {}
+
+// CHECK-OLD: sil_global hidden @$s4main1x{{[^ ]+}} : ${{(@[a-zA-Z_]+ )?}}(Int) -> ()
+// CHECK-NEW: sil_global hidden @$s4main1x{{[^ ]+}} : ${{(@[a-zA-Z_]+ )?}}(Double) -> ()
+internal var x = testParamType
+
+// CHECK-DEPS: topLevel interface  '' InterestingType false

test/Incremental/PrivateDependencies/private-function-return-type-fine.swift

@@ -0,0 +1,19 @@
+// REQUIRES: shell
+// Also uses awk:
+// XFAIL OS=windows
+
+// RUN: %target-swift-frontend -emit-silgen -primary-file %s %S/Inputs/InterestingType.swift -DOLD -emit-reference-dependencies-path %t.swiftdeps -module-name main -experimental-private-intransitive-dependencies | %FileCheck %s -check-prefix=CHECK-OLD
+// RUN: %S/../../Inputs/process_fine_grained_swiftdeps.sh <%t.swiftdeps >%t-processed.swiftdeps
+// RUN: %FileCheck -check-prefix=CHECK-DEPS %s < %t-processed.swiftdeps
+
+// RUN: %target-swift-frontend -emit-silgen -primary-file %s %S/Inputs/InterestingType.swift -DNEW -emit-reference-dependencies-path %t.swiftdeps -module-name main -experimental-private-intransitive-dependencies | %FileCheck %s -check-prefix=CHECK-NEW
+// RUN: %S/../../Inputs/process_fine_grained_swiftdeps.sh <%t.swiftdeps >%t-processed.swiftdeps
+// RUN: %FileCheck -check-prefix=CHECK-DEPS %s < %t-processed.swiftdeps
+
+private func testReturnType() -> InterestingType { fatalError() }
+
+// CHECK-OLD: sil_global @$s4main1x{{[^ ]+}} : $Int
+// CHECK-NEW: sil_global @$s4main1x{{[^ ]+}} : $Double
+public var x = testReturnType() + 0
+
+// CHECK-DEPS: topLevel interface  '' InterestingType false