Merge pull request swiftlang#28702 from davidungar/only-one-dependency-file

[Incremental, Driver] Only emit a real make-style dependency file for one frontend job.

Author: David Ungar

include/swift/Driver/Compilation.h

@@ -257,6 +257,21 @@ class Compilation {
   /// limit filelists will be used.
   size_t FilelistThreshold;
 
+  /// Because each frontend job outputs the same info in its .d file, only do it
+  /// on the first job that actually runs. Write out dummies for the rest of the
+  /// jobs. This hack saves a lot of time in the build system when incrementally
+  /// building a project with many files. Record if a scheduled job has already
+  /// added -emit-dependency-path.
+  bool HaveAlreadyAddedDependencyPath = false;
+
+  /// When set, only the first scheduled frontend job gets the argument needed
+  /// to produce a make-style dependency file. The other jobs create dummy files
+  /// in the driver. This hack speeds up incremental compilation by reducing the
+  /// time for the build system to read each dependency file, which are all
+  /// identical. This optimization can be disabled by passing
+  /// -disable-only-one-dependency-file on the command line.
+  const bool OnlyOneDependencyFile;
+
   /// Scaffolding to permit experimentation with finer-grained dependencies and
   /// faster rebuilds.
   const bool EnableFineGrainedDependencies;
@@ -309,6 +324,7 @@ class Compilation {
               bool SaveTemps = false,
               bool ShowDriverTimeCompilation = false,
               std::unique_ptr<UnifiedStatsReporter> Stats = nullptr,
+              bool OnlyOneDependencyFile = false,
               bool EnableFineGrainedDependencies = false,
               bool VerifyFineGrainedDependencyGraphAfterEveryImport = false,
               bool EmitFineGrainedDependencyDotFileAfterEveryImport = false,
@@ -427,6 +443,14 @@ class Compilation {
     return FilelistThreshold;
   }
 
+  /// Called to decide whether to add a dependency path argument, or whether to
+  /// create a dummy file. Responds by invoking one of the two passed-in
+  /// functions.
+  void addDependencyPathOrCreateDummy(
+      const CommandOutput &Output,
+      function_ref<void(StringRef)> addDependencyPath,
+      function_ref<void(StringRef)> createDummy);
+
   UnifiedStatsReporter *getStatsReporter() const {
     return Stats.get();
   }

include/swift/Option/Options.td

@@ -139,6 +139,11 @@ def enable_fine_grained_dependencies :
 Flag<["-"], "enable-fine-grained-dependencies">, Flags<[FrontendOption, HelpHidden]>,
 HelpText<"Experimental work-in-progress to be more selective about incremental recompilation">;
 
+
+def disable_only_one_dependency_file :
+Flag<["-"], "disable-only-one-dependency-file">, Flags<[DoesNotAffectIncrementalBuild]>,
+ HelpText<"Disables incremental build optimization that only produces one dependencies file">;
+
 def enable_source_range_dependencies :
 Flag<["-"], "enable-source-range-dependencies">, Flags<[]>,
 HelpText<"Try using source range information">;

lib/Driver/Compilation.cpp

@@ -121,13 +121,15 @@ Compilation::Compilation(DiagnosticEngine &Diags,
                          bool SaveTemps,
                          bool ShowDriverTimeCompilation,
                          std::unique_ptr<UnifiedStatsReporter> StatsReporter,
+                         bool OnlyOneDependencyFile,
                          bool EnableFineGrainedDependencies,
                          bool VerifyFineGrainedDependencyGraphAfterEveryImport,
                          bool EmitFineGrainedDependencyDotFileAfterEveryImport,
                          bool FineGrainedDependenciesIncludeIntrafileOnes,
                          bool EnableSourceRangeDependencies,
                          bool CompareIncrementalSchemes,
-                         StringRef CompareIncrementalSchemesPath)
+                         StringRef CompareIncrementalSchemesPath
+                         )
   : Diags(Diags), TheToolChain(TC),
     TheOutputInfo(OI),
     Level(Level),
@@ -149,14 +151,16 @@ Compilation::Compilation(DiagnosticEngine &Diags,
     ShowDriverTimeCompilation(ShowDriverTimeCompilation),
     Stats(std::move(StatsReporter)),
     FilelistThreshold(FilelistThreshold),
+    OnlyOneDependencyFile(OnlyOneDependencyFile),
     EnableFineGrainedDependencies(EnableFineGrainedDependencies),
     VerifyFineGrainedDependencyGraphAfterEveryImport(
       VerifyFineGrainedDependencyGraphAfterEveryImport),
     EmitFineGrainedDependencyDotFileAfterEveryImport(
       EmitFineGrainedDependencyDotFileAfterEveryImport),
     FineGrainedDependenciesIncludeIntrafileOnes(
       FineGrainedDependenciesIncludeIntrafileOnes),
-    EnableSourceRangeDependencies(EnableSourceRangeDependencies) {
+    EnableSourceRangeDependencies(EnableSourceRangeDependencies)
+    {
     if (CompareIncrementalSchemes)
       IncrementalComparator.emplace(
       // Ensure the references are to inst vars, NOT arguments
@@ -2032,3 +2036,19 @@ unsigned Compilation::countSwiftInputs() const {
       ++inputCount;
   return inputCount;
 }
+
+void Compilation::addDependencyPathOrCreateDummy(
+    const CommandOutput &Output,
+    function_ref<void(StringRef)> addDependencyPath,
+    function_ref<void(StringRef)> createDummy) {
+  StringRef path =
+      Output.getAdditionalOutputForType(file_types::TY_Dependencies);
+  if (path.empty())
+    return;
+  if (HaveAlreadyAddedDependencyPath && OnlyOneDependencyFile)
+    createDummy(path);
+  else {
+    addDependencyPath(path);
+    HaveAlreadyAddedDependencyPath = true;
+  }
+}

lib/Driver/Driver.cpp

@@ -951,7 +951,11 @@ Driver::buildCompilation(const ToolChain &TC,
         ArgList->hasArg(options::OPT_driver_time_compilation);
     std::unique_ptr<UnifiedStatsReporter> StatsReporter =
         createStatsReporter(ArgList.get(), Inputs, OI, DefaultTargetTriple);
+
+    const bool OnlyOneDependencyFile =
+        !ArgList->hasArg(options::OPT_disable_only_one_dependency_file);
     // relies on the new dependency graph
+
     const bool EnableFineGrainedDependencies =
         ArgList->hasArg(options::OPT_enable_fine_grained_dependencies);
 
@@ -984,6 +988,7 @@ Driver::buildCompilation(const ToolChain &TC,
         SaveTemps,
         ShowDriverTimeCompilation,
         std::move(StatsReporter),
+        OnlyOneDependencyFile,
         EnableFineGrainedDependencies,
         VerifyFineGrainedDependencyGraphAfterEveryImport,
         EmitFineGrainedDependencyDotFileAfterEveryImport,

lib/Driver/ToolChains.cpp

@@ -34,6 +34,8 @@
 #include "llvm/Support/Process.h"
 #include "llvm/Support/Program.h"
 
+#include <fstream>
+
 using namespace swift;
 using namespace swift::driver;
 using namespace llvm::opt;
@@ -671,8 +673,17 @@ void ToolChain::JobContext::addFrontendSupplementaryOutputArguments(
            "mode!");
   }
 
-  addOutputsOfType(arguments, Output, Args, file_types::TY_Dependencies,
-                   "-emit-dependencies-path");
+  C.addDependencyPathOrCreateDummy(
+      Output,
+      [&](StringRef) {
+        addOutputsOfType(arguments, Output, Args, file_types::TY_Dependencies,
+                         "-emit-dependencies-path");
+      },
+      [&](StringRef dependencyFile) {
+        // Create an empty file
+        std::ofstream(dependencyFile.str().c_str());
+      });
+
   addOutputsOfType(arguments, Output, Args, file_types::TY_SwiftDeps,
                    "-emit-reference-dependencies-path");
   addOutputsOfType(arguments, Output, Args, file_types::TY_SwiftRanges,