Add --depfile flag to inputs and multi-inputs tools.

This forces the load of depfile entries from either the Ninja log
or explicit depfiles to be performed when visiting the graph, thus
making them visible in the result.

- graph.h, graph.cc: Add optional ImplicitDepLoader* argument to
  InputsCollector constructor, and if provided, use it to load deps
  during the visit if needed.

- ninja.cc: Modify `inputs` and `multi-inputs` implementation to
  support a new `--depfile` or `-D` flag, and create an
  ImplicitDepLoader instance when it is used to initialize the
  InputsCollector instances.

  NOTE: using std::make_unique<> fails on our MacOS CI builder, which
  still uses -std=gnu+11 for some unknown reason, so this uses
  `std::unique_ptr<T>::reset(new T(...))` instead :-/

- output_test.py: Add regression tests + fix minor typing issues.

Fixes #2618

Author: digit-google

doc/manual.asciidoc

@@ -299,7 +299,12 @@ executed in order, may be used to rebuild those targets, assuming that all
 output files are out of date.
 
 `inputs`:: given a list of targets, print a list of all inputs used to
-rebuild those targets.
+rebuild those targets. By default, this only includes static inputs declared
+in the Ninja build plan itself.
+
+Since Ninja 1.14, the `--depfile` flag can be used to include Ninja log / depfile
+inputs in the result. Note that if the build plan was regenerated since the last
+build, the log will contain stale entries, which may result in an incorrect output.
 _Available since Ninja 1.11._
 
 `multi-inputs`:: print one or more sets of inputs required to build targets.
@@ -341,6 +346,10 @@ target3 file3.c
 +
 _Note that a given input may appear for several targets if it is used by more
 than one targets._
+
+Like the `inputs` tool, this only reports static inputs from the build plan itself,
+but since Ninja 1.14, the `--depfile` flag can be used to also includes entries
+from the Ninja log or explicit depfiles.
 _Available since Ninja 1.13._
 
 `clean`:: remove built files. By default, it removes all built files

misc/output_test.py

@@ -58,7 +58,7 @@ def remove_non_visible_lines(raw_output: bytes) -> str:
 class BuildDir:
     def __init__(self, build_ninja: str):
         self.build_ninja = dedent(build_ninja)
-        self.d = None
+        self.d : None | tempfile.TemporaryDirectory[str] = None
 
     def __enter__(self):
         self.d = tempfile.TemporaryDirectory()
@@ -72,6 +72,7 @@ def __exit__(self, exc_type, exc_val, exc_tb):
 
     @property
     def path(self) -> str:
+        assert self.d
         return os.path.realpath(self.d.name)
 
 
@@ -111,13 +112,13 @@ def run(
         try:
             if pipe:
                 output = subprocess.check_output(
-                    [ninja_cmd], shell=True, cwd=self.d.name, env=env)
+                    [ninja_cmd], shell=True, cwd=self.path, env=env)
             elif platform.system() == 'Darwin':
                 output = subprocess.check_output(['script', '-q', '/dev/null', 'bash', '-c', ninja_cmd],
-                                                 cwd=self.d.name, env=env)
+                                                 cwd=self.path, env=env)
             else:
                 output = subprocess.check_output(['script', '-qfec', ninja_cmd, '/dev/null'],
-                                                 cwd=self.d.name, env=env)
+                                                 cwd=self.path, env=env)
         except subprocess.CalledProcessError as err:
             if print_err_output:
               sys.stdout.buffer.write(err.output)
@@ -366,8 +367,8 @@ def test_depfile_directory_creation(self) -> None:
             self.assertEqual(b.run('', pipe=True), dedent('''\
                 [1/1] touch somewhere/out && echo "somewhere/out: extra" > somewhere_else/out.d
                 '''))
-            self.assertTrue(os.path.isfile(os.path.join(b.d.name, "somewhere", "out")))
-            self.assertTrue(os.path.isfile(os.path.join(b.d.name, "somewhere_else", "out.d")))
+            self.assertTrue(os.path.isfile(os.path.join(b.path, "somewhere", "out")))
+            self.assertTrue(os.path.isfile(os.path.join(b.path, "somewhere_else", "out.d")))
 
     def test_status(self) -> None:
         self.assertEqual(run(''), 'ninja: no work to do.\n')
@@ -468,6 +469,19 @@ def test_tool_inputs(self) -> None:
           f'''in1\0out1\0out 2\0'''
         )
 
+    def test_tool_inputs_with_depfile_entries(self) -> None:
+        build_ninja = '''
+rule touch
+  command = touch $out && echo "$out: extra extra2" > $depfile
+build out: touch
+  depfile = out.d
+'''
+        with BuildDir(build_ninja) as b:
+          # Run the plan and create the log entry for 'out'
+          b.run('out')
+          self.assertEqual(b.run(flags='-t inputs out'), '')
+          self.assertEqual(b.run(flags='-t inputs --depfile out'), 'extra\nextra2\n')
+
 
     def test_tool_compdb_targets(self) -> None:
         plan = '''
@@ -556,6 +570,55 @@ def test_tool_multi_inputs(self) -> None:
         )
 
 
+    def test_tool_multi_inputs_with_depfile_entries(self) -> None:
+        build_ninja = '''
+rule touch1
+  command = touch $out && echo "$out: extra extra2" > $depfile
+rule touch2
+  command = touch $out && echo "$out: extra extra3" > $depfile
+build out1: touch1
+  depfile = out1.d
+build out2: touch2
+  depfile = out2.d
+'''
+        with BuildDir(build_ninja) as b:
+          # Run the plan and create the log entries for 'out1' and 'out2'
+          b.run('out1 out2')
+
+          self.assertEqual(b.run(flags='-t multi-inputs'), '')
+          self.assertEqual(b.run(flags='-t multi-inputs out1'), '')
+          self.assertEqual(b.run(flags='-t multi-inputs out2'), '')
+          self.assertEqual(b.run(flags='-t multi-inputs out1 out2'), '')
+          self.assertEqual(b.run(flags='-t multi-inputs out2 out1'), '')
+
+          self.assertEqual(b.run(flags='-t multi-inputs --depfile'), '''out1<TAB>extra
+out1<TAB>extra2
+out2<TAB>extra
+out2<TAB>extra3
+'''.replace('<TAB>', '\t'))
+
+          self.assertEqual(b.run(flags='-t multi-inputs --depfile out1'), '''out1<TAB>extra
+out1<TAB>extra2
+'''.replace('<TAB>', '\t'))
+
+          self.assertEqual(b.run(flags='-t multi-inputs --depfile out2'), '''out2<TAB>extra
+out2<TAB>extra3
+'''.replace('<TAB>', '\t'))
+
+          self.assertEqual(b.run(flags='-t multi-inputs --depfile out1 out2'), '''out1<TAB>extra
+out1<TAB>extra2
+out2<TAB>extra
+out2<TAB>extra3
+'''.replace('<TAB>', '\t'))
+
+          self.assertEqual(b.run(flags='-t multi-inputs --depfile out2 out1'), '''out2<TAB>extra
+out2<TAB>extra3
+out1<TAB>extra
+out1<TAB>extra2
+'''.replace('<TAB>', '\t'))
+
+
+
     def test_explain_output(self):
         b = BuildDir('''\
             build .FORCE: phony

src/graph.cc

@@ -762,11 +762,26 @@ vector<Node*>::iterator ImplicitDepLoader::PreallocateSpace(Edge* edge,
 }
 
 void InputsCollector::VisitNode(const Node* node) {
-  const Edge* edge = node->in_edge();
+  Edge* edge = node->in_edge();
 
   if (!edge)  // A source file.
     return;
 
+  if (implicit_dep_loader_ && !edge->deps_loaded_) {
+    // Record that the deps were loaded in |deps_loaded_| as
+    // multiple visits to the same edge can be performed by
+    // repeated InputsCollector uses, as for the multi-inputs tool.
+    edge->deps_loaded_ = true;
+
+    // Ignore errors when loading depfile entries.
+    std::string err;
+    if (!implicit_dep_loader_->LoadDeps(edge, &err)) {
+      // Print the error as a warning on stderr when an error occurred during
+      // the load.
+      Warning("%s", err.c_str());
+    }
+  }
+
   // Add inputs of the producing edge to the result,
   // except if they are themselves produced by a phony
   // edge.

src/graph.h

@@ -440,6 +440,12 @@ class EdgePriorityQueue:
 /// vector.
 ///
 struct InputsCollector {
+  /// Constructor allows passing a pointer to an optional ImplicitDepLoader
+  /// which will be used to load implicit dependencies from the Ninja log
+  /// or existing depfiles. Note that this will mutate the graph.
+  explicit InputsCollector(ImplicitDepLoader* implicit_dep_loader = nullptr)
+      : implicit_dep_loader_(implicit_dep_loader) {}
+
   /// Visit a single @arg node during this collection.
   void VisitNode(const Node* node);
 
@@ -459,6 +465,7 @@ struct InputsCollector {
   }
 
  private:
+  ImplicitDepLoader* implicit_dep_loader_ = nullptr;
   std::vector<const Node*> inputs_;
   std::set<const Node*> visited_nodes_;
 };

src/ninja.cc

@@ -784,14 +784,20 @@ int NinjaMain::ToolInputs(const Options* options, int argc, char* argv[]) {
 
   optind = 1;
   int opt;
+  bool include_depfile_inputs = false;
+  enum { OPT_DEPFILE = 1 };
   const option kLongOptions[] = { { "help", no_argument, NULL, 'h' },
                                   { "no-shell-escape", no_argument, NULL, 'E' },
                                   { "print0", no_argument, NULL, '0' },
+                                  { "depfile", no_argument, NULL, OPT_DEPFILE },
                                   { "dependency-order", no_argument, NULL,
                                     'd' },
                                   { NULL, 0, NULL, 0 } };
   while ((opt = getopt_long(argc, argv, "h0Ed", kLongOptions, NULL)) != -1) {
     switch (opt) {
+    case OPT_DEPFILE:
+      include_depfile_inputs = true;
+      break;
     case 'd':
       dependency_order = true;
       break;
@@ -815,6 +821,7 @@ int NinjaMain::ToolInputs(const Options* options, int argc, char* argv[]) {
 "  -0, --print0            Use \\0, instead of \\n as a line terminator.\n"
 "  -E, --no-shell-escape   Do not shell escape the result.\n"
 "  -d, --dependency-order  Sort results by dependency order.\n"
+"  --depfile               Include depfile inputs in the result.\n"
       );
       // clang-format on
       return 1;
@@ -830,7 +837,13 @@ int NinjaMain::ToolInputs(const Options* options, int argc, char* argv[]) {
     return 1;
   }
 
-  InputsCollector collector;
+  std::unique_ptr<ImplicitDepLoader> implicit_dep_loader;
+  if (include_depfile_inputs) {
+    implicit_dep_loader.reset(new ImplicitDepLoader(
+        &state_, &deps_log_, &disk_interface_, nullptr, nullptr));
+  }
+
+  InputsCollector collector(implicit_dep_loader.get());
   for (const Node* node : nodes)
     collector.VisitNode(node);
 
@@ -861,13 +874,18 @@ int NinjaMain::ToolMultiInputs(const Options* options, int argc, char* argv[]) {
   int opt;
   char terminator = '\n';
   const char* delimiter = "\t";
+  bool include_depfile_inputs = false;
+  enum { OPT_DEPFILE = 1 };
   const option kLongOptions[] = { { "help", no_argument, NULL, 'h' },
-                                  { "delimiter", required_argument, NULL,
-                                    'd' },
+                                  { "delimiter", required_argument, NULL, 'd' },
+                                  { "depfile", no_argument, NULL, OPT_DEPFILE },
                                   { "print0", no_argument, NULL, '0' },
                                   { NULL, 0, NULL, 0 } };
   while ((opt = getopt_long(argc, argv, "d:h0", kLongOptions, NULL)) != -1) {
     switch (opt) {
+    case OPT_DEPFILE:
+      include_depfile_inputs = true;
+      break;
     case 'd':
       delimiter = optarg;
       break;
@@ -888,6 +906,7 @@ int NinjaMain::ToolMultiInputs(const Options* options, int argc, char* argv[]) {
 "Options:\n"
 "  -h, --help                   Print this message.\n"
 "  -d  --delimiter=DELIM        Use DELIM instead of TAB for field delimiter.\n"
+"  --depfile                    Include depfile inputs in the result.\n"
 "  -0, --print0                 Use \\0, instead of \\n as a line terminator.\n"
       );
       // clang-format on
@@ -904,8 +923,14 @@ int NinjaMain::ToolMultiInputs(const Options* options, int argc, char* argv[]) {
     return 1;
   }
 
+  std::unique_ptr<ImplicitDepLoader> implicit_dep_loader;
+  if (include_depfile_inputs) {
+    implicit_dep_loader.reset(new ImplicitDepLoader(
+        &state_, &deps_log_, &disk_interface_, nullptr, nullptr));
+  }
+
   for (const Node* node : nodes) {
-    InputsCollector collector;
+    InputsCollector collector(implicit_dep_loader.get());
 
     collector.VisitNode(node);
     std::vector<std::string> inputs = collector.GetInputsAsStrings();