Add --depfile flag to inputs and multi-inputs tools.

This forces the load of depfile entries from either the Ninja log
or explicit depfiles to be performed when visiting the graph, thus
making them visible in the result.

- graph.h, graph.cc: Add optional ImplicitDepLoader* argument to
  InputsCollector constructor, and if provided, use it to load deps
  during the visit if needed.

- ninja.cc: Modify `inputs` and `multi-inputs` implementation to
  support a new `--depfile` flag, and create an ImplicitDepLoader
  instance when it is used to initialize the InputsCollector
  instances.

- output_test.py: Add regression tests + fix minor typing issues.
  Note in particular that rules are defined with and without
  `deps = gcc` as this triggers different code paths in Ninja
  in the deps loader.

Fixes #2618

Author: digit-google

doc/manual.asciidoc

@@ -299,7 +299,12 @@ executed in order, may be used to rebuild those targets, assuming that all
 output files are out of date.
 
 `inputs`:: given a list of targets, print a list of all inputs used to
-rebuild those targets.
+rebuild those targets. By default, this only includes static inputs declared
+in the Ninja build plan itself.
+
+Since Ninja 1.14, the `--depfile` flag can be used to include Ninja log / depfile
+inputs in the result. Note that if the build plan was regenerated since the last
+build, the log will contain stale entries, which may result in an incorrect output.
 _Available since Ninja 1.11._
 
 `multi-inputs`:: print one or more sets of inputs required to build targets.
@@ -341,6 +346,10 @@ target3 file3.c
 +
 _Note that a given input may appear for several targets if it is used by more
 than one targets._
+
+Like the `inputs` tool, this only reports static inputs from the build plan itself,
+but since Ninja 1.14, the `--depfile` flag can be used to also includes entries
+from the Ninja log or explicit depfiles.
 _Available since Ninja 1.13._
 
 `clean`:: remove built files. By default, it removes all built files

misc/output_test.py

@@ -58,7 +58,7 @@ def remove_non_visible_lines(raw_output: bytes) -> str:
 class BuildDir:
     def __init__(self, build_ninja: str):
         self.build_ninja = dedent(build_ninja)
-        self.d = None
+        self.d : None | tempfile.TemporaryDirectory[str] = None
 
     def __enter__(self):
         self.d = tempfile.TemporaryDirectory()
@@ -70,8 +70,16 @@ def __enter__(self):
     def __exit__(self, exc_type, exc_val, exc_tb):
         self.d.cleanup()
 
+    def write_file(self, path: str, content: str) -> None:
+        assert self.d
+        file_path = os.path.join(self.d.name, path)
+        os.makedirs(os.path.dirname(file_path), exist_ok=True)
+        with open(file_path, "wt") as f:
+            f.write(content)
+
     @property
     def path(self) -> str:
+        assert self.d
         return os.path.realpath(self.d.name)
 
 
@@ -111,13 +119,13 @@ def run(
         try:
             if pipe:
                 output = subprocess.check_output(
-                    [ninja_cmd], shell=True, cwd=self.d.name, env=env)
+                    [ninja_cmd], shell=True, cwd=self.path, env=env)
             elif platform.system() == 'Darwin':
                 output = subprocess.check_output(['script', '-q', '/dev/null', 'bash', '-c', ninja_cmd],
-                                                 cwd=self.d.name, env=env)
+                                                 cwd=self.path, env=env)
             else:
                 output = subprocess.check_output(['script', '-qfec', ninja_cmd, '/dev/null'],
-                                                 cwd=self.d.name, env=env)
+                                                 cwd=self.path, env=env)
         except subprocess.CalledProcessError as err:
             if print_err_output:
               sys.stdout.buffer.write(err.output)
@@ -366,8 +374,8 @@ def test_depfile_directory_creation(self) -> None:
             self.assertEqual(b.run('', pipe=True), dedent('''\
                 [1/1] touch somewhere/out && echo "somewhere/out: extra" > somewhere_else/out.d
                 '''))
-            self.assertTrue(os.path.isfile(os.path.join(b.d.name, "somewhere", "out")))
-            self.assertTrue(os.path.isfile(os.path.join(b.d.name, "somewhere_else", "out.d")))
+            self.assertTrue(os.path.isfile(os.path.join(b.path, "somewhere", "out")))
+            self.assertTrue(os.path.isfile(os.path.join(b.path, "somewhere_else", "out.d")))
 
     def test_status(self) -> None:
         self.assertEqual(run(''), 'ninja: no work to do.\n')
@@ -468,6 +476,39 @@ def test_tool_inputs(self) -> None:
           f'''in1\0out1\0out 2\0'''
         )
 
+    def test_tool_inputs_with_depfile_entries(self) -> None:
+        build_ninja = '''
+rule touch
+  command = touch $out && echo "$out: $extras" > $depfile
+  depfile = $out.d
+
+build out1: touch in1
+  deps = gcc
+  extras = extra extra2
+
+build out2: touch in2
+  extras = extra3 extra4
+  # No deps = gcc here intentionally
+'''
+        with BuildDir(build_ninja) as b:
+          # Do not run the build (empty deps log).
+          self.assertEqual(b.run(flags='-t inputs out1'), 'in1\n')
+          self.assertEqual(b.run(flags='-t inputs --depfile out1'), 'in1\n')
+
+          self.assertEqual(b.run(flags='-t inputs out2'), 'in2\n')
+          self.assertEqual(b.run(flags='-t inputs --depfile out2'), 'in2\n')
+
+          # Run the plan and create the log entry for 'out'
+          b.write_file("in1", "")
+          b.write_file("in2", "")
+          b.run('out1 out2')
+
+          # Check with a full deps log.
+          self.assertEqual(b.run(flags='-t inputs out1'), 'in1\n')
+          self.assertEqual(b.run(flags='-t inputs --depfile out1'), 'extra\nextra2\nin1\n')
+
+          self.assertEqual(b.run(flags='-t inputs out2'), 'in2\n')
+          self.assertEqual(b.run(flags='-t inputs --depfile out2'), 'extra3\nextra4\nin2\n')
 
     def test_tool_compdb_targets(self) -> None:
         plan = '''
@@ -556,6 +597,90 @@ def test_tool_multi_inputs(self) -> None:
         )
 
 
+    def test_tool_multi_inputs_with_depfile_entries(self) -> None:
+        build_ninja = '''
+rule touch1
+  command = touch $out && echo "$out: extra extra2" > $depfile
+  depfile = $out.d
+  deps = gcc
+
+rule touch2
+  command = touch $out && echo "$out: extra extra3" > $depfile
+  depfile = $out.d
+  # Do not use deps = gcc here intentionally
+
+build out1: touch1 in1
+build out2: touch2 in2
+'''
+
+        # Helper function to replace <TAB> with real tabs in input text.
+        def t(text):
+            if text[0] == '\n':  # skip initial newline if any.
+              text = text[1:]
+            return text.replace("<TAB>", "\t")
+
+        with BuildDir(build_ninja) as b:
+          # Run the plan and create the log entries for 'out1' and 'out2'
+          b.write_file("in1", "1")
+          b.write_file("in2", "2")
+          b.run('out1 out2')
+
+          self.assertEqual(b.run(flags='-t multi-inputs'), t('''
+out1<TAB>in1
+out2<TAB>in2
+'''))
+          self.assertEqual(b.run(flags='-t multi-inputs out1'), t('out1<TAB>in1\n'))
+          self.assertEqual(b.run(flags='-t multi-inputs out2'), t('out2<TAB>in2\n'))
+          self.assertEqual(b.run(flags='-t multi-inputs out1 out2'), t('''
+out1<TAB>in1
+out2<TAB>in2
+'''))
+          self.assertEqual(b.run(flags='-t multi-inputs out2 out1'), t('''
+out2<TAB>in2
+out1<TAB>in1
+'''))
+
+          self.assertEqual(b.run(flags='-t multi-inputs --depfile'), t('''
+out1<TAB>in1
+out1<TAB>extra
+out1<TAB>extra2
+out2<TAB>in2
+out2<TAB>extra
+out2<TAB>extra3
+'''))
+
+          self.assertEqual(b.run(flags='-t multi-inputs --depfile out1'), t('''
+out1<TAB>in1
+out1<TAB>extra
+out1<TAB>extra2
+'''))
+
+          self.assertEqual(b.run(flags='-t multi-inputs --depfile out2'), t('''
+out2<TAB>in2
+out2<TAB>extra
+out2<TAB>extra3
+'''))
+
+          self.assertEqual(b.run(flags='-t multi-inputs --depfile out1 out2'), t('''
+out1<TAB>in1
+out1<TAB>extra
+out1<TAB>extra2
+out2<TAB>in2
+out2<TAB>extra
+out2<TAB>extra3
+'''))
+
+          self.assertEqual(b.run(flags='-t multi-inputs --depfile out2 out1'), t('''
+out2<TAB>in2
+out2<TAB>extra
+out2<TAB>extra3
+out1<TAB>in1
+out1<TAB>extra
+out1<TAB>extra2
+'''))
+
+
+
     def test_explain_output(self):
         b = BuildDir('''\
             build .FORCE: phony

src/graph.cc

@@ -762,11 +762,26 @@ vector<Node*>::iterator ImplicitDepLoader::PreallocateSpace(Edge* edge,
 }
 
 void InputsCollector::VisitNode(const Node* node) {
-  const Edge* edge = node->in_edge();
+  Edge* edge = node->in_edge();
 
   if (!edge)  // A source file.
     return;
 
+  if (implicit_dep_loader_ && !edge->deps_loaded_) {
+    // Record that the deps were loaded in |deps_loaded_| as
+    // multiple visits to the same edge can be performed by
+    // repeated InputsCollector uses, as for the multi-inputs tool.
+    edge->deps_loaded_ = true;
+
+    // Ignore errors when loading depfile entries.
+    std::string err;
+    if (!implicit_dep_loader_->LoadDeps(edge, &err) && !err.empty()) {
+      // Print the error as a warning on stderr when an error occurred during
+      // the load.
+      Warning("%s", err.c_str());
+    }
+  }
+
   // Add inputs of the producing edge to the result,
   // except if they are themselves produced by a phony
   // edge.

src/graph.h

@@ -440,6 +440,12 @@ class EdgePriorityQueue:
 /// vector.
 ///
 struct InputsCollector {
+  /// Constructor allows passing a pointer to an optional ImplicitDepLoader
+  /// which will be used to load implicit dependencies from the Ninja log
+  /// or existing depfiles. Note that this will mutate the graph.
+  explicit InputsCollector(ImplicitDepLoader* implicit_dep_loader = nullptr)
+      : implicit_dep_loader_(implicit_dep_loader) {}
+
   /// Visit a single @arg node during this collection.
   void VisitNode(const Node* node);
 
@@ -459,6 +465,7 @@ struct InputsCollector {
   }
 
  private:
+  ImplicitDepLoader* implicit_dep_loader_ = nullptr;
   std::vector<const Node*> inputs_;
   std::set<const Node*> visited_nodes_;
 };