perf: only consider tracked files when building toolchain hashes

Issue: #663

Author: ahal

src/taskgraph/util/hash.py

@@ -4,9 +4,10 @@
 
 import functools
 import hashlib
-from pathlib import Path
+import os
 
 from taskgraph.util import path as mozpath
+from taskgraph.util.vcs import get_repository
 
 
 @functools.lru_cache(maxsize=None)
@@ -52,8 +53,5 @@ def _find_matching_files(base_path, pattern):
 
 @functools.lru_cache(maxsize=None)
 def _get_all_files(base_path):
-    return [
-        mozpath.normsep(str(path))
-        for path in Path(base_path).rglob("*")
-        if path.is_file()
-    ]
+    repo = get_repository(os.getcwd())
+    return repo.get_tracked_files(base_path)

src/taskgraph/util/vcs.py

@@ -69,7 +69,7 @@ def base_rev(self) -> str:
 
     @property
     @abstractmethod
-    def branch(self) -> str | None:
+    def branch(self) -> Optional[str]:
         """Current branch or bookmark the checkout has active."""
 
     @property
@@ -125,6 +125,15 @@ def get_url(self, remote: Optional[str]) -> str:
     def get_commit_message(self, revision: Optional[str]) -> str:
         """Commit message of specified revision or current commit."""
 
+    @abstractmethod
+    def get_tracked_files(self, *paths: str, rev: Optional[str] = None) -> List[str]:
+        """Return list of tracked files.
+
+        ``*paths`` are path specifiers to limit results to.
+        ``rev`` is a revision specifier at which to retrieve the files.
+        Defaults to the parent of the working copy if unspecified.
+        """
+
     @abstractmethod
     def get_changed_files(
         self,
@@ -280,6 +289,10 @@ def _files_template(self, diff_filter):
             template += "{file_mods % '{file}\\n'}"
         return template
 
+    def get_tracked_files(self, *paths, rev=None):
+        rev = rev or "."
+        return self.run("files", "-r", rev, *paths).splitlines()
+
     def get_changed_files(self, diff_filter=None, mode=None, rev=None, base_rev=None):
         diff_filter = diff_filter or "ADM"
         if rev is None:
@@ -462,6 +475,10 @@ def get_commit_message(self, revision=None):
         revision = revision or "HEAD"
         return self.run("log", "-n1", "--format=%B", revision)
 
+    def get_tracked_files(self, *paths, rev=None):
+        rev = rev or "HEAD"
+        return self.run("ls-tree", "-r", "--name-only", rev, *paths).splitlines()
+
     def get_changed_files(self, diff_filter=None, mode=None, rev=None, base_rev=None):
         diff_filter = diff_filter or "ADM"
         mode = mode or "unstaged"

test/test_util_vcs.py

@@ -4,6 +4,7 @@
 
 import os
 import subprocess
+from pathlib import Path
 from textwrap import dedent
 
 import pytest
@@ -211,7 +212,31 @@ def test_default_branch_cloned_metadata(tmpdir, default_git_branch, repo):
 
 
 def assert_files(actual, expected):
-    assert set(map(os.path.basename, actual)) == set(expected)
+    assert set(actual) == set(expected)
+
+
+def test_get_tracked_files(repo):
+    assert_files(repo.get_tracked_files(), ["first_file"])
+
+    second_file = Path(repo.path) / "subdir" / "second_file"
+    second_file.parent.mkdir()
+    second_file.write_text("foo")
+    assert_files(repo.get_tracked_files(), ["first_file"])
+
+    repo.run("add", str(second_file))
+    assert_files(repo.get_tracked_files(), ["first_file"])
+
+    repo.run("commit", "-m", "Add second file")
+    rev = ".~1" if repo.tool == "hg" else "HEAD~1"
+    assert_files(repo.get_tracked_files(), ["first_file", "subdir/second_file"])
+    assert_files(repo.get_tracked_files("subdir"), ["subdir/second_file"])
+    assert_files(repo.get_tracked_files(rev=rev), ["first_file"])
+
+    if repo.tool == "git":
+        assert_files(repo.get_tracked_files("subdir", rev=rev), [])
+    elif repo.tool == "hg":
+        with pytest.raises(subprocess.CalledProcessError):
+            repo.get_tracked_files("subdir", rev=rev)
 
 
 def test_get_changed_files_no_changes(repo):