feat(run-task): implement shallow git clones

Shallow clones yield a massive improvement to clone performance, at the
expense of making it tricky to determine the files that were modified.

Author: ahal

src/taskgraph/run-task/run-task

@@ -571,13 +571,17 @@ def git_fetch(
     *targets: str,
     remote: str = "origin",
     tags: bool = False,
+    shallow: bool = False,
     env: Optional[dict[str, str]] = None,
 ):
     args = ["git", "fetch"]
     if tags:
         # `--force` is needed to be able to update an existing outdated tag.
         args.extend(["--tags", "--force"])
 
+    if shallow:
+        args.append("--depth=1")
+
     args.extend([remote, *set(targets)])
     retry_required_command(b"vcs", args, cwd=destination_path, extra_env=env)
 
@@ -643,6 +647,7 @@ def git_checkout(
     head_rev: Optional[str],
     ssh_key_file: Optional[Path],
     ssh_known_hosts_file: Optional[Path],
+    shallow: bool = False,
 ):
     assert head_ref or head_rev
 
@@ -688,10 +693,18 @@ def git_checkout(
         args = [
             "git",
             "clone",
-            base_repo if base_repo else head_repo,
-            destination_path,
         ]
 
+        if shallow:
+            args.extend(["--depth=1", "--no-checkout"])
+
+        args.extend(
+            [
+                base_repo if base_repo else head_repo,
+                destination_path,
+            ]
+        )
+
         retry_required_command(b"vcs", args, extra_env=env)
 
     # For Github based repos, base_rev often doesn't refer to an ancestor of
@@ -700,7 +713,7 @@ def git_checkout(
     # that consumers can compute the merge-base or files modified between the
     # two as needed.
     if base_rev and base_rev != NULL_REVISION:
-        git_fetch(destination_path, base_rev, env=env)
+        git_fetch(destination_path, base_rev, shallow=shallow, env=env)
 
     # If a head_ref was provided, it might be tag, so we need to make sure we fetch
     # those. This is explicitly only done when base and head repo match,
@@ -715,15 +728,17 @@ def git_checkout(
     targets = []
     if head_ref:
         targets.append(head_ref)
-    if not targets:
-        # If head_ref wasn't provided, we fallback to head_rev.
+    if not head_ref or (shallow and head_rev):
+        # If head_ref wasn't provided, we fallback to head_rev. If we have a
+        # shallow clone, head_rev needs to be fetched independently regardless.
         targets.append(head_rev)
 
     git_fetch(
         destination_path,
         *targets,
         remote=head_repo,
         tags=tags,
+        shallow=shallow,
         env=env,
     )
 
@@ -911,11 +926,17 @@ def add_vcs_arguments(parser, project, name):
         f"--{project}-sparse-profile",
         help=f"Path to sparse profile for {name} checkout",
     )
+    parser.add_argument(
+        f"--{project}-shallow-clone",
+        action="store_true",
+        help=f"Use shallow clone for {name}",
+    )
 
 
 def collect_vcs_options(args, project, name):
     checkout = getattr(args, f"{project}_checkout")
     sparse_profile = getattr(args, f"{project}_sparse_profile")
+    shallow_clone = getattr(args, f"{project}_shallow_clone")
 
     env_prefix = project.upper()
 
@@ -960,6 +981,7 @@ def collect_vcs_options(args, project, name):
         "repo-type": repo_type,
         "ssh-secret-name": private_key_secret,
         "pip-requirements": pip_requirements,
+        "shallow-clone": shallow_clone,
     }
 
 
@@ -1008,6 +1030,7 @@ def vcs_checkout_from_args(options):
                 head_rev,
                 ssh_key_file,
                 ssh_known_hosts_file,
+                shallow=options.get("shallow-clone", False),
             )
         elif options["repo-type"] == "hg":
             revision = hg_checkout(

test/test_scripts_run_task.py

@@ -152,9 +152,10 @@ def test_install_pip_requirements_with_uv(
 
 
 @pytest.mark.parametrize(
-    "env,extra_expected",
+    "args,env,extra_expected",
     [
         pytest.param(
+            {},
             {
                 "REPOSITORY_TYPE": "hg",
                 "BASE_REPOSITORY": "https://hg.mozilla.org/mozilla-central",
@@ -165,20 +166,38 @@ def test_install_pip_requirements_with_uv(
             {
                 "base-repo": "https://hg.mozilla.org/mozilla-unified",
             },
-        )
+            id="hg",
+        ),
+        pytest.param(
+            {"myrepo_shallow_clone": True},
+            {
+                "REPOSITORY_TYPE": "git",
+                "HEAD_REPOSITORY": "https://github.com/test/repo.git",
+                "HEAD_REV": "abc123",
+            },
+            {"shallow-clone": True},
+            id="git_with_shallow_clone",
+        ),
     ],
 )
-def test_collect_vcs_options(monkeypatch, run_task_mod, env, extra_expected):
+def test_collect_vcs_options(
+    monkeypatch,
+    run_task_mod,
+    args,
+    env,
+    extra_expected,
+):
     name = "myrepo"
     checkout = "checkout"
 
     monkeypatch.setattr(os, "environ", {})
     for k, v in env.items():
         monkeypatch.setenv(f"{name.upper()}_{k.upper()}", v)
 
-    args = Namespace()
-    setattr(args, f"{name}_checkout", checkout)
-    setattr(args, f"{name}_sparse_profile", False)
+    args.setdefault(f"{name}_checkout", checkout)
+    args.setdefault(f"{name}_shallow_clone", False)
+    args.setdefault(f"{name}_sparse_profile", False)
+    args = Namespace(**args)
 
     result = run_task_mod.collect_vcs_options(args, name, name)
 
@@ -194,6 +213,7 @@ def test_collect_vcs_options(monkeypatch, run_task_mod, env, extra_expected):
         "head-ref": env.get("HEAD_REF"),
         "head-rev": env.get("HEAD_REV"),
         "repo-type": env.get("REPOSITORY_TYPE"),
+        "shallow-clone": False,
         "ssh-secret-name": env.get("SSH_SECRET_NAME"),
         "sparse-profile": False,
         "store-path": env.get("HG_STORE_PATH"),
@@ -456,6 +476,104 @@ def test_git_checkout_with_commit(
     assert current_rev == head_rev
 
 
+def test_git_checkout_shallow(
+    mock_stdin,
+    run_task_mod,
+    mock_git_repo,
+    tmp_path,
+):
+    destination = tmp_path / "destination"
+
+    # Git ignores `--depth` when cloning from local directories, so use file://
+    # protocol to force shallow clone.
+    repo_url = f"file://{mock_git_repo['path']}"
+    base_rev = mock_git_repo["main"][-1]
+    head_rev = mock_git_repo["branch"][-1]
+
+    # Use shallow clone with head_ref != head_rev
+    run_task_mod.git_checkout(
+        destination_path=str(destination),
+        head_repo=repo_url,
+        base_repo=repo_url,
+        base_rev=base_rev,
+        head_ref="mybranch",
+        head_rev=head_rev,
+        ssh_key_file=None,
+        ssh_known_hosts_file=None,
+        shallow=True,
+    )
+    shallow_file = destination / ".git" / "shallow"
+    assert shallow_file.exists()
+
+    # Verify we're on the correct commit
+    final_rev = subprocess.check_output(
+        ["git", "rev-parse", "HEAD"],
+        cwd=str(destination),
+        universal_newlines=True,
+    ).strip()
+    assert final_rev == head_rev
+
+    # Verify both base_rev and head_rev are available.
+    for sha in (base_rev, head_rev):
+        result = subprocess.run(
+            ["git", "cat-file", "-t", sha],
+            cwd=str(destination),
+            capture_output=True,
+            text=True,
+        )
+        assert result.returncode == 0, f"Commit {sha} should be available"
+        assert result.stdout.strip() == "commit"
+
+
+def test_git_fetch_shallow(
+    mock_stdin,
+    run_task_mod,
+    mock_git_repo,
+    tmp_path,
+):
+    destination = tmp_path / "destination"
+
+    # Git ignores `--depth` when cloning from local directories, so use file://
+    # protocol to force shallow clone.
+    repo_url = f"file://{mock_git_repo['path']}"
+
+    run_task_mod.run_command(
+        b"vcs",
+        [
+            "git",
+            "clone",
+            "--depth=1",
+            "--no-checkout",
+            repo_url,
+            str(destination),
+        ],
+    )
+    shallow_file = destination / ".git" / "shallow"
+    assert shallow_file.exists()
+
+    # Verify base_rev doesn't exist yet
+    base_rev = mock_git_repo["branch"][-1]
+    result = subprocess.run(
+        ["git", "cat-file", "-t", base_rev],
+        cwd=str(destination),
+        capture_output=True,
+        text=True,
+    )
+    assert result.returncode != 0
+
+    run_task_mod.git_fetch(str(destination), base_rev, remote=repo_url, shallow=True)
+
+    # Verify base_rev is now available
+    result = subprocess.run(
+        ["git", "cat-file", "-t", base_rev],
+        cwd=str(destination),
+        capture_output=True,
+        text=True,
+    )
+    assert result.returncode == 0
+    assert result.stdout.strip() == "commit"
+
+
 def test_display_python_version_should_output_python_versions_title(
     run_task_mod, capsys
 ):