feat: add a cli option o force using locally changed files

This is a port of:
https://hg.mozilla.org/mozilla-central/rev/0c6f696169f0

Author: ahal

src/taskgraph/main.py

@@ -121,7 +121,7 @@ def get_taskgraph_generator(root, parameters):
     return TaskGraphGenerator(root_dir=root, parameters=parameters)
 
 
-def format_taskgraph(options, parameters, logfile=None):
+def format_taskgraph(options, parameters, overrides, logfile=None):
     import taskgraph
     from taskgraph.parameters import parameters_loader
 
@@ -139,7 +139,7 @@ def format_taskgraph(options, parameters, logfile=None):
     if isinstance(parameters, str):
         parameters = parameters_loader(
             parameters,
-            overrides={"target-kinds": options.get("target_kinds")},
+            overrides=overrides,
             strict=False,
         )
 
@@ -173,7 +173,7 @@ def dump_output(out, path=None, params_spec=None):
     print(out + "\n", file=fh)
 
 
-def generate_taskgraph(options, parameters, logdir):
+def generate_taskgraph(options, parameters, overrides, logdir):
     from taskgraph.parameters import Parameters
 
     def logfile(spec):
@@ -189,14 +189,16 @@ def logfile(spec):
     # tracebacks a little more readable and avoids additional process overhead.
     if len(parameters) == 1:
         spec = parameters[0]
-        out = format_taskgraph(options, spec, logfile(spec))
+        out = format_taskgraph(options, spec, overrides, logfile(spec))
         dump_output(out, options["output_file"])
         return 0
 
     futures = {}
     with ProcessPoolExecutor(max_workers=options["max_workers"]) as executor:
         for spec in parameters:
-            f = executor.submit(format_taskgraph, options, spec, logfile(spec))
+            f = executor.submit(
+                format_taskgraph, options, spec, overrides, logfile(spec)
+            )
             futures[f] = spec
 
     returncode = 0
@@ -293,6 +295,15 @@ def logfile(spec):
     "generations will happen from the same invocation (one per parameters "
     "specified).",
 )
+@argument(
+    "--force-local-files-changed",
+    default=False,
+    action="store_true",
+    help="Compute the 'files-changed' parameter from local version control, "
+    "even when explicitly using a parameter set that already has it defined. "
+    "Note that this is already the default behaviour when no parameters are "
+    "specified.",
+)
 @argument(
     "--no-optimize",
     dest="optimize",
@@ -367,9 +378,11 @@ def show_taskgraph(options):
     diffdir = None
     output_file = options["output_file"]
 
-    if options["diff"]:
+    if options["diff"] or options["force_local_files_changed"]:
         repo = get_repository(os.getcwd())
 
+    if options["diff"]:
+        assert repo is not None
         if not repo.working_directory_clean():
             print(
                 "abort: can't diff taskgraph with dirty working directory",
@@ -393,15 +406,22 @@ def show_taskgraph(options):
         )
         print(f"Generating {options['graph_attr']} @ {cur_rev}", file=sys.stderr)
 
+    overrides = {
+        "target-kinds": options.get("target_kinds"),
+    }
     parameters: List[Any[str, Parameters]] = options.pop("parameters")
     if not parameters:
-        overrides = {
-            "target-kinds": options.get("target_kinds"),
-        }
         parameters = [
             parameters_loader(None, strict=False, overrides=overrides)
         ]  # will use default values
 
+        # This is the default behaviour anyway, so no need to re-compute.
+        options["force_local_files_changed"] = False
+
+    elif options["force_local_files_changed"]:
+        assert repo is not None
+        overrides["files-changed"] = sorted(repo.get_changed_files("AM"))
+
     for param in parameters[:]:
         if isinstance(param, str) and os.path.isdir(param):
             parameters.remove(param)
@@ -427,7 +447,7 @@ def show_taskgraph(options):
         # to setup its `mach` based logging.
         setup_logging()
 
-    ret = generate_taskgraph(options, parameters, logdir)
+    ret = generate_taskgraph(options, parameters, overrides, logdir)
 
     if options["diff"]:
         assert diffdir is not None
@@ -451,7 +471,7 @@ def show_taskgraph(options):
                 diffdir, f"{options['graph_attr']}_{base_rev_file}"
             )
             print(f"Generating {options['graph_attr']} @ {base_rev}", file=sys.stderr)
-            ret |= generate_taskgraph(options, parameters, logdir)
+            ret |= generate_taskgraph(options, parameters, overrides, logdir)
         finally:
             repo.update(cur_rev)
 

test/test_main.py

@@ -2,6 +2,7 @@
 # http://creativecommons.org/publicdomain/zero/1.0/
 
 import os
+import sys
 from pathlib import Path
 from textwrap import dedent
 
@@ -48,7 +49,7 @@ def fake_get_taskgraph_generator(*args):
         ("morphed", ["_fake-t-0", "_fake-t-1"]),
     ),
 )
-def test_show_taskgraph(run_taskgraph, capsys, attr, expected):
+def test_show_taskgraph_attr(run_taskgraph, capsys, attr, expected):
     res = run_taskgraph([attr])
     assert res == 0
 
@@ -72,6 +73,34 @@ def test_show_taskgraph_parallel(run_taskgraph):
     assert res == 1
 
 
+def test_show_taskgraph_force_local_files_changed(mocker, run_taskgraph):
+    repo = mocker.MagicMock()
+    repo.get_changed_files.return_value = ["foo.txt"]
+
+    m = mocker.MagicMock()
+    m.get_repository.return_value = repo
+    mocker.patch.dict(sys.modules, {"taskgraph.util.vcs": m})
+
+    res = run_taskgraph(["full"])
+    assert res == 0
+    assert not repo.get_changed_files.called
+
+    res = run_taskgraph(["full", "--force-local-files-changed"])
+    assert res == 0
+    assert not repo.get_changed_files.called
+
+    res = run_taskgraph(
+        [
+            "full",
+            "--force-local-files-changed",
+            "-p",
+            "taskcluster/test/params/mc-onpush.yml",
+        ]
+    )
+    assert res == 0
+    assert repo.get_changed_files.call_count == 1
+
+
 def test_tasks_regex(run_taskgraph, capsys):
     run_taskgraph(["full", "--tasks=_.*-t-1"])
     out, _ = capsys.readouterr()