Merge branch 'main' into schema-conversion

Author: abhishekmadan30

CHANGELOG.md

@@ -1,23 +1,34 @@
 # Change Log
 
+## [15.1.0] - 2025-08-29
+
+### Added
+
+- New function `taskgraph.generator.load_tasks_for_kinds` to get multiple kinds in one pass
+- Allow the `os-groups` generic-worker feature on linux and mac
+
+### Fixed
+
+- taskgraph build-image regression introduced in 14.6.0
+
 ## [15.0.1] - 2025-08-14
 
-## Reverted
+### Reverted
 
 - multiprocess kind processing (#746)
 
 ## [15.0.0] - 2025-08-13
 
-## Added
+### Added
 
 - Process kinds in parallel with a pool of processes (#738)
 - use `uv` to check python version, when available(#733)
 
-## Changed
+### Changed
 
 - BREAKING CHANGE: Use `primary-dependency-label` attribute to find primary dependency of tasks (#736)
 
-## Fixed
+### Fixed
 
 - Fallback to guessing repo root in config.py (#742)
 

CONTRIBUTING.rst

@@ -157,15 +157,16 @@ In order to release a new version of Taskgraph, you will need to:
 
 1. Update ``CHANGELOG.md``
 2. Update ``version`` in ``pyproject.toml``
-3. Commit, and land the above changes with a commit message like "chore: bump <version>"
-4. Draft a release in Github pointing to the above commit.
+3. Run ``uv lock``
+4. Commit, and land the above changes with a commit message like "chore: bump <version>"
+5. Draft a release in Github pointing to the above commit.
 
    a. Create a new tag of the form ``X.Y.Z``
    b. Ensure "Set as latest release" is checked
    c. Submit the release
 
-5. Wait for the ``pypi-publish`` Github workflow and ``push-image-decision`` task to finish.
-6. Verify that expected version has been published to `pypi
+6. Wait for the ``pypi-publish`` Github workflow and ``push-image-decision`` task to finish.
+7. Verify that expected version has been published to `pypi
    <https://pypi.org/project/taskcluster-taskgraph>`__ and pushed to `DockerHub`_.
 
 .. _DockerHub: https://hub.docker.com/r/mozillareleases/taskgraph/tags
@@ -176,7 +177,7 @@ Releasing pytest-taskgraph
 There's also a Pytest plugin packaged under ``packages/pytest-taskgraph``. The
 release process for this package is:
 
-1. Update ``version`` in ``packages/pytest-taskgraph/pyproject.toml``
+1. Update ``version`` in ``packages/pytest-taskgraph/pyproject.toml``, and run ``uv lock``
 2. Commit and land the changes with a commit message like "chore: bump pytest-taskgraph <version>"
 3. Draft a release in Github pointing to the above commit.
 

packages/pytest-taskgraph/src/pytest_taskgraph/fixtures/gen.py

@@ -30,7 +30,7 @@ def fake_loader(kind, path, config, parameters, loaded_tasks):
             "attributes": {"_tasknum": str(i)},
             "task": {
                 "i": i,
-                "metadata": {"name": f"t-{i}"},
+                "metadata": {"name": f"{kind}-t-{i}"},
                 "deadline": "soon",
             },
             "dependencies": dependencies,

pyproject.toml

@@ -1,7 +1,7 @@
 ### Project
 [project]
 name = "taskcluster-taskgraph"
-version = "15.0.1"
+version = "15.1.0"
 description = "Build taskcluster taskgraphs"
 readme = "README.rst"
 authors = [

src/taskgraph/generator.py

@@ -374,7 +374,7 @@ def _run(self):
         if parameters["enable_always_target"]:
             always_target_tasks = {
                 t.label
-                for t in full_task_graph.tasks.values()  # type: ignore
+                for t in full_task_graph.tasks.values()
                 if t.attributes.get("always_target")
                 if parameters["enable_always_target"] is True
                 or t.kind in parameters["enable_always_target"]
@@ -388,7 +388,7 @@ def _run(self):
         target_graph = full_task_graph.graph.transitive_closure(requested_tasks)
         target_task_graph = TaskGraph(
             {l: all_tasks[l] for l in target_graph.nodes},
-            target_graph,  # type: ignore
+            target_graph,
         )
         yield self.verify(
             "target_task_graph", target_task_graph, graph_config, parameters
@@ -446,19 +446,28 @@ def verify(self, name, *args, **kwargs):
             return name, args[0]
 
 
-def load_tasks_for_kind(parameters, kind, root_dir=None):
+def load_tasks_for_kinds(parameters, kinds, root_dir=None):
     """
-    Get all the tasks of a given kind.
+    Get all the tasks of the given kinds.
 
     This function is designed to be called from outside of taskgraph.
     """
     # make parameters read-write
     parameters = dict(parameters)
-    parameters["target-kinds"] = [kind]
+    parameters["target-kinds"] = kinds
     parameters = parameters_loader(spec=None, strict=False, overrides=parameters)
     tgg = TaskGraphGenerator(root_dir=root_dir, parameters=parameters)
     return {
         task.task["metadata"]["name"]: task
         for task in tgg.full_task_set
-        if task.kind == kind
+        if task.kind in kinds
     }
+
+
+def load_tasks_for_kind(parameters, kind, root_dir=None):
+    """
+    Get all the tasks of a given kind.
+
+    This function is designed to be called from outside of taskgraph.
+    """
+    return load_tasks_for_kinds(parameters, [kind], root_dir)

src/taskgraph/main.py

@@ -590,10 +590,10 @@ def build_image(args):
     graph_config = load_graph_config(root)
 
     if args["context_only"] is None:
-        build_image(args["image_name"], args["tag"], os.environ, graph_config)
+        build_image(args["image_name"], args["tag"], graph_config, os.environ)
     else:
         build_context(
-            args["image_name"], args["context_only"], os.environ, graph_config
+            args["image_name"], args["context_only"], graph_config, os.environ
         )
 
 

src/taskgraph/taskgraph.py

@@ -41,7 +41,7 @@ def __contains__(self, label):
 
     def __iter__(self):
         "Iterate over tasks in undefined order"
-        return iter(self.tasks.values())  # type: ignore
+        return iter(self.tasks.values())
 
     def to_json(self):
         "Return a JSON-able object representing the task graph, as documented"

src/taskgraph/transforms/task.py

@@ -869,11 +869,6 @@ def set_defaults(config, tasks):
         elif worker["implementation"] == "generic-worker":
             worker.setdefault("env", {})
             worker.setdefault("os-groups", [])
-            if worker["os-groups"] and worker["os"] != "windows":
-                raise Exception(
-                    "os-groups feature of generic-worker is only supported on "
-                    "Windows, not on {}".format(worker["os"])
-                )
             worker.setdefault("chain-of-trust", False)
         elif worker["implementation"] in (
             "scriptworker-signing",

src/taskgraph/util/cached_tasks.py

@@ -37,8 +37,8 @@ def add_optimization(
     """
     if (digest is None) == (digest_data is None):
         raise Exception("Must pass exactly one of `digest` and `digest_data`.")
-    if digest is None:
-        digest = hashlib.sha256("\n".join(digest_data).encode("utf-8")).hexdigest()  # type: ignore
+    elif digest is None and digest_data is not None:
+        digest = hashlib.sha256("\n".join(digest_data).encode("utf-8")).hexdigest()
 
     if "cached-task-prefix" in config.graph_config["taskgraph"]:
         cache_prefix = config.graph_config["taskgraph"]["cached-task-prefix"]

src/taskgraph/util/schema.py

@@ -5,6 +5,8 @@
 
 import pprint
 from typing import List
+import re
+from collections.abc import Mapping
 
 import msgspec
 
@@ -192,6 +194,7 @@ def resolve_keyed_by(
 ]
 
 
+
 class Schema(msgspec.Struct, kw_only=True, omit_defaults=True, rename="kebab"):
     """
     Base schema class that extends msgspec.Struct.