refactor: create new 'pytest-taskgraph' module to house test fixtures

ahal · ahal · commit 8b3482717ff2 · 2024-12-11T16:57:01.000-05:00
These fixtures are useful for any project that wants to test Taskgraph
related transforms or scripts. By putting them in a package, we can
share them to any other repos that may need them.
diff --git a/.github/workflows/pypi-publish.yml b/.github/workflows/pypi-publish.yml
@@ -3,8 +3,9 @@ on:
   release:
     types: [published]
 jobs:
-  pypi-publish:
+  pypi-publish-taskcluster-taskgraph:
     name: upload release to PyPI
+    if: ${{ ! startswith(github.ref, 'refs/tags/pytest-taskgraph') }}
     runs-on: ubuntu-latest
     environment: release
     permissions:
@@ -16,9 +17,32 @@ jobs:
         with:
           python-version: '3.11'
           cache: 'pip'
-      - name: Build package distributions
+      - name: Build taskcluster-taskgraph package distributions
         run: |
           pip install build
           python -m build
       - name: Publish package distributions to PyPI
         uses: pypa/gh-action-pypi-publish@release/v1
+  pypi-publish-pytest-taskgraph:
+    name: upload release to PyPI
+    if: startswith(github.ref, 'refs/tags/pytest-taskgraph')
+    runs-on: ubuntu-latest
+    environment: release
+    permissions:
+      id-token: write
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+          cache: 'pip'
+      - name: Build pytest-taskgraph package distributions
+        working-directory: packages/pytest-taskgraph
+        run: |
+          pip install build
+          python -m build
+      - name: Publish package distributions to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          packages-dir: packages/pytest-taskgraph/dist
diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst
@@ -150,8 +150,8 @@ Or if you'd like to update a specific dependency:
 
    uv sync -P <package>
 
-Releasing
----------
+Releasing taskcluster-taskgraph
+-------------------------------
 
 In order to release a new version of Taskgraph, you will need to:
 
@@ -161,11 +161,25 @@ In order to release a new version of Taskgraph, you will need to:
 4. Create a release in Github pointing to the above commit. Be sure to also
    create a new tag matching this version.
 5. Wait for the ``pypi-publish`` Github workflow and ``push-image-decision`` task to finish.
-6. Verify that expected version has been published to `pypi`_ and pushed to `DockerHub`_.
+6. Verify that expected version has been published to `pypi
+   <https://pypi.org/project/taskcluster-taskgraph>`__ and pushed to `DockerHub`_.
 
-.. _pypi: https://pypi.org/project/taskcluster-taskgraph
 .. _DockerHub: https://hub.docker.com/r/mozillareleases/taskgraph/tags
 
+Releasing pytest-taskgraph
+--------------------------
+
+There's also a Pytest plugin packaged under ``packages/pytest-taskgraph``. The
+release process for this package is:
+
+1. Update ``version`` in ``packages/pytest-taskgraph/pyproject.toml``
+2. Commit and land the changes with a commit message like "chore: bump pytest-taskgraph <version>"
+3. Create a release in Github pointing to the above commit. Be sure to also
+   create a new tag of the form ``pytest-taskgraph-v<version>``.
+4. Wait for the ``pypi-publish`` Github workflow and ``push-image-decision`` task to finish.
+5. Verify that expected version has been published to `pypi <https://pypi.org/project/pytest-taskgraph>`__.
+
+
 Building the Package
 --------------------
 
diff --git a/packages/pytest-taskgraph/README.md b/packages/pytest-taskgraph/README.md
diff --git a/packages/pytest-taskgraph/pyproject.toml b/packages/pytest-taskgraph/pyproject.toml
@@ -0,0 +1,17 @@
+[project]
+name = "pytest-taskgraph"
+version = "0.1.0"
+description = "Add your description here"
+readme = "README.md"
+authors = [
+    { name = "Andrew Halberstadt", email = "ahal@mozilla.com" }
+]
+requires-python = ">=3.8"
+dependencies = ["pytest", "taskcluster-taskgraph>=12.1.0"]
+
+[project.entry-points.pytest11]
+pytest-taskgraph = "pytest_taskgraph"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
diff --git a/packages/pytest-taskgraph/src/pytest_taskgraph/__init__.py b/packages/pytest-taskgraph/src/pytest_taskgraph/__init__.py
@@ -0,0 +1,2 @@
+from .fixtures.gen import *  # noqa
+from .fixtures.vcs import *  # noqa
diff --git a/packages/pytest-taskgraph/src/pytest_taskgraph/fixtures/gen.py b/packages/pytest-taskgraph/src/pytest_taskgraph/fixtures/gen.py
diff --git a/packages/pytest-taskgraph/src/pytest_taskgraph/fixtures/vcs.py b/packages/pytest-taskgraph/src/pytest_taskgraph/fixtures/vcs.py
diff --git a/packages/pytest-taskgraph/uv.lock b/packages/pytest-taskgraph/uv.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -45,6 +45,9 @@ taskgraph = "taskgraph.main:main"
 Repository = "https://github.com/taskcluster/taskgraph"
 Issues = "https://github.com/taskcluster/taskgraph/issues"
 
+[tool.uv.sources]
+pytest-taskgraph = { path = "packages/pytest-taskgraph" }
+
 [tool.uv]
 dev-dependencies = [
   "coverage",
@@ -53,6 +56,7 @@ dev-dependencies = [
   "pyright",
   "pytest",
   "pytest-mock",
+  "pytest-taskgraph",
   "responses",
   "sphinx",
   "sphinx-autobuild",
@@ -79,7 +83,7 @@ xfail_strict = true
 
 [tool.coverage.run]
 branch = true
-source = ["src/taskgraph/", "src/taskgraph/run-task/", "test"]
+source = ["src/taskgraph/", "src/taskgraph/run-task/", "packages", "test"]
 
 ### Lint and Format
 [tool.ruff]
@@ -103,7 +107,7 @@ ignore = [
 ]
 
 [tool.ruff.lint.isort]
-known-first-party = ["taskgraph"]
+known-first-party = ["pytest-taskgraph", "taskgraph"]
 
 [tool.pyright]
 include = ["src"]
diff --git a/taskcluster/config.yml b/taskcluster/config.yml
@@ -15,6 +15,8 @@ index:
 task-priority: low
 
 taskgraph:
+    register: self_taskgraph:register
+    decision-parameters: 'self_taskgraph.custom_parameters:decision_parameters'
     repositories:
         ci:
             name: Taskgraph
diff --git a/taskcluster/docker/decision/Dockerfile b/taskcluster/docker/decision/Dockerfile
@@ -4,6 +4,7 @@ LABEL maintainer="Release Engineering <release+taskgraph@mozilla.com>"
 ENV UV_PROJECT_ENVIRONMENT=/setup/taskgraph
 
 # %include src
+# %include packages
 # %include .hatch_build.py
 # %include README.rst
 # %include pyproject.toml
diff --git a/taskcluster/self_taskgraph/__init__.py b/taskcluster/self_taskgraph/__init__.py
@@ -0,0 +1,21 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from importlib import import_module
+
+
+def register(graph_config):
+    """Setup for task generation."""
+    # Import sibling modules, triggering decorators in the process
+    _import_modules(
+        [
+            "custom_parameters",
+            "custom_target_tasks",
+        ]
+    )
+
+
+def _import_modules(modules):
+    for module in modules:
+        import_module(f".{module}", package=__name__)
diff --git a/taskcluster/self_taskgraph/custom_parameters.py b/taskcluster/self_taskgraph/custom_parameters.py
@@ -0,0 +1,8 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+def decision_parameters(graph_config, parameters):
+    if parameters["tasks_for"] == "github-release":
+        parameters["target_tasks_method"] = "release"
diff --git a/taskcluster/self_taskgraph/custom_target_tasks.py b/taskcluster/self_taskgraph/custom_target_tasks.py
@@ -0,0 +1,18 @@
+from taskgraph.target_tasks import register_target_task, standard_filter
+
+
+@register_target_task("release")
+def target_tasks_release(full_task_graph, parameters, graph_config):
+    tasks = {
+        l: t for l, t in full_task_graph.tasks.items() if standard_filter(t, parameters)
+    }
+
+    # Ensure we don't run `taskcluster-taskgraph` release tasks when publishing
+    # `pytest-taskgraph`.
+    if parameters["head_ref"].startswith("refs/tags/pytest-taskgraph"):
+        tasks = {
+            l: t
+            for l, t in tasks.items()
+            if "github-release" not in t.attributes.get("run_on_tasks_for", [])
+        }
+    return list(tasks.keys())
diff --git a/taskcluster/test/params/main-repo-release-pytest-taskgraph.yml b/taskcluster/test/params/main-repo-release-pytest-taskgraph.yml
@@ -0,0 +1,31 @@
+---
+base_ref: release-test
+base_repository: https://github.com/taskcluster/taskgraph
+base_rev: 5fe502ea8fd2a3f21bba2e8b99bb483507cac130
+build_date: 1707294824
+build_number: 1
+do_not_optimize: []
+enable_always_target: true
+existing_tasks: {}
+files_changed:
+  - src/taskgraph/decision.py
+  - test/test_decision.py
+filters:
+  - target_tasks_method
+head_ref: refs/tags/pytest-taskgraph-v1.0.0
+head_repository: https://github.com/taskcluster/taskgraph
+head_rev: release-test
+head_tag: pytest-taskgraph-v1.0.0
+level: '3'
+moz_build_date: '20240207083344'
+next_version: null
+optimize_strategies: null
+optimize_target_tasks: true
+owner: user@example.com
+project: taskgraph
+pushdate: 0
+pushlog_id: '0'
+repository_type: git
+target_tasks_method: release
+tasks_for: github-release
+version: null
diff --git a/taskcluster/test/params/main-repo-release-taskcluster-taskgraph.yml b/taskcluster/test/params/main-repo-release-taskcluster-taskgraph.yml
@@ -12,10 +12,10 @@ files_changed:
   - test/test_decision.py
 filters:
   - target_tasks_method
-head_ref: release-test
+head_ref: refs/tags/1.0.0
 head_repository: https://github.com/taskcluster/taskgraph
 head_rev: release-test
-head_tag: ''
+head_tag: 1.0.0
 level: '3'
 moz_build_date: '20240207083344'
 next_version: null
@@ -26,6 +26,6 @@ project: taskgraph
 pushdate: 0
 pushlog_id: '0'
 repository_type: git
-target_tasks_method: default
+target_tasks_method: release
 tasks_for: github-release
 version: null
diff --git a/test/conftest.py b/test/conftest.py
@@ -4,11 +4,10 @@
 import pytest
 from responses import RequestsMock
 
-from .fixtures.gen import *  # noqa
-from .fixtures.vcs import *  # noqa
-
 here = Path(__file__).parent
 
+pytest_plugins = ("pytest-taskgraph",)
+
 # Disable as much system/user level configuration as we can to avoid
 # interference with tests.
 # This ignores ~/.hgrc
diff --git a/test/test_actions_rebuild_cached_tasks.py b/test/test_actions_rebuild_cached_tasks.py
@@ -3,13 +3,12 @@
 """
 
 import pytest
+from pytest_taskgraph import make_graph, make_task
 
 from taskgraph import create
 from taskgraph.actions import trigger_action_callback
 from taskgraph.util import taskcluster as tc_util
 
-from .conftest import make_graph, make_task
-
 
 @pytest.fixture
 def run_action(capsys, mocker, monkeypatch, parameters, graph_config):
diff --git a/test/test_generator.py b/test/test_generator.py
@@ -4,13 +4,12 @@
 
 
 import pytest
+from pytest_taskgraph import FakeKind, WithFakeKind, fake_load_graph_config
 
 from taskgraph import generator, graph
 from taskgraph.generator import Kind, load_tasks_for_kind
 from taskgraph.loader.default import loader as default_loader
 
-from .conftest import FakeKind, WithFakeKind, fake_load_graph_config
-
 
 def test_kind_ordering(maketgg):
     "When task kinds depend on each other, they are loaded in postorder"
diff --git a/test/test_optimize.py b/test/test_optimize.py
@@ -6,6 +6,7 @@
 from functools import partial
 
 import pytest
+from pytest_taskgraph import make_graph, make_task
 
 from taskgraph.graph import Graph
 from taskgraph.optimize import base as optimize_mod
@@ -19,8 +20,6 @@
 from taskgraph.task import Task
 from taskgraph.taskgraph import TaskGraph
 
-from .conftest import make_graph, make_task
-
 
 class Remove(OptimizationStrategy):
     def should_remove_task(self, task, params, arg):
diff --git a/test/test_optimize_strategies.py b/test/test_optimize_strategies.py
@@ -7,9 +7,9 @@
 from time import mktime
 
 import pytest
+from pytest_taskgraph import make_task
 
 from taskgraph.optimize.strategies import IndexSearch, SkipUnlessChanged
-from test.fixtures.gen import make_task
 
 
 @pytest.fixture
diff --git a/test/test_transform_task_context.py b/test/test_transform_task_context.py
@@ -6,11 +6,11 @@
 from copy import deepcopy
 from pprint import pprint
 
+from pytest_taskgraph import FakeParameters
+
 from taskgraph.transforms import task_context
 from taskgraph.transforms.base import TransformConfig
 
-from .conftest import FakeParameters
-
 here = os.path.abspath(os.path.dirname(__file__))
 
 TASK_DEFAULTS = {
diff --git a/test/test_transforms_cached_tasks.py b/test/test_transforms_cached_tasks.py
@@ -2,11 +2,10 @@
 from pprint import pprint
 
 import pytest
+from pytest_taskgraph import make_task
 
 from taskgraph.transforms import cached_tasks
 
-from .conftest import make_task
-
 
 def handle_exception(obj, exc=None):
     if exc:
diff --git a/test/test_transforms_from_deps.py b/test/test_transforms_from_deps.py
@@ -5,11 +5,10 @@
 from pprint import pprint
 
 import pytest
+from pytest_taskgraph import make_task
 
 from taskgraph.transforms import from_deps
 
-from .conftest import make_task
-
 
 def handle_exception(obj, exc=None):
     if exc:
diff --git a/test/test_transforms_task.py b/test/test_transforms_task.py
@@ -8,12 +8,11 @@
 from pprint import pprint
 
 import pytest
+from pytest_taskgraph import FakeParameters
 
 from taskgraph.transforms import task
 from taskgraph.transforms.base import TransformConfig
 
-from .conftest import FakeParameters
-
 TASK_DEFAULTS = {
     "description": "fake description",
     "name": "fake-task-name",
diff --git a/test/test_util_dependencies.py b/test/test_util_dependencies.py
@@ -1,6 +1,6 @@
-from taskgraph.util.dependencies import get_dependencies, get_primary_dependency
+from pytest_taskgraph import make_task
 
-from .conftest import make_task
+from taskgraph.util.dependencies import get_dependencies, get_primary_dependency
 
 
 def test_get_dependencies(make_transform_config):
diff --git a/test/test_util_verify.py b/test/test_util_verify.py
diff --git a/uv.lock b/uv.lock

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+from .fixtures.gen import * # noqa`
	`2`	`+from .fixtures.vcs import * # noqa`