feat: add support for outputting a mermaid flowchart of the kind graph (#834)

Especially in repositories with large graphs, it can be useful to visualize the relationship between the various kinds.

I looked at directly outputting svg files of the graphs here, but decided against it because rendering mermaid diagrams requires a browser or puppeteer AFAICT; and I don't want to add that dependency.

I also looked at adding support for visualizing entire graphs, but those quickly get very unwieldy and I decided against it. (This might be useful in conjunction with a new `--target-task` option, but that's more than I'm willing to take on at this time.)

Author: bhearsum

src/taskgraph/generator.py

@@ -250,6 +250,15 @@ def graph_config(self):
         """
         return self._run_until("graph_config")
 
+    @property
+    def kind_graph(self):
+        """
+        The dependency graph of kinds.
+
+        @type: Graph
+        """
+        return self._run_until("kind_graph")
+
     def _load_kinds(self, graph_config, target_kinds=None):
         if target_kinds:
             # docker-image is an implicit dependency that never appears in
@@ -422,6 +431,8 @@ def _run(self):
                 set(target_kinds) | {"docker-image"}
             )
 
+        yield "kind_graph", kind_graph
+
         logger.info("Generating full task set")
         # Current parallel generation relies on multiprocessing, and forking.
         # This causes problems on Windows and macOS due to how new processes

src/taskgraph/main.py

@@ -64,6 +64,31 @@ def format_taskgraph_yaml(taskgraph):
     return yaml.safe_dump(taskgraph.to_json(), default_flow_style=False)
 
 
+def format_kind_graph_mermaid(kind_graph):
+    """
+    Convert a kind dependency graph to Mermaid flowchart format.
+
+    @param kind_graph: Graph object containing kind nodes and dependencies
+    @return: String representation of the graph in Mermaid format
+    """
+    lines = ["flowchart TD"]
+
+    # Add nodes (kinds)
+    for node in sorted(kind_graph.nodes):
+        # Sanitize node names for Mermaid (replace hyphens with underscores for IDs)
+        node_id = node.replace("-", "_")
+        lines.append(f"    {node_id}[{node}]")
+
+    # Add edges (dependencies)
+    # Reverse the edge direction: if left depends on right, show right --> left
+    for left, right, _ in sorted(kind_graph.edges):
+        left_id = left.replace("-", "_")
+        right_id = right.replace("-", "_")
+        lines.append(f"    {right_id} --> {left_id}")
+
+    return "\n".join(lines)
+
+
 def get_filtered_taskgraph(taskgraph, tasksregex, exclude_keys):
     """
     Filter all the tasks on basis of a regular expression
@@ -225,6 +250,69 @@ def logfile(spec):
     return returncode
 
 
+@command(
+    "kind-graph",
+    help="Generate a Mermaid flowchart diagram source file for the kind dependency graph. To render as a graph, run the output of this command through the Mermaid CLI or an online renderer.",
+)
+@argument("--root", "-r", help="root of the taskgraph definition relative to topsrcdir")
+@argument("--quiet", "-q", action="store_true", help="suppress all logging output")
+@argument(
+    "--verbose", "-v", action="store_true", help="include debug-level logging output"
+)
+@argument(
+    "--parameters",
+    "-p",
+    default=None,
+    help="Parameters to use for the generation. Can be a path to file (.yml or "
+    ".json; see `taskcluster/docs/parameters.rst`), a url, of the form "
+    "`project=mozilla-central` to download latest parameters file for the specified "
+    "project from CI, or of the form `task-id=<decision task id>` to download "
+    "parameters from the specified decision task.",
+)
+@argument(
+    "-o",
+    "--output-file",
+    default=None,
+    help="file path to store generated output.",
+)
+@argument(
+    "-k",
+    "--target-kind",
+    dest="target_kinds",
+    action="append",
+    default=[],
+    help="only return kinds and their dependencies.",
+)
+def show_kind_graph(options):
+    from taskgraph.parameters import parameters_loader  # noqa: PLC0415
+
+    if options.pop("verbose", False):
+        logging.root.setLevel(logging.DEBUG)
+
+    setup_logging()
+
+    target_kinds = options.get("target_kinds", [])
+    parameters = parameters_loader(
+        options.get("parameters"),
+        strict=False,
+        overrides={"target-kinds": target_kinds},
+    )
+
+    tgg = get_taskgraph_generator(options.get("root"), parameters)
+    kind_graph = tgg.kind_graph
+
+    output = format_kind_graph_mermaid(kind_graph)
+
+    if output_file := options.get("output_file"):
+        with open(output_file, "w") as fh:
+            print(output, file=fh)
+        print(f"Kind graph written to {output_file}", file=sys.stderr)
+    else:
+        print(output)
+
+    return 0
+
+
 @command(
     "tasks",
     help="Show the full task set in the task graph. The full task set includes all tasks defined by any kind, without edges (dependencies) between them.",

test/test_generator.py

@@ -338,3 +338,44 @@ def test_kind_load_tasks(monkeypatch, graph_config, parameters, datadir, kind_co
     )
     tasks = kind.load_tasks(parameters, {}, False)
     assert tasks
+
+
+def test_kind_graph(maketgg):
+    "The kind_graph property has all kinds and their dependencies"
+    tgg = maketgg(
+        kinds=[
+            ("_fake3", {"kind-dependencies": ["_fake2", "_fake1"]}),
+            ("_fake2", {"kind-dependencies": ["_fake1"]}),
+            ("_fake1", {"kind-dependencies": []}),
+        ]
+    )
+    kind_graph = tgg.kind_graph
+    assert isinstance(kind_graph, graph.Graph)
+    assert kind_graph.nodes == {"_fake1", "_fake2", "_fake3"}
+    assert kind_graph.edges == {
+        ("_fake3", "_fake2", "kind-dependency"),
+        ("_fake3", "_fake1", "kind-dependency"),
+        ("_fake2", "_fake1", "kind-dependency"),
+    }
+
+
+def test_kind_graph_with_target_kinds(maketgg):
+    "The kind_graph property respects target_kinds parameter"
+    tgg = maketgg(
+        kinds=[
+            ("_fake3", {"kind-dependencies": ["_fake2"]}),
+            ("_fake2", {"kind-dependencies": ["_fake1"]}),
+            ("_fake1", {"kind-dependencies": []}),
+            ("_other", {"kind-dependencies": []}),
+            ("docker-image", {"kind-dependencies": []}),  # Add docker-image
+        ],
+        params={"target-kinds": ["_fake2"]},
+    )
+    kind_graph = tgg.kind_graph
+    # Should only include _fake2, _fake1, and docker-image (implicit dependency)
+    assert "_fake2" in kind_graph.nodes
+    assert "_fake1" in kind_graph.nodes
+    assert "docker-image" in kind_graph.nodes
+    # _fake3 and _other should not be included
+    assert "_fake3" not in kind_graph.nodes
+    assert "_other" not in kind_graph.nodes

test/test_main.py

@@ -12,7 +12,7 @@
 import taskgraph
 from taskgraph.actions import registry
 from taskgraph.graph import Graph
-from taskgraph.main import get_filtered_taskgraph
+from taskgraph.main import format_kind_graph_mermaid, get_filtered_taskgraph
 from taskgraph.main import main as taskgraph_main
 from taskgraph.task import Task
 from taskgraph.taskgraph import TaskGraph
@@ -521,3 +521,124 @@ def test_load_task_command_with_task_id(run_load_task):
         user=None,
         custom_image=None,
     )
+
+
+def test_format_kind_graph_mermaid():
+    """Test conversion of kind graph to Mermaid format"""
+    # Test with simple graph
+    kinds = frozenset(["docker-image"])
+    edges = frozenset()
+    kind_graph = Graph(kinds, edges)
+
+    output = format_kind_graph_mermaid(kind_graph)
+    assert "flowchart TD" in output
+    assert "docker_image[docker-image]" in output
+
+    # Test with complex graph with dependencies
+    kinds = frozenset(["docker-image", "build", "test", "lint"])
+    edges = frozenset(
+        [
+            ("build", "docker-image", "kind-dependency"),
+            ("test", "build", "kind-dependency"),
+            ("lint", "docker-image", "kind-dependency"),
+        ]
+    )
+    kind_graph = Graph(kinds, edges)
+
+    output = format_kind_graph_mermaid(kind_graph)
+    lines = output.split("\n")
+
+    assert lines[0] == "flowchart TD"
+    # Check all nodes are present
+    assert any("build[build]" in line for line in lines)
+    assert any("docker_image[docker-image]" in line for line in lines)
+    assert any("lint[lint]" in line for line in lines)
+    assert any("test[test]" in line for line in lines)
+    # Check edges are reversed (dependencies point to dependents)
+    assert any("docker_image --> build" in line for line in lines)
+    assert any("build --> test" in line for line in lines)
+    assert any("docker_image --> lint" in line for line in lines)
+
+
+def test_show_kinds_command(run_taskgraph, capsys):
+    """Test the kinds command outputs Mermaid format"""
+    res = run_taskgraph(
+        ["kind-graph"],
+        kinds=[
+            ("_fake", {"kind-dependencies": []}),
+        ],
+    )
+    assert res == 0
+
+    out, _ = capsys.readouterr()
+    assert "flowchart TD" in out
+    assert "_fake[_fake]" in out
+
+
+def test_show_kinds_with_dependencies(run_taskgraph, capsys):
+    """Test the kinds command with kind dependencies"""
+    res = run_taskgraph(
+        ["kind-graph"],
+        kinds=[
+            ("_fake3", {"kind-dependencies": ["_fake2"]}),
+            ("_fake2", {"kind-dependencies": ["_fake1"]}),
+            ("_fake1", {"kind-dependencies": []}),
+        ],
+    )
+    assert res == 0
+
+    out, _ = capsys.readouterr()
+    assert "flowchart TD" in out
+    # Check all kinds are present
+    assert "_fake1[_fake1]" in out
+    assert "_fake2[_fake2]" in out
+    assert "_fake3[_fake3]" in out
+    # Check edges are present and reversed
+    assert "_fake1 --> _fake2" in out
+    assert "_fake2 --> _fake3" in out
+
+
+def test_show_kinds_output_file(run_taskgraph, tmpdir):
+    """Test the kinds command writes to file"""
+    output_file = tmpdir.join("kinds.mmd")
+    assert not output_file.check()
+
+    res = run_taskgraph(
+        ["kind-graph", f"--output-file={output_file.strpath}"],
+        kinds=[
+            ("_fake", {"kind-dependencies": []}),
+        ],
+    )
+    assert res == 0
+    assert output_file.check()
+
+    content = output_file.read_text("utf-8")
+    assert "flowchart TD" in content
+    assert "_fake[_fake]" in content
+
+
+def test_show_kinds_with_target_kinds(run_taskgraph, capsys):
+    """Test the kinds command with --target-kind filter"""
+    res = run_taskgraph(
+        ["kind-graph", "-k", "_fake2"],
+        kinds=[
+            ("_fake3", {"kind-dependencies": ["_fake2"]}),
+            ("_fake2", {"kind-dependencies": ["_fake1"]}),
+            ("_fake1", {"kind-dependencies": []}),
+            ("_other", {"kind-dependencies": []}),
+            ("docker-image", {"kind-dependencies": []}),
+        ],
+        params={"target-kinds": ["_fake2"]},
+    )
+    assert res == 0
+
+    out, _ = capsys.readouterr()
+    assert "flowchart TD" in out
+    # Should include _fake2 and its dependencies
+    assert "_fake2[_fake2]" in out
+    assert "_fake1[_fake1]" in out
+    # Should include docker-image (implicit dependency for target_kinds)
+    assert "docker_image[docker-image]" in out
+    # Should not include _fake3 or _other
+    assert "_fake3" not in out
+    assert "_other" not in out