From 9faeca8426b665adadb737b534e12af8c71cc358 Mon Sep 17 00:00:00 2001 From: Arran Ireland Date: Mon, 12 Aug 2024 18:00:46 +0100 Subject: [PATCH 1/4] Add bst graph command Adds a new command to bst called 'graph' which reimplements the same API and functionality as contrib/bst-graph. The implementation details are within _stream.py as another function called 'graph'. Part of #1915. --- requirements/requirements.in | 1 + requirements/requirements.txt | 1 + src/buildstream/_frontend/cli.py | 26 ++++++++++++++++++++++++ src/buildstream/_stream.py | 34 ++++++++++++++++++++++++++++++++ 4 files changed, 62 insertions(+) diff --git a/requirements/requirements.in b/requirements/requirements.in index 8d9a8ec0b..26e35e784 100644 --- a/requirements/requirements.in +++ b/requirements/requirements.in @@ -1,4 +1,5 @@ Click >= 7.0 +graphviz grpcio Jinja2 >= 2.10 pluginbase diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 71485fc8a..feeaae345 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -1,4 +1,5 @@ click==8.1.7 +graphviz==0.20.3 grpcio==1.65.1 Jinja2==3.1.4 pluginbase==1.0.1 diff --git a/src/buildstream/_frontend/cli.py b/src/buildstream/_frontend/cli.py index 89475b5a1..50fc26080 100644 --- a/src/buildstream/_frontend/cli.py +++ b/src/buildstream/_frontend/cli.py @@ -1654,3 +1654,29 @@ def artifact_delete(app, artifacts, deps): """Remove artifacts from the local cache""" with app.initialized(): app.stream.artifact_delete(artifacts, selection=deps) + + +################################################################### +# Graph Command # +################################################################### +@cli.command(short_help="Render pipeline dependency graph.") +@click.option( + "--format", + "-f", + "format_", + metavar="FORMAT", + default="dot", + type=click.STRING, + help="Render format: e.g. `dot`, `pdf`, `png`, `svg`, etc", +) +@click.option( + "--view", + is_flag=True, + help="Open the rendered graph with the default application", +) +@click.argument("element", nargs=1, required=True, type=click.Path()) +@click.pass_obj +def graph(app, element, format_, view): + """Render dependency graph.""" + with app.initialized(): + app.stream.graph(element, format_, view) diff --git a/src/buildstream/_stream.py b/src/buildstream/_stream.py index 2266b27f0..0802a6cbe 100644 --- a/src/buildstream/_stream.py +++ b/src/buildstream/_stream.py @@ -27,6 +27,7 @@ from contextlib import contextmanager, suppress from collections import deque from typing import List, Tuple, Optional, Iterable, Callable +from graphviz import Digraph from ._artifactelement import verify_artifact_ref, ArtifactElement from ._artifactproject import ArtifactProject @@ -1228,6 +1229,39 @@ def redirect_element_names(self, elements): return list(output_elements) + # graph() + # + # Renders a dependency graph for the target element, in the given format and optionally opens it. + # + # Args: + # target (str): The target element from which to build a dependency graph. + # + def graph(self, target, format_, view): + graph_ = Digraph() + + for e in self.load_selection([target], selection=_PipelineSelection.ALL, need_state=False): + name = e._get_full_name() + build_deps = set(dep._get_full_name() for dep in e._dependencies(_Scope.BUILD, recurse=False) if dep) + runtime_deps = set(dep._get_full_name() for dep in e._dependencies(_Scope.RUN, recurse=False) if dep) + + graph_.node(name) + for dep in build_deps: + graph_.edge(name, dep, label='build-dep') + for dep in runtime_deps: + graph_.edge(name, dep, label='runtime-dep') + + graph_name = os.path.basename(target) + graph_name, _ = os.path.splitext(graph_name) + graph_path = graph_.render(cleanup=True, + filename=graph_name, + format=format_, + view=view) + + if graph_path: + self._context.messenger.info(f"Rendered dependency graph: {graph_path}") + else: + self._context.messenger.warn("Failed to render graph") + # get_state() # # Get the State object owned by Stream From bd1aaffe169215ea3a30588713c70491f7fa9d7f Mon Sep 17 00:00:00 2001 From: Arran Ireland Date: Tue, 13 Aug 2024 10:04:22 +0100 Subject: [PATCH 2/4] Make buildtime and runtime graphs separate renders It can already be difficult to tell what's going on in large graphs. This commit helps make the graphs clearer and smaller. However this is no longer the same functionality as in the original contrib/bst-graph. Part of #1915. --- src/buildstream/_stream.py | 48 +++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/src/buildstream/_stream.py b/src/buildstream/_stream.py index 0802a6cbe..099dfe663 100644 --- a/src/buildstream/_stream.py +++ b/src/buildstream/_stream.py @@ -1237,30 +1237,30 @@ def redirect_element_names(self, elements): # target (str): The target element from which to build a dependency graph. # def graph(self, target, format_, view): - graph_ = Digraph() - - for e in self.load_selection([target], selection=_PipelineSelection.ALL, need_state=False): - name = e._get_full_name() - build_deps = set(dep._get_full_name() for dep in e._dependencies(_Scope.BUILD, recurse=False) if dep) - runtime_deps = set(dep._get_full_name() for dep in e._dependencies(_Scope.RUN, recurse=False) if dep) - - graph_.node(name) - for dep in build_deps: - graph_.edge(name, dep, label='build-dep') - for dep in runtime_deps: - graph_.edge(name, dep, label='runtime-dep') - - graph_name = os.path.basename(target) - graph_name, _ = os.path.splitext(graph_name) - graph_path = graph_.render(cleanup=True, - filename=graph_name, - format=format_, - view=view) - - if graph_path: - self._context.messenger.info(f"Rendered dependency graph: {graph_path}") - else: - self._context.messenger.warn("Failed to render graph") + def _render(scope): + g = Digraph() + scope_name = {_Scope.BUILD: 'buildtime', _Scope.RUN: 'runtime'}[scope] + + for element in self.load_selection([target], selection=_PipelineSelection.ALL, need_state=False): + name = element._get_full_name() + g.node(name) + dependencies = {d._get_full_name() for d in element._dependencies(scope, recurse=False) if d} + + for dep in dependencies: + g.edge(name, dep) + + filename = os.path.basename(target) + filename, _ = os.path.splitext(filename) + filename = f'{filename}.{scope_name}' + path = g.render(cleanup=True, filename=filename, format=format_, view=view) + + if path: + self._context.messenger.info(f'Rendered dependency graph: {path}') + else: + self._context.messenger.warn('Failed to render graph') + + _render(_Scope.BUILD) + _render(_Scope.RUN) # get_state() # From beaf9f0f74e5993930827c656b8f81e629c82305 Mon Sep 17 00:00:00 2001 From: Arran Ireland Date: Tue, 13 Aug 2024 10:07:16 +0100 Subject: [PATCH 3/4] Remove contrib/bst-graph This is no longer needed when reimplemented into buildstream itself. Part of #1915. --- contrib/bst-graph | 130 ---------------------------------------------- 1 file changed, 130 deletions(-) delete mode 100755 contrib/bst-graph diff --git a/contrib/bst-graph b/contrib/bst-graph deleted file mode 100755 index 3fe93e1ff..000000000 --- a/contrib/bst-graph +++ /dev/null @@ -1,130 +0,0 @@ -#!/usr/bin/env python3 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Authors: -# Chandan Singh -# -'''Print dependency graph of given element(s) in DOT format. - -This script must be run from the same directory where you would normally -run `bst` commands. - -When `--format` option is specified, the output will also be rendered in the -given format. A file with name `bst-graph.{format}` will be created in the same -directory. To use this option, you must have the `graphviz` command line tool -installed. -''' - -import argparse -import subprocess -import re - -from graphviz import Digraph -from ruamel.yaml import YAML - -def parse_args(): - '''Handle parsing of command line arguments. - - Returns: - A argparse.Namespace object - ''' - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - 'element', nargs='*', - help='Name of the element' - ) - parser.add_argument( - '--format', - help='Redner the graph in given format (`pdf`, `png`, `svg` etc)' - ) - parser.add_argument( - '--view', action='store_true', - help='Open the rendered graph with the default application' - ) - return parser.parse_args() - - -def parse_graph(lines): - '''Return nodes and edges of the parsed grpah. - - Args: - lines: List of lines in format 'NAME|BUILD-DEPS|RUNTIME-DEPS' - - Returns: - Tuple of format (nodes,build_deps,runtime_deps) - Each member of build_deps and runtime_deps is also a tuple. - ''' - parser = YAML(typ="safe") - nodes = set() - build_deps = set() - runtime_deps = set() - for line in lines: - line = line.strip() - if not line: - continue - # It is safe to split on '|' as it is not a valid character for - # element names. - name, build_dep, runtime_dep = line.split('|') - - build_dep = parser.load(build_dep) - runtime_dep = parser.load(runtime_dep) - - nodes.add(name) - [build_deps.add((name, dep)) for dep in build_dep if dep] - [runtime_deps.add((name, dep)) for dep in runtime_dep if dep] - - return nodes, build_deps, runtime_deps - - -def generate_graph(nodes, build_deps, runtime_deps): - '''Generate graph from given nodes and edges. - - Args: - nodes: set of nodes - build_deps: set of tuples of build depdencies - runtime_deps: set of tuples of runtime depdencies - - Returns: - A graphviz.Digraph object - ''' - graph = Digraph() - for node in nodes: - graph.node(node) - for source, target in build_deps: - graph.edge(source, target, label='build-dep') - for source, target in runtime_deps: - graph.edge(source, target, label='runtime-dep') - return graph - - -def main(): - args = parse_args() - cmd = ['bst', 'show', '--format', '%{name}|%{build-deps}|%{runtime-deps}||'] - if 'element' in args: - cmd += args.element - graph_lines = subprocess.check_output(cmd, universal_newlines=True) - # NOTE: We generate nodes and edges before giving them to graphviz as - # the library does not de-deuplicate them. - nodes, build_deps, runtime_deps = parse_graph(re.split("\|\|", graph_lines)) - graph = generate_graph(nodes, build_deps, runtime_deps) - print(graph.source) - if args.format: - graph.render(cleanup=True, - filename='bst-graph', - format=args.format, - view=args.view) - - -if __name__ == '__main__': - main() From 43aa419e5aaeb05e4034d80752bd447ab891b769 Mon Sep 17 00:00:00 2001 From: Arran Ireland Date: Tue, 20 Aug 2024 17:19:21 +0100 Subject: [PATCH 4/4] Remove graphviz dependency and rendering --- requirements/requirements.in | 1 - requirements/requirements.txt | 1 - src/buildstream/_frontend/cli.py | 17 +++++-------- src/buildstream/_stream.py | 29 ++++++++++------------ src/buildstream/_tree.py | 42 ++++++++++++++++++++++++++++++++ 5 files changed, 61 insertions(+), 29 deletions(-) create mode 100644 src/buildstream/_tree.py diff --git a/requirements/requirements.in b/requirements/requirements.in index 26e35e784..8d9a8ec0b 100644 --- a/requirements/requirements.in +++ b/requirements/requirements.in @@ -1,5 +1,4 @@ Click >= 7.0 -graphviz grpcio Jinja2 >= 2.10 pluginbase diff --git a/requirements/requirements.txt b/requirements/requirements.txt index feeaae345..71485fc8a 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -1,5 +1,4 @@ click==8.1.7 -graphviz==0.20.3 grpcio==1.65.1 Jinja2==3.1.4 pluginbase==1.0.1 diff --git a/src/buildstream/_frontend/cli.py b/src/buildstream/_frontend/cli.py index 50fc26080..38c621e05 100644 --- a/src/buildstream/_frontend/cli.py +++ b/src/buildstream/_frontend/cli.py @@ -1659,7 +1659,7 @@ def artifact_delete(app, artifacts, deps): ################################################################### # Graph Command # ################################################################### -@cli.command(short_help="Render pipeline dependency graph.") +@cli.command(short_help="Output pipeline dependency graph.") @click.option( "--format", "-f", @@ -1667,16 +1667,11 @@ def artifact_delete(app, artifacts, deps): metavar="FORMAT", default="dot", type=click.STRING, - help="Render format: e.g. `dot`, `pdf`, `png`, `svg`, etc", + help="Output format: e.g. `dot` and `pkl`", ) -@click.option( - "--view", - is_flag=True, - help="Open the rendered graph with the default application", -) -@click.argument("element", nargs=1, required=True, type=click.Path()) +@click.argument("target", nargs=1, required=True, type=click.Path()) @click.pass_obj -def graph(app, element, format_, view): - """Render dependency graph.""" +def graph(app, target, format_): + """Output dependency graph.""" with app.initialized(): - app.stream.graph(element, format_, view) + app.stream.graph(target, format_) diff --git a/src/buildstream/_stream.py b/src/buildstream/_stream.py index 099dfe663..d2774d02e 100644 --- a/src/buildstream/_stream.py +++ b/src/buildstream/_stream.py @@ -24,10 +24,10 @@ import shutil import tarfile import tempfile + from contextlib import contextmanager, suppress from collections import deque from typing import List, Tuple, Optional, Iterable, Callable -from graphviz import Digraph from ._artifactelement import verify_artifact_ref, ArtifactElement from ._artifactproject import ArtifactProject @@ -50,7 +50,7 @@ from ._state import State from .types import _KeyStrength, _PipelineSelection, _Scope, _HostMount from .plugin import Plugin -from . import utils, node, _yaml, _site, _pipeline +from . import utils, node, _tree, _yaml, _site, _pipeline # Stream() @@ -1231,33 +1231,30 @@ def redirect_element_names(self, elements): # graph() # - # Renders a dependency graph for the target element, in the given format and optionally opens it. + # Outputs a dependency graph for the target element, in the given format. + # The dot file format can be rendered using graphviz. # # Args: # target (str): The target element from which to build a dependency graph. + # format_ ('dot'/'pkl'): A .pkl dictionary or graphviz .dot file. # - def graph(self, target, format_, view): + def graph(self, target, format_): def _render(scope): - g = Digraph() + tree = _tree.Tree() scope_name = {_Scope.BUILD: 'buildtime', _Scope.RUN: 'runtime'}[scope] for element in self.load_selection([target], selection=_PipelineSelection.ALL, need_state=False): - name = element._get_full_name() - g.node(name) dependencies = {d._get_full_name() for d in element._dependencies(scope, recurse=False) if d} for dep in dependencies: - g.edge(name, dep) + tree.link(element._get_full_name(), dep) - filename = os.path.basename(target) - filename, _ = os.path.splitext(filename) - filename = f'{filename}.{scope_name}' - path = g.render(cleanup=True, filename=filename, format=format_, view=view) + path = os.path.basename(target) + path, _ = os.path.splitext(path) + path = f'{path}.{scope_name}' + tree.save(path, format_) - if path: - self._context.messenger.info(f'Rendered dependency graph: {path}') - else: - self._context.messenger.warn('Failed to render graph') + self._context.messenger.info(f'Rendered dependency graph: {path}') _render(_Scope.BUILD) _render(_Scope.RUN) diff --git a/src/buildstream/_tree.py b/src/buildstream/_tree.py new file mode 100644 index 000000000..b87019fe7 --- /dev/null +++ b/src/buildstream/_tree.py @@ -0,0 +1,42 @@ +import os +import pickle as pkl + +from typing import List + +class Tree: + def __init__(self): + self.nodes = set() + self.links = dict() + + def link(self, predecessor: str, successor: str) -> None: + self.links.setdefault(successor, []) + self.links.setdefault(predecessor, []) + + if successor == predecessor: + raise ValueError(f'Predecessor and successor are identical: {successor}={predecessor}') + elif successor in self.links[predecessor]: + raise ValueError(f'Cycle detected - not valid tree: {successor}<=>{predecessor}') + + for n in [predecessor, successor]: + self.nodes.add(n) + + self.links[successor].append(predecessor) + + def save(self, path: str, format_: str = 'dot') -> None: + """Saves the tree to disk. + + dot: A graphviz .dot file. + pkl: A treelib compatible dictionary .pkl file. + """ + with open(f'{path}.{format_}', 'w+') as file: + if format_ == 'pkl': + pkl.dump(self.links, file) + elif format_ == 'dot': + lines = [f'digraph "{path}" {{'] + lines.extend(f' "{n}" [label="{n}"]' for n in self.nodes) + lines.extend(f' "{p}" -> "{s}"' for s, ps in self.links.items() for p in ps) + lines.append(f'}}') + + file.writelines([f'{l}\n' for l in lines]) + else: + raise ValueError(f'Unknown format: {format_}')