diff --git a/contrib/bst-graph b/contrib/bst-graph index 63c73bef6..ceb783b65 100755 --- a/contrib/bst-graph +++ b/contrib/bst-graph @@ -29,10 +29,10 @@ installed. import argparse import subprocess import re +import json import urllib.parse from graphviz import Digraph -from ruamel.yaml import YAML def parse_args(): '''Handle parsing of command line arguments. @@ -72,34 +72,25 @@ def unique_node_name(s): return urllib.parse.quote_plus(s) -def parse_graph(lines): +def parse_graph(output): '''Return nodes and edges of the parsed grpah. Args: - lines: List of lines in format 'NAME|BUILD-DEPS|RUNTIME-DEPS' + output: json output Returns: Tuple of format (nodes,build_deps,runtime_deps) Each member of build_deps and runtime_deps is also a tuple. ''' - parser = YAML(typ="safe") + project = json.loads(output) nodes = set() build_deps = set() runtime_deps = set() - for line in lines: - line = line.strip() - if not line: - continue - # It is safe to split on '|' as it is not a valid character for - # element names. - name, build_dep, runtime_dep = line.split('|') - - build_dep = parser.load(build_dep) - runtime_dep = parser.load(runtime_dep) - + for element in project["elements"]: + name = element["name"] nodes.add(name) - [build_deps.add((name, dep)) for dep in build_dep if dep] - [runtime_deps.add((name, dep)) for dep in runtime_dep if dep] + [build_deps.add((name, dep)) for dep in element.get("build_dependencies", [])] + [runtime_deps.add((name, dep)) for dep in element.get("runtime_dependencies", [])] return nodes, build_deps, runtime_deps @@ -127,13 +118,13 @@ def generate_graph(nodes, build_deps, runtime_deps): def main(): args = parse_args() - cmd = ['bst', 'show', '--format', '%{name}|%{build-deps}|%{runtime-deps}||'] + cmd = ['bst', 'inspect'] if 'element' in args: cmd += args.element - graph_lines = subprocess.check_output(cmd, universal_newlines=True) + json_output = subprocess.check_output(cmd, universal_newlines=True) # NOTE: We generate nodes and edges before giving them to graphviz as # the library does not de-deuplicate them. - nodes, build_deps, runtime_deps = parse_graph(re.split(r"\|\|", graph_lines)) + nodes, build_deps, runtime_deps = parse_graph(json_output) graph = generate_graph(nodes, build_deps, runtime_deps) print(graph.source) diff --git a/src/buildstream/_frontend/cli.py b/src/buildstream/_frontend/cli.py index 9d7619bae..fb0d68dee 100644 --- a/src/buildstream/_frontend/cli.py +++ b/src/buildstream/_frontend/cli.py @@ -24,6 +24,7 @@ from ..types import _CacheBuildTrees, _SchedulerErrorAction, _PipelineSelection, _HostMount, _Scope from .._remotespec import RemoteSpec, RemoteSpecPurpose from ..utils import UtilError +from .inspect import Inspector ################################################################## @@ -531,6 +532,74 @@ def build( ) +################################################################## +# Inspect Command # +################################################################## +@cli.command(name="inspect", short_help="Inspect Project Information") +@click.option( + "--except", "except_", multiple=True, type=click.Path(readable=False), help="Except certain dependencies" +) +@click.option( + "--deps", + "-d", + default=_PipelineSelection.ALL, + show_default=True, + type=FastEnumType( + _PipelineSelection, + [ + _PipelineSelection.NONE, + _PipelineSelection.RUN, + _PipelineSelection.BUILD, + _PipelineSelection.ALL, + ], + ), + help="The dependencies to show", +) +@click.argument("elements", nargs=-1, type=click.Path(readable=False)) +@click.pass_obj +def inspect(app, except_, elements, deps): + """Access structured data about a given buildstream project and it's computed elements. + + Specifying no elements will result in showing the default targets + of the project. If no default targets are configured, all project + elements will be shown. + + When this command is executed from a workspace directory, the default + is to show the workspace element. + + By default this will show all of the dependencies of the + specified target element. + + Specify ``--deps`` to control which elements to show: + + \b + none: No dependencies, just the element itself + run: Runtime dependencies, including the element itself + build: Build time dependencies, excluding the element itself + all: All dependencies + + Examples: + + # A specific target (glob pattern) + \n + bst inspect public/*.bst + + + # With a dependency target + \n + bst inspect -d run + + + # Show each remote file source (with the help of jq) + \n + bst inspect -d all | jq '.elements.[].sources | select( . != null ) | .[] | select( .medium == "remote-file") + + """ + with app.initialized(): + inspector = Inspector(app.stream, app.project, app.context) + inspector.dump_to_stdout(elements, except_=except_, selection=deps) + + ################################################################## # Show Command # ################################################################## diff --git a/src/buildstream/_frontend/inspect.py b/src/buildstream/_frontend/inspect.py new file mode 100644 index 000000000..6cd097b7b --- /dev/null +++ b/src/buildstream/_frontend/inspect.py @@ -0,0 +1,251 @@ +import json +import sys +from dataclasses import dataclass, fields, is_dataclass +from enum import StrEnum + +from .._project import ProjectConfig as _BsProjectConfig, Project as _BsProject +from .._pluginfactory.pluginorigin import PluginType +from .._options import OptionPool +from .._stream import Stream +from ..types import _PipelineSelection, _Scope, _ProjectInformation +from ..node import MappingNode +from ..element import Element + +from .. import _yaml +from .. import _site + + +class _DependencyKind(StrEnum): + ALL = "all" + RUNTIME = "runtime" + BUILD = "build" + + +@dataclass +class _Dependency: + name: str + junction: str | None + kind: _DependencyKind + + +# Inspectable Elements as serialized to the terminal +@dataclass +class _Element: + name: str + description: str + workspace: any + environment: dict[str, str] + variables: dict[str, str] + dependencies: list[_Dependency] + sources: list[dict[str, str]] + + +# Representation of a cache server +@dataclass +class _CacheServer: + url: str + instance: str + + def __init__(self, spec): + self.url = spec.url + self.instance = spec.instance_name + + +# String representation of loaded plugins +@dataclass +class _Plugin: + name: str + description: str + plugin_type: PluginType + + +# A single project loaded from the current configuration +@dataclass +class _Project: + name: str + junction: str | None + options: [(str, str)] + plugins: [_Plugin] + elements: [_Element] + + +# Default values defined for each element within +@dataclass +class _Defaults: + environment: dict[str, str] + + +# Wrapper object ecapsulating the entire output of `bst inspect` +@dataclass +class _InspectOutput: + projects: [_Project] + defaults: _Defaults + + +# Recursively dump the dataclass into a serializable dictionary. Null values +# are dropped from the output. +def _dump_dataclass(_cls): + d = dict() + if not is_dataclass(_cls): + raise Exception("BUG: obj must be a dataclass") + for field in fields(_cls): + value = getattr(_cls, field.name) + if value is None: # hide null values + continue + if is_dataclass(value): + d[field.name] = _dump_dataclass(value) + elif isinstance(value, list): + items = [] + for item in value: + if is_dataclass(item): + # check if it's a list of dataclasses + items.append(_dump_dataclass(item)) + else: + items.append(item) + d[field.name] = items + else: + d[field.name] = value + return d + + +def _dump_option_pool(options: OptionPool): + opts = dict() + return options.export_variables(opts) + + +def _maybe_strip_node_info(obj): + out = dict() + if obj and hasattr(obj, "strip_node_info"): + return obj.strip_node_info() + + +# Inspect elements from a given Buildstream project +class Inspector: + def __init__(self, stream: Stream, project: _BsProject, context): + self.stream = stream + self.project = project + self.context = context + # Load config defaults so we can only show them once instead of + # for each element unless they are distinct + _default_config = _yaml.load(_site.default_project_config, shortname="projectconfig.yaml") + self.default_environment = _default_config.get_mapping("environment").strip_node_info() + + def _get_element(self, element: Element): + sources = [] + for source in element.sources(): + source_infos = source.collect_source_info() + + if source_infos is not None: + serialized_sources = [] + for s in source_infos: + serialized = s.serialize() + serialized_sources.append(serialized) + + sources += serialized_sources + + junction_name = None + project = element._get_project() + if project: + if hasattr(project, "junction") and project.junction: + junction_name = project.junction.name + + named_by_kind = { + str(_DependencyKind.ALL): {}, + str(_DependencyKind.BUILD): {}, + str(_DependencyKind.RUNTIME): {}, + } + + dependencies = [] + for dependency in element._dependencies(_Scope.ALL, recurse=True): + named_by_kind[str(_DependencyKind.ALL)][dependency.name] = dependency + for dependency in element._dependencies(_Scope.BUILD, recurse=True): + named_by_kind[str(_DependencyKind.BUILD)][dependency.name] = dependency + for dependency in element._dependencies(_Scope.RUN, recurse=True): + named_by_kind[str(_DependencyKind.RUNTIME)][dependency.name] = dependency + + for dependency in named_by_kind[str(_DependencyKind.ALL)].values(): + dependencies.append(_Dependency(name=dependency.name, junction=junction_name, kind=_DependencyKind.ALL)) + + # Filter out dependencies covered by ALL + + for name, dependency in named_by_kind[str(_DependencyKind.BUILD)].items(): + if not name in named_by_kind[str(_DependencyKind.ALL)]: + dependencies.append( + _Dependency(name=dependency.name, junction=junction_name, kind=_DependencyKind.BUILD) + ) + + for name, dependency in named_by_kind[str(_DependencyKind.RUNTIME)].items(): + if not name in named_by_kind[str(_DependencyKind.ALL)]: + dependencies.append( + _Dependency(name=dependency.name, junction=junction_name, kind=_DependencyKind.RUNTIME) + ) + + environment = dict() + for key, value in element.get_environment().items(): + if key in self.default_environment and self.default_environment[key] == value: + continue + environment[key] = value + + return _Element( + name=element._get_full_name(), + description=" ".join(element._description.splitlines()), + workspace=element._get_workspace(), + variables=dict(element._Element__variables), + environment=environment, + sources=sources, + dependencies=dependencies, + ) + + def _get_project(self, info: _ProjectInformation, project: _BsProject, elements: [Element]) -> _Project: + plugins = [] + plugins.extend( + [ + _Plugin(name=plugin[0], description=plugin[3], plugin_type=PluginType.ELEMENT.value) + for plugin in project.element_factory.list_plugins() + ] + ) + plugins.extend( + [ + _Plugin(name=plugin[0], description=plugin[3], plugin_type=PluginType.SOURCE.value) + for plugin in project.source_factory.list_plugins() + ] + ) + plugins.extend( + [ + _Plugin(name=plugin[0], description=plugin[3], plugin_type=PluginType.SOURCE_MIRROR.value) + for plugin in project.source_mirror_factory.list_plugins() + ] + ) + + options = _dump_option_pool(project.options) + + junction = None + if hasattr(project, "junction") and project.junction: + junction = project.junction._get_full_name() + + return _Project( + name=project.name, + junction=junction, + options=options, + plugins=plugins, + elements=[self._get_element(element) for element in elements], + ) + + def _get_output(self, elements: [Element]) -> _InspectOutput: + return _InspectOutput( + projects=[ + self._get_project(wrapper, wrapper.project, elements) for wrapper in self.project.loaded_projects() + ], + defaults=_Defaults(environment=self.default_environment), + ) + + def dump_to_stdout(self, elements=[], except_=[], selection=_PipelineSelection.NONE): + if not elements: + elements = self.project.get_default_targets() + + elements = [element for element in filter(lambda name: name not in except_, elements)] + + dependencies = self.stream.load_selection(elements, selection=selection, except_targets=[]) + + output = self._get_output(dependencies) + json.dump(_dump_dataclass(output), sys.stdout) diff --git a/tests/frontend/completions.py b/tests/frontend/completions.py index 38ad6c831..48075ee3c 100644 --- a/tests/frontend/completions.py +++ b/tests/frontend/completions.py @@ -22,7 +22,7 @@ # Project directory DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "completions") -MAIN_COMMANDS = ["artifact ", "build ", "help ", "init ", "shell ", "show ", "source ", "workspace "] +MAIN_COMMANDS = ["artifact ", "build ", "help ", "init ", "inspect ", "shell ", "show ", "source ", "workspace "] MAIN_OPTIONS = [ "--builders ", @@ -236,6 +236,7 @@ def test_option_directory(datafiles, cli, cmd, word_idx, expected, subdir): [ # When running in the project directory ("project", "bst show ", 2, [e + " " for e in PROJECT_ELEMENTS], None), + ("project", "bst inspect ", 2, [e + " " for e in PROJECT_ELEMENTS], None), ( "project", "bst build com", @@ -335,7 +336,7 @@ def test_argument_element_invalid(datafiles, cli, project, cmd, word_idx, expect ("bst he", 1, ["help "]), ("bst help ", 2, MAIN_COMMANDS), ("bst help artifact ", 3, ARTIFACT_COMMANDS), - ("bst help in", 2, ["init "]), + ("bst help in", 2, ["init ", "inspect "]), ("bst help source ", 3, SOURCE_COMMANDS), ("bst help artifact ", 3, ARTIFACT_COMMANDS), ("bst help w", 2, ["workspace "]), diff --git a/tests/frontend/inspect.py b/tests/frontend/inspect.py new file mode 100644 index 000000000..c87b37715 --- /dev/null +++ b/tests/frontend/inspect.py @@ -0,0 +1,95 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Pylint doesn't play well with fixtures and dependency injection from pytest +# pylint: disable=redefined-outer-name + +import os +import pytest +import json +from dataclasses import dataclass + +from buildstream._testing import cli # pylint: disable=unused-import + + +DATA_DIR = os.path.join( + os.path.dirname(os.path.realpath(__file__)), +) + +# check to see if a source exists in an element +@dataclass +class _Source: + name: str # element name + kind: str + version: str + +def _element_by_name(elements, name): + for element in elements: + if element["name"] == name: + return element + +def _assert_has_elements(elements, expected): + n_elements = len(elements) + n_expected = len(expected) + if len(elements) != len(expected): + raise Exception(f"Expected {n_expected} elements, got {n_elements}") + for expected_name in expected: + if _element_by_name(elements, expected_name) is None: + raise Exception(f"Element {expected_name} is missing") + +def _assert_has_source(elements, expected: _Source): + element = _element_by_name(elements, expected.name) + if element is None: + raise Exception(f"Cannot find element {expected.name}") + if "sources" in element: + for source in element["sources"]: + kind = source["kind"] + version = source["version"] + if kind == expected.kind and version == expected.version: + return + raise Exception(f"Element {expected.name} does not contain the expected source") + +@pytest.mark.parametrize( + "flags,elements", + [ + ([], ["import-local-files.bst", "import-remote-files.bst", "target.bst"]), + (["*.bst", "**/*.bst"],["import-local-files.bst", "import-remote-files.bst", "target.bst"]), + ([], ["import-local-files.bst", "import-remote-files.bst", "target.bst"]), + (["--deps", "all"], ["import-local-files.bst", "import-remote-files.bst", "target.bst"]), + (["import-*.bst"], ["import-local-files.bst", "import-remote-files.bst"]) + ], +) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "inspect")) +def test_inspect_simple(cli, datafiles, flags, elements): + project = str(datafiles) + result = cli.run(project=project, silent=True, args=["inspect"] + flags) + output = json.loads(result.output) + _assert_has_elements(output["elements"], elements) + + +@pytest.mark.parametrize( + "flags,sources", + [ + ([], [ + _Source(name="import-remote-files.bst", kind="remote", version="d1bc8d3ba4afc7e109612cb73acbdddac052c93025aa1f82942edabb7deb82a1"), + _Source(name="import-remote-files.bst", kind="tar", version="d1bc8d3ba4afc7e109612cb73acbdddac052c93025aa1f82942edabb7deb82a1"), + ]), + ], +) +@pytest.mark.datafiles(os.path.join(DATA_DIR, "inspect")) +def test_inspect_sources(cli, datafiles, flags, sources): + project = str(datafiles) + result = cli.run(project=project, silent=True, args=["inspect"] + flags) + output = json.loads(result.output) + [_assert_has_source(output["elements"], source) for source in sources] diff --git a/tests/frontend/inspect/elements/import-local-files.bst b/tests/frontend/inspect/elements/import-local-files.bst new file mode 100644 index 000000000..ea81d747a --- /dev/null +++ b/tests/frontend/inspect/elements/import-local-files.bst @@ -0,0 +1,6 @@ +kind: import +sources: +- kind: local + path: files +config: + target: / diff --git a/tests/frontend/inspect/elements/import-remote-files.bst b/tests/frontend/inspect/elements/import-remote-files.bst new file mode 100644 index 000000000..35d41e39e --- /dev/null +++ b/tests/frontend/inspect/elements/import-remote-files.bst @@ -0,0 +1,8 @@ +kind: import +sources: + - kind: remote + url: example:foo.bar.bin + ref: d1bc8d3ba4afc7e109612cb73acbdddac052c93025aa1f82942edabb7deb82a1 + - kind: tar + url: example:baz.qux.tar.gz + ref: d1bc8d3ba4afc7e109612cb73acbdddac052c93025aa1f82942edabb7deb82a1 diff --git a/tests/frontend/inspect/elements/target.bst b/tests/frontend/inspect/elements/target.bst new file mode 100644 index 000000000..3459480f9 --- /dev/null +++ b/tests/frontend/inspect/elements/target.bst @@ -0,0 +1,12 @@ +kind: stack +description: | + + Main stack target for the bst build test + +depends: +- import-local-files.bst +- import-remote-files.bst + +config: + build-commands: + - "cc greeting.c -o greeting" diff --git a/tests/frontend/inspect/files/greeting.c b/tests/frontend/inspect/files/greeting.c new file mode 100644 index 000000000..d685fa9d1 --- /dev/null +++ b/tests/frontend/inspect/files/greeting.c @@ -0,0 +1,12 @@ +#include +#include + +void do_greeting() { + __uid_t uid; + uid = getuid(); + printf("Hello, %d, nice to meet you! \n", uid); +} + +int main() { + do_greeting(); +} diff --git a/tests/frontend/inspect/project.conf b/tests/frontend/inspect/project.conf new file mode 100644 index 000000000..204b5f186 --- /dev/null +++ b/tests/frontend/inspect/project.conf @@ -0,0 +1,10 @@ +# Project config for frontend build test +name: test +min-version: 2.0 +element-path: elements + +variables: + schema: https + +aliases: + example: "%{schema}://example.org/"