diff --git a/src/buildstream/_frontend/cli.py b/src/buildstream/_frontend/cli.py index 9d7619bae..35bb01a6a 100644 --- a/src/buildstream/_frontend/cli.py +++ b/src/buildstream/_frontend/cli.py @@ -14,6 +14,7 @@ import os import re import sys +import json from functools import partial import shutil @@ -21,7 +22,7 @@ from .. import _yaml from .._exceptions import BstError, LoadError, AppError, RemoteError from .complete import main_bashcomplete, complete_path, CompleteUnhandled -from ..types import _CacheBuildTrees, _SchedulerErrorAction, _PipelineSelection, _HostMount, _Scope +from ..types import _CacheBuildTrees, _ElementKind, _SchedulerErrorAction, _PipelineSelection, _HostMount, _Scope from .._remotespec import RemoteSpec, RemoteSpecPurpose from ..utils import UtilError @@ -74,6 +75,7 @@ def __repr__(self): # Override of click's main entry point # ################################################################## + # search_command() # # Helper function to get a command and context object @@ -538,6 +540,7 @@ def build( @click.option( "--except", "except_", multiple=True, type=click.Path(readable=False), help="Except certain dependencies" ) +@click.option("--json", "as_json", is_flag=True, help="Output the information as machine readable JSON") @click.option( "--deps", "-d", @@ -552,7 +555,29 @@ def build( _PipelineSelection.ALL, ], ), - help="The dependencies to show", + help="Types of Elements to Show", +) +@click.option( + "--kind", + "-k", + default=[], + show_default=True, + multiple=True, + type=FastEnumType( + _ElementKind, + [ + _ElementKind.ALL, + _ElementKind.KEY, + _ElementKind.KEY_FULL, + _ElementKind.STATE, + _ElementKind.SOURCES, + _ElementKind.DEPENDENCIES, + _ElementKind.BUILD_DEPENDENCIES, + _ElementKind.RUNTIME_DEPENDENCIES, + _ElementKind.CAS_ARTIFACTS, + ], + ), + help="Kinds of element information to display in JSON", ) @click.option( "--order", @@ -572,7 +597,7 @@ def build( ) @click.argument("elements", nargs=-1, type=click.Path(readable=False)) @click.pass_obj -def show(app, elements, deps, except_, order, format_): +def show(app, elements, deps, as_json, except_, order, kind, format_): """Show elements in the pipeline Specifying no elements will result in showing the default targets @@ -629,19 +654,44 @@ def show(app, elements, deps, except_, order, format_): \b bst show target.bst --format \\ $'---------- %{name} ----------\\n%{vars}' + + **JSON OUTPUT** + + The ``--json`` flag will cause bst to output information in machine readable + JSON. When using this flag you may also specify ``--kind`` to control the + type of information that is output. + + To dump everything: + \b + bst show --json --kind all """ with app.initialized(): + if as_json and format_: + raise AppError("--format and --json are mutually exclusive") + + if format_ and kind: + raise AppError("--format does not support --kind") + if not format_: format_ = app.context.log_element_format # First determine whether we need to go about querying the local cache # and spending time setting up remotes. - state_match = re.search(r"%(\{(state)[^%]*?\})", format_) - key_match = re.search(r"%(\{(key)[^%]*?\})", format_) - full_key_match = re.search(r"%(\{(full-key)[^%]*?\})", format_) - artifact_cas_digest_match = re.search(r"%(\{(artifact-cas-digest)[^%]*?\})", format_) - need_state = bool(state_match or key_match or full_key_match or artifact_cas_digest_match) + need_state = False + if not as_json: + state_match = re.search(r"%(\{(state)[^%]*?\})", format_) + key_match = re.search(r"%(\{(key)[^%]*?\})", format_) + full_key_match = re.search(r"%(\{(full-key)[^%]*?\})", format_) + artifact_cas_digest_match = re.search(r"%(\{(artifact-cas-digest)[^%]*?\})", format_) + need_state = bool(state_match or key_match or full_key_match or artifact_cas_digest_match) + else: + need_state = bool( + _ElementKind.STATE in kind + or _ElementKind.KEY in kind + or _ElementKind.KEY_FULL in kind + or _ElementKind.CAS_ARTIFACTS in kind + ) if not elements: elements = app.project.get_default_targets() @@ -657,8 +707,12 @@ def show(app, elements, deps, except_, order, format_): if order == "alpha": dependencies = sorted(dependencies) - report = app.logger.show_pipeline(dependencies, format_) - click.echo(report) + if as_json: + serialized = app.logger.dump_pipeline(dependencies, kind) + print(json.dumps(serialized)) + else: + report = app.logger.show_pipeline(dependencies, format_) + click.echo(report) ################################################################## diff --git a/src/buildstream/_frontend/widget.py b/src/buildstream/_frontend/widget.py index 09b824ff3..8c0eddec3 100644 --- a/src/buildstream/_frontend/widget.py +++ b/src/buildstream/_frontend/widget.py @@ -22,7 +22,7 @@ import click from .profile import Profile -from ..types import _Scope +from ..types import _Scope, _ElementKind, _ElementState from .. import _yaml from .. import __version__ as bst_version from .. import FileType @@ -327,6 +327,120 @@ def __init__( logfile_tokens = self._parse_logfile_format(context.log_message_format, content_profile, format_profile) self._columns.extend(logfile_tokens) + def _read_state(self, element): + try: + if not element._has_all_sources_resolved(): + return _ElementState.NO_REFERENCE + else: + if element.get_kind() == "junction": + return _ElementState.JUNCTION + elif not element._can_query_cache(): + return _ElementState.WAITING + elif element._cached_failure(): + return _ElementState.FAILED + elif element._cached_success(): + return _ElementState.CACHED + elif not element._can_query_source_cache(): + return _ElementState.WAITING + elif element._fetch_needed(): + return _ElementState.FETCH_NEEDED + elif element._buildable(): + return _ElementState.BUILDABLE + else: + return _ElementState.WAITING + except BstError as e: + # Provide context to plugin error + e.args = ("Failed to determine state for {}: {}".format(element._get_full_name(), str(e)),) + raise e + + # Dump the pipeline as a serializable object + def dump_pipeline(self, dependencies, kinds=[]): + + show_all = _ElementKind.ALL in kinds + + elements = [] + + for element in dependencies: + + # Default values always output + serialized_element = { + "name": element._get_full_name(), + } + + description = " ".join(element._description.splitlines()) + if description: + serialized_element["description"] = description + + workspace = element._get_workspace() + if workspace: + serialized_element["workspace"] = workspace + + if show_all or _ElementKind.STATE in kinds: + serialized_element["state"] = self._read_state(element).value + + if show_all or _ElementKind.KEY in kinds: + serialized_element["key"] = element._get_display_key().brief + + if show_all or _ElementKind.KEY_FULL in kinds: + serialized_element["key_full"] = element._get_display_key().full + + if show_all or _ElementKind.VARIABLES in kinds: + serialized_element["variables"] = dict(element._Element__variables) + + if show_all or _ElementKind.ENVIRONMENT in kinds: + serialized_element["environment"] = dict(element._Element__environment) + + if show_all or _ElementKind.CAS_ARTIFACTS in kinds: + # BUG: Due to the assersion within .get_artifact this will + # error but there is no other way to determine if an artifact + # exists and we only want to show this value for informational + # purposes. + try: + artifact = element._get_artifact() + if artifact.cached(): + serialized_element["artifact"] = { + "files": artifact.get_files(), + "digest": artifact_files._get_digest(), + } + except: + pass + + if show_all or _ElementKind.SOURCES in kinds: + all_source_infos = [] + for source in element.sources(): + source_infos = source.collect_source_info() + + if source_infos is not None: + serialized_sources = [] + for s in source_infos: + serialized = s.serialize() + serialized_sources.append(serialized) + + all_source_infos += serialized_sources + serialized_element["sources"] = all_source_infos + + # Show dependencies + if show_all or _ElementKind.DEPENDENCIES in kinds: + serialized_element["dependencies"] = [ + e._get_full_name() for e in element._dependencies(_Scope.ALL, recurse=False) + ] + + # Show build dependencies + if show_all or _ElementKind.BUILD_DEPENDENCIES in kinds: + serialized_element["build-dependencies"] = [ + e._get_full_name() for e in element._dependencies(_Scope.BUILD, recurse=False) + ] + + # Show runtime dependencies + if show_all or _ElementKind.RUNTIME_DEPENDENCIES in kinds: + serialized_element["runtime-dependencies"] = [ + e._get_full_name() for e in element._dependencies(_Scope.RUN, recurse=False) + ] + + elements.append(serialized_element) + + return elements + # show_pipeline() # # Display a list of elements in the specified format. @@ -359,30 +473,23 @@ def show_pipeline(self, dependencies, format_): line = p.fmt_subst(line, "full-key", key.full, fg="yellow", dim=dim_keys) line = p.fmt_subst(line, "description", description, fg="yellow", dim=dim_keys) - try: - if not element._has_all_sources_resolved(): - line = p.fmt_subst(line, "state", "no reference", fg="red") - else: - if element.get_kind() == "junction": - line = p.fmt_subst(line, "state", "junction", fg="magenta") - elif not element._can_query_cache(): - line = p.fmt_subst(line, "state", "waiting", fg="blue") - elif element._cached_failure(): - line = p.fmt_subst(line, "state", "failed", fg="red") - elif element._cached_success(): - line = p.fmt_subst(line, "state", "cached", fg="magenta") - elif not element._can_query_source_cache(): - line = p.fmt_subst(line, "state", "waiting", fg="blue") - elif element._fetch_needed(): - line = p.fmt_subst(line, "state", "fetch needed", fg="red") - elif element._buildable(): - line = p.fmt_subst(line, "state", "buildable", fg="green") - else: - line = p.fmt_subst(line, "state", "waiting", fg="blue") - except BstError as e: - # Provide context to plugin error - e.args = ("Failed to determine state for {}: {}".format(element._get_full_name(), str(e)),) - raise e + element_state = self._read_state(element) + if element_state == _ElementState.NO_REFERENCE: + line = p.fmt_subst(line, "state", "no reference", fg="red") + elif element_state == _ElementState.JUNCTION: + line = p.fmt_subst(line, "state", "junction", fg="magenta") + elif element_state == _ElementState.FAILED: + line = p.fmt_subst(line, "state", "failed", fg="red") + elif element_state == _ElementState.CACHED: + line = p.fmt_subst(line, "state", "cached", fg="magenta") + elif element_state == _ElementState.WAITING: + line = p.fmt_subst(line, "state", "waiting", fg="blue") + elif element_state == _ElementState.FETCH_NEEDED: + line = p.fmt_subst(line, "state", "fetch needed", fg="red") + elif element_state == _ElementState.BUILDABLE: + line = p.fmt_subst(line, "state", "buildable", fg="green") + else: + raise BstError(f"Unreachable State: {element_state}") # Element configuration if "%{config" in format_: diff --git a/src/buildstream/types.py b/src/buildstream/types.py index 2ee92afa7..05adaed5d 100644 --- a/src/buildstream/types.py +++ b/src/buildstream/types.py @@ -311,6 +311,70 @@ def __str__(self): return str(self.value) +# _ElementKind() +# +# Used to indicate which values you are interested in when dumping the +# pipeline to a serializable format. +class _ElementKind(FastEnum): + + # Show all available pipeline information + ALL = "all" + + # Show the short version of an element key + KEY = "key" + + # Show the full key information + KEY_FULL = "key-full" + + # Show the Element State + STATE = "state" + + # Show source provenance information + SOURCES = "source" + + # Dependencies + DEPENDENCIES = "dependences" + + # Build dependencies + BUILD_DEPENDENCIES = "build-dependencies" + + # Runtime dependencies + RUNTIME_DEPENDENCIES = "runtime-dependencies" + + # CAS Artifacts + CAS_ARTIFACTS = "cas-artifacts" + + # Element Variables + VARIABLES = "variables" + + # Element Environment + ENVIRONMENT = "environment" + + +# Used to indicate the state of a given element +class _ElementState(FastEnum): + # Cannot determine the element state + NO_REFERENCE = "no-reference" + + # The element has failed + FAILED = "failed" + + # The element is a junction + JUNCTION = "junction" + + # The element is waiting + WAITING = "waiting" + + # The element is cached + CACHED = "cached" + + # The element needs to be loaded from a remote source + FETCH_NEEDED = "fetch-needed" + + # The element my be built + BUILDABLE = "buildable" + + # _ProjectInformation() # # A descriptive object about a project.