Feature: Add support for generating json from bst show command

This adds a new flag called --json to the `bst show` command which when
specified will make bst output a machine readable JSON dump of the dependency
state within the current buildstream project. Having JSON support makes it
easier to write tools that need to inspect the configuration of a buildstream
enabled project.

Author: kevinschoon

src/buildstream/_frontend/cli.py

@@ -14,14 +14,15 @@
 import os
 import re
 import sys
+import json
 from functools import partial
 
 import shutil
 import click
 from .. import _yaml
 from .._exceptions import BstError, LoadError, AppError, RemoteError
 from .complete import main_bashcomplete, complete_path, CompleteUnhandled
-from ..types import _CacheBuildTrees, _SchedulerErrorAction, _PipelineSelection, _HostMount, _Scope
+from ..types import _CacheBuildTrees, _ElementKind, _SchedulerErrorAction, _PipelineSelection, _HostMount, _Scope
 from .._remotespec import RemoteSpec, RemoteSpecPurpose
 from ..utils import UtilError
 
@@ -74,6 +75,7 @@ def __repr__(self):
 #            Override of click's main entry point                #
 ##################################################################
 
+
 # search_command()
 #
 # Helper function to get a command and context object
@@ -538,6 +540,7 @@ def build(
 @click.option(
     "--except", "except_", multiple=True, type=click.Path(readable=False), help="Except certain dependencies"
 )
+@click.option("--json", "as_json", is_flag=True, help="Output the information as machine readable JSON")
 @click.option(
     "--deps",
     "-d",
@@ -552,7 +555,29 @@ def build(
             _PipelineSelection.ALL,
         ],
     ),
-    help="The dependencies to show",
+    help="Types of Elements to Show",
+)
+@click.option(
+    "--kind",
+    "-k",
+    default=[],
+    show_default=True,
+    multiple=True,
+    type=FastEnumType(
+        _ElementKind,
+        [
+            _ElementKind.ALL,
+            _ElementKind.KEY,
+            _ElementKind.KEY_FULL,
+            _ElementKind.STATE,
+            _ElementKind.SOURCES,
+            _ElementKind.DEPENDENCIES,
+            _ElementKind.BUILD_DEPENDENCIES,
+            _ElementKind.RUNTIME_DEPENDENCIES,
+            _ElementKind.CAS_ARTIFACTS,
+        ],
+    ),
+    help="Kinds of element information to display in JSON",
 )
 @click.option(
     "--order",
@@ -572,7 +597,7 @@ def build(
 )
 @click.argument("elements", nargs=-1, type=click.Path(readable=False))
 @click.pass_obj
-def show(app, elements, deps, except_, order, format_):
+def show(app, elements, deps, as_json, except_, order, kind, format_):
     """Show elements in the pipeline
 
     Specifying no elements will result in showing the default targets
@@ -629,19 +654,44 @@ def show(app, elements, deps, except_, order, format_):
     \b
         bst show target.bst --format \\
             $'---------- %{name} ----------\\n%{vars}'
+
+    **JSON OUTPUT**
+
+    The ``--json`` flag will cause bst to output information in machine readable
+    JSON. When using this flag you may also specify ``--kind`` to control the
+    type of information that is output.
+
+    To dump everything:
+    \b
+        bst show --json --kind all
     """
     with app.initialized():
 
+        if as_json and format_:
+            raise AppError("--format and --json are mutually exclusive")
+
+        if format_ and kind:
+            raise AppError("--format does not support --kind")
+
         if not format_:
             format_ = app.context.log_element_format
 
         # First determine whether we need to go about querying the local cache
         # and spending time setting up remotes.
-        state_match = re.search(r"%(\{(state)[^%]*?\})", format_)
-        key_match = re.search(r"%(\{(key)[^%]*?\})", format_)
-        full_key_match = re.search(r"%(\{(full-key)[^%]*?\})", format_)
-        artifact_cas_digest_match = re.search(r"%(\{(artifact-cas-digest)[^%]*?\})", format_)
-        need_state = bool(state_match or key_match or full_key_match or artifact_cas_digest_match)
+        need_state = False
+        if not as_json:
+            state_match = re.search(r"%(\{(state)[^%]*?\})", format_)
+            key_match = re.search(r"%(\{(key)[^%]*?\})", format_)
+            full_key_match = re.search(r"%(\{(full-key)[^%]*?\})", format_)
+            artifact_cas_digest_match = re.search(r"%(\{(artifact-cas-digest)[^%]*?\})", format_)
+            need_state = bool(state_match or key_match or full_key_match or artifact_cas_digest_match)
+        else:
+            need_state = bool(
+                _ElementKind.STATE in kind
+                or _ElementKind.KEY in kind
+                or _ElementKind.KEY_FULL in kind
+                or _ElementKind.CAS_ARTIFACTS in kind
+            )
 
         if not elements:
             elements = app.project.get_default_targets()
@@ -657,8 +707,12 @@ def show(app, elements, deps, except_, order, format_):
         if order == "alpha":
             dependencies = sorted(dependencies)
 
-        report = app.logger.show_pipeline(dependencies, format_)
-        click.echo(report)
+        if as_json:
+            serialized = app.logger.dump_pipeline(dependencies, kind)
+            print(json.dumps(serialized))
+        else:
+            report = app.logger.show_pipeline(dependencies, format_)
+            click.echo(report)
 
 
 ##################################################################

src/buildstream/_frontend/widget.py

@@ -22,7 +22,7 @@
 import click
 
 from .profile import Profile
-from ..types import _Scope
+from ..types import _Scope, _ElementKind, _ElementState
 from .. import _yaml
 from .. import __version__ as bst_version
 from .. import FileType
@@ -327,6 +327,120 @@ def __init__(
         logfile_tokens = self._parse_logfile_format(context.log_message_format, content_profile, format_profile)
         self._columns.extend(logfile_tokens)
 
+    def _read_state(self, element):
+        try:
+            if not element._has_all_sources_resolved():
+                return _ElementState.NO_REFERENCE
+            else:
+                if element.get_kind() == "junction":
+                    return _ElementState.JUNCTION
+                elif not element._can_query_cache():
+                    return _ElementState.WAITING
+                elif element._cached_failure():
+                    return _ElementState.FAILED
+                elif element._cached_success():
+                    return _ElementState.CACHED
+                elif not element._can_query_source_cache():
+                    return _ElementState.WAITING
+                elif element._fetch_needed():
+                    return _ElementState.FETCH_NEEDED
+                elif element._buildable():
+                    return _ElementState.BUILDABLE
+                else:
+                    return _ElementState.WAITING
+        except BstError as e:
+            # Provide context to plugin error
+            e.args = ("Failed to determine state for {}: {}".format(element._get_full_name(), str(e)),)
+            raise e
+
+    # Dump the pipeline as a serializable object
+    def dump_pipeline(self, dependencies, kinds=[]):
+
+        show_all = _ElementKind.ALL in kinds
+
+        elements = []
+
+        for element in dependencies:
+
+            # Default values always output
+            serialized_element = {
+                "name": element._get_full_name(),
+            }
+
+            description = " ".join(element._description.splitlines())
+            if description:
+                serialized_element["description"] = description
+
+            workspace = element._get_workspace()
+            if workspace:
+                serialized_element["workspace"] = workspace
+
+            if show_all or _ElementKind.STATE in kinds:
+                serialized_element["state"] = self._read_state(element).value
+
+            if show_all or _ElementKind.KEY in kinds:
+                serialized_element["key"] = element._get_display_key().brief
+
+            if show_all or _ElementKind.KEY_FULL in kinds:
+                serialized_element["key_full"] = element._get_display_key().full
+
+            if show_all or _ElementKind.VARIABLES in kinds:
+                serialized_element["variables"] = dict(element._Element__variables)
+
+            if show_all or _ElementKind.ENVIRONMENT in kinds:
+                serialized_element["environment"] = dict(element._Element__environment)
+
+            if show_all or _ElementKind.CAS_ARTIFACTS in kinds:
+                # BUG: Due to the assersion within .get_artifact this will 
+                # error but there is no other way to determine if an artifact
+                # exists and we only want to show this value for informational 
+                # purposes.
+                try:
+                    artifact = element._get_artifact()
+                    if artifact.cached():
+                        serialized_element["artifact"] = {
+                            "files": artifact.get_files(),
+                            "digest": artifact_files._get_digest(),
+                        }
+                except:
+                    pass
+
+            if show_all or _ElementKind.SOURCES in kinds:
+                all_source_infos = []
+                for source in element.sources():
+                    source_infos = source.collect_source_info()
+
+                    if source_infos is not None:
+                        serialized_sources = []
+                        for s in source_infos:
+                            serialized = s.serialize()
+                            serialized_sources.append(serialized)
+
+                        all_source_infos += serialized_sources
+                serialized_element["sources"] = all_source_infos
+
+            # Show dependencies
+            if show_all or _ElementKind.DEPENDENCIES in kinds:
+                serialized_element["dependencies"] = [
+                    e._get_full_name() for e in element._dependencies(_Scope.ALL, recurse=False)
+                ]
+
+            # Show build dependencies
+            if show_all or _ElementKind.BUILD_DEPENDENCIES in kinds:
+                serialized_element["build-dependencies"] = [
+                    e._get_full_name() for e in element._dependencies(_Scope.BUILD, recurse=False)
+                ]
+
+            # Show runtime dependencies
+            if show_all or _ElementKind.RUNTIME_DEPENDENCIES in kinds:
+                serialized_element["runtime-dependencies"] = [
+                    e._get_full_name() for e in element._dependencies(_Scope.RUN, recurse=False)
+                ]
+
+            elements.append(serialized_element)
+
+        return elements
+
     # show_pipeline()
     #
     # Display a list of elements in the specified format.
@@ -359,30 +473,23 @@ def show_pipeline(self, dependencies, format_):
             line = p.fmt_subst(line, "full-key", key.full, fg="yellow", dim=dim_keys)
             line = p.fmt_subst(line, "description", description, fg="yellow", dim=dim_keys)
 
-            try:
-                if not element._has_all_sources_resolved():
-                    line = p.fmt_subst(line, "state", "no reference", fg="red")
-                else:
-                    if element.get_kind() == "junction":
-                        line = p.fmt_subst(line, "state", "junction", fg="magenta")
-                    elif not element._can_query_cache():
-                        line = p.fmt_subst(line, "state", "waiting", fg="blue")
-                    elif element._cached_failure():
-                        line = p.fmt_subst(line, "state", "failed", fg="red")
-                    elif element._cached_success():
-                        line = p.fmt_subst(line, "state", "cached", fg="magenta")
-                    elif not element._can_query_source_cache():
-                        line = p.fmt_subst(line, "state", "waiting", fg="blue")
-                    elif element._fetch_needed():
-                        line = p.fmt_subst(line, "state", "fetch needed", fg="red")
-                    elif element._buildable():
-                        line = p.fmt_subst(line, "state", "buildable", fg="green")
-                    else:
-                        line = p.fmt_subst(line, "state", "waiting", fg="blue")
-            except BstError as e:
-                # Provide context to plugin error
-                e.args = ("Failed to determine state for {}: {}".format(element._get_full_name(), str(e)),)
-                raise e
+            element_state = self._read_state(element)
+            if element_state == _ElementState.NO_REFERENCE:
+                line = p.fmt_subst(line, "state", "no reference", fg="red")
+            elif element_state == _ElementState.JUNCTION:
+                line = p.fmt_subst(line, "state", "junction", fg="magenta")
+            elif element_state == _ElementState.FAILED:
+                line = p.fmt_subst(line, "state", "failed", fg="red")
+            elif element_state == _ElementState.CACHED:
+                line = p.fmt_subst(line, "state", "cached", fg="magenta")
+            elif element_state == _ElementState.WAITING:
+                line = p.fmt_subst(line, "state", "waiting", fg="blue")
+            elif element_state == _ElementState.FETCH_NEEDED:
+                line = p.fmt_subst(line, "state", "fetch needed", fg="red")
+            elif element_state == _ElementState.BUILDABLE:
+                line = p.fmt_subst(line, "state", "buildable", fg="green")
+            else:
+                raise BstError(f"Unreachable State: {element_state}")
 
             # Element configuration
             if "%{config" in format_:

src/buildstream/types.py

@@ -311,6 +311,70 @@ def __str__(self):
         return str(self.value)
 
 
+# _ElementKind()
+#
+# Used to indicate which values you are interested in when dumping the
+# pipeline to a serializable format.
+class _ElementKind(FastEnum):
+
+    # Show all available pipeline information
+    ALL = "all"
+
+    # Show the short version of an element key
+    KEY = "key"
+
+    # Show the full key information
+    KEY_FULL = "key-full"
+
+    # Show the Element State
+    STATE = "state"
+
+    # Show source provenance information
+    SOURCES = "source"
+
+    # Dependencies
+    DEPENDENCIES = "dependences"
+
+    # Build dependencies
+    BUILD_DEPENDENCIES = "build-dependencies"
+
+    # Runtime dependencies
+    RUNTIME_DEPENDENCIES = "runtime-dependencies"
+
+    # CAS Artifacts
+    CAS_ARTIFACTS = "cas-artifacts"
+
+    # Element Variables
+    VARIABLES = "variables"
+
+    # Element Environment
+    ENVIRONMENT = "environment"
+
+
+# Used to indicate the state of a given element
+class _ElementState(FastEnum):
+    # Cannot determine the element state
+    NO_REFERENCE = "no-reference"
+
+    # The element has failed
+    FAILED = "failed"
+
+    # The element is a junction
+    JUNCTION = "junction"
+
+    # The element is waiting
+    WAITING = "waiting"
+
+    # The element is cached
+    CACHED = "cached"
+
+    # The element needs to be loaded from a remote source
+    FETCH_NEEDED = "fetch-needed"
+
+    # The element my be built
+    BUILDABLE = "buildable"
+
+
 # _ProjectInformation()
 #
 # A descriptive object about a project.