wip

Author: joshua-zivkovic

src/buildstream/_artifact.py

@@ -37,6 +37,7 @@
 from .sandbox._config import SandboxConfig
 from ._variables import Variables
 
+
 # An Artifact class to abstract artifact operations
 # from the Element class
 #

src/buildstream/_elementsources.py

@@ -32,6 +32,7 @@
 
     # pylint: enable=cyclic-import
 
+
 # An ElementSources object represents the combined sources of an element.
 class ElementSources:
     def __init__(self, context: Context, project: "Project", plugin: Plugin):

src/buildstream/_frontend/cli.py

@@ -84,6 +84,7 @@ def __repr__(self):
 #            Override of click's main entry point                #
 ##################################################################
 
+
 # search_command()
 #
 # Helper function to get a command and context object

src/buildstream/element.py

@@ -72,14 +72,30 @@
 from itertools import chain
 import string
 from threading import Lock
-from typing import cast, TYPE_CHECKING, Dict, Iterator, Iterable, List, Optional, Set, Sequence
+from typing import (
+    cast,
+    TYPE_CHECKING,
+    Dict,
+    Iterator,
+    Iterable,
+    List,
+    Optional,
+    Set,
+    Sequence,
+)
 
 from pyroaring import BitMap  # pylint: disable=no-name-in-module
 
 from . import _yaml
 from ._variables import Variables
 from ._versions import BST_CORE_ARTIFACT_VERSION
-from ._exceptions import BstError, LoadError, ImplError, SourceCacheError, CachedFailure
+from ._exceptions import (
+    BstError,
+    LoadError,
+    ImplError,
+    SourceCacheError,
+    CachedFailure,
+)
 from .exceptions import ErrorDomain, LoadErrorReason
 from .utils import FileListResult, BST_ARBITRARY_TIMESTAMP
 from . import utils
@@ -90,7 +106,13 @@
 from .sandbox import _SandboxFlags, SandboxCommandError
 from .sandbox._config import SandboxConfig
 from .sandbox._sandboxremote import SandboxRemote
-from .types import _Scope, _CacheBuildTrees, _KeyStrength, OverlapAction, _DisplayKey
+from .types import (
+    _Scope,
+    _CacheBuildTrees,
+    _KeyStrength,
+    OverlapAction,
+    _DisplayKey,
+)
 from ._artifact import Artifact
 from ._elementproxy import ElementProxy
 from ._elementsources import ElementSources
@@ -136,7 +158,13 @@ def __init__(
         collect: Optional[str] = None,
         temporary: bool = False,
     ):
-        super().__init__(message, detail=detail, domain=ErrorDomain.ELEMENT, reason=reason, temporary=temporary)
+        super().__init__(
+            message,
+            detail=detail,
+            domain=ErrorDomain.ELEMENT,
+            reason=reason,
+            temporary=temporary,
+        )
 
         self.collect = collect
 
@@ -438,7 +466,10 @@ def sources(self) -> Iterator["Source"]:
         return self.__sources.sources()
 
     def dependencies(
-        self, selection: Optional[Sequence["Element"]] = None, *, recurse: bool = True
+        self,
+        selection: Optional[Sequence["Element"]] = None,
+        *,
+        recurse: bool = True,
     ) -> Iterator["Element"]:
         """A generator function which yields the build dependencies of the given element.
 
@@ -529,7 +560,8 @@ def node_subst_vars(self, node: "ScalarNode") -> str:
         """
         # FIXME: remove this
         warnings.warn(
-            "configuration is now automatically expanded, this is a no-op and will be removed.", DeprecationWarning
+            "configuration is now automatically expanded, this is a no-op and will be removed.",
+            DeprecationWarning,
         )
         return node.as_str()
 
@@ -550,12 +582,17 @@ def node_subst_sequence_vars(self, node: "SequenceNode[ScalarNode]") -> List[str
         """
         # FIXME: remove this
         warnings.warn(
-            "configuration is now automatically expanded, this is a no-op and will be removed.", DeprecationWarning
+            "configuration is now automatically expanded, this is a no-op and will be removed.",
+            DeprecationWarning,
         )
         return node.as_str_list()
 
     def compute_manifest(
-        self, *, include: Optional[List[str]] = None, exclude: Optional[List[str]] = None, orphans: bool = True
+        self,
+        *,
+        include: Optional[List[str]] = None,
+        exclude: Optional[List[str]] = None,
+        orphans: bool = True,
     ) -> str:
         """Compute and return this element's selective manifest
 
@@ -653,7 +690,12 @@ def stage_artifact(
         #
         with overlap_collector.session(action, path):
             result = self._stage_artifact(
-                sandbox, path=path, action=action, include=include, exclude=exclude, orphans=orphans
+                sandbox,
+                path=path,
+                action=action,
+                include=include,
+                exclude=exclude,
+                orphans=orphans,
             )
 
         return result
@@ -698,7 +740,14 @@ def stage_dependency_artifacts(
 
         with overlap_collector.session(action, path):
             for dep in self.dependencies(selection):
-                dep._stage_artifact(sandbox, path=path, include=include, exclude=exclude, orphans=orphans, owner=self)
+                dep._stage_artifact(
+                    sandbox,
+                    path=path,
+                    include=include,
+                    exclude=exclude,
+                    orphans=orphans,
+                    owner=self,
+                )
 
     def integrate(self, sandbox: "Sandbox") -> None:
         """Integrate currently staged filesystem against this artifact.
@@ -718,7 +767,12 @@ def integrate(self, sandbox: "Sandbox") -> None:
         if bstdata is not None:
             with sandbox.batch():
                 for command in bstdata.get_str_list("integration-commands", []):
-                    sandbox.run(["sh", "-e", "-c", command], env=environment, cwd="/", label=command)
+                    sandbox.run(
+                        ["sh", "-e", "-c", command],
+                        env=environment,
+                        cwd="/",
+                        label=command,
+                    )
 
     def stage_sources(self, sandbox: "Sandbox", directory: str) -> None:
         """Stage this element's sources to a directory in the sandbox
@@ -888,7 +942,10 @@ def visit(element, scope, visited):
                     visited[0].add(element._unique_id)
                     visited[1].add(element._unique_id)
 
-                    for dep in chain(element.__build_dependencies, element.__runtime_dependencies):
+                    for dep in chain(
+                        element.__build_dependencies,
+                        element.__runtime_dependencies,
+                    ):
                         if dep._unique_id not in visited[0] and dep._unique_id not in visited[1]:
                             yield from visit(dep, _Scope.ALL, visited)
 
@@ -987,7 +1044,11 @@ def _stage_artifact(
                 "No artifacts have been cached yet for that element\n"
                 + "Try building the element first with `bst build`\n"
             )
-            raise ElementError("No artifacts to stage", detail=detail, reason="uncached-checkout-attempt")
+            raise ElementError(
+                "No artifacts to stage",
+                detail=detail,
+                reason="uncached-checkout-attempt",
+            )
 
         # Time to use the artifact, check once more that it's there
         self.__assert_cached()
@@ -1030,10 +1091,26 @@ def _stage_artifact(
     #                              yet produced artifacts, or if forbidden overlaps
     #                              occur.
     #
-    def _stage_dependency_artifacts(self, sandbox, scope, *, path=None, include=None, exclude=None, orphans=True):
+    def _stage_dependency_artifacts(
+        self,
+        sandbox,
+        scope,
+        *,
+        path=None,
+        include=None,
+        exclude=None,
+        orphans=True,
+    ):
         with self._overlap_collectors[sandbox].session(OverlapAction.WARNING, path):
             for dep in self._dependencies(scope):
-                dep._stage_artifact(sandbox, path=path, include=include, exclude=exclude, orphans=orphans, owner=self)
+                dep._stage_artifact(
+                    sandbox,
+                    path=path,
+                    include=include,
+                    exclude=exclude,
+                    orphans=orphans,
+                    owner=self,
+                )
 
     # _new_from_load_element():
     #
@@ -1519,7 +1596,11 @@ def _stage_sources_at(self, vdirectory):
                 import_dir = staged_sources
 
             # Set update_mtime to ensure deterministic mtime of sources at build time
-            vdirectory._import_files_internal(import_dir, update_mtime=BST_ARBITRARY_TIMESTAMP, collect_result=False)
+            vdirectory._import_files_internal(
+                import_dir,
+                update_mtime=BST_ARBITRARY_TIMESTAMP,
+                collect_result=False,
+            )
 
         # Ensure deterministic owners of sources at build time
         vdirectory._set_deterministic_user()
@@ -1716,7 +1797,10 @@ def _assemble(self):
 
                 # Print the environment at the beginning of the log file.
                 env_dump = _yaml.roundtrip_dump_string(sandbox._get_configured_environment() or self.get_environment())
-                self.log("Build environment for element {}".format(self.name), detail=env_dump)
+                self.log(
+                    "Build environment for element {}".format(self.name),
+                    detail=env_dump,
+                )
 
                 # Step 2 - Stage
                 self.__stage(sandbox)
@@ -1925,7 +2009,12 @@ def _load_artifact(self, *, pull, strict=None):
             return False
 
         # In non-strict mode retry with weak cache key
-        artifact = Artifact(self, context, strict_key=self.__strict_cache_key, weak_key=self.__weak_cache_key)
+        artifact = Artifact(
+            self,
+            context,
+            strict_key=self.__strict_cache_key,
+            weak_key=self.__weak_cache_key,
+        )
         artifact.query_cache()
 
         # Attempt to pull artifact with the weak cache key
@@ -2060,7 +2149,16 @@ def _push(self):
     #    usebuildtree (bool): Use the buildtree as its source
     #
     # Returns: Exit code
-    def _shell(self, scope=None, *, mounts=None, isolate=False, prompt=None, command=None, usebuildtree=False):
+    def _shell(
+        self,
+        scope=None,
+        *,
+        mounts=None,
+        isolate=False,
+        prompt=None,
+        command=None,
+        usebuildtree=False,
+    ):
 
         with self._prepare_sandbox(scope, shell=True, usebuildtree=usebuildtree) as sandbox:
             environment = sandbox._get_configured_environment() or self.get_environment()
@@ -2132,7 +2230,10 @@ def _open_workspace(self):
         # additional support from Source implementations.
         #
         os.makedirs(context.builddir, exist_ok=True)
-        with utils._tempdir(dir=context.builddir, prefix="workspace-{}".format(self.normal_name)) as temp:
+        with utils._tempdir(
+            dir=context.builddir,
+            prefix="workspace-{}".format(self.normal_name),
+        ) as temp:
             self.__sources.init_workspace(temp)
 
             # Now hardlink the files into the workspace target.
@@ -2283,7 +2384,8 @@ def _fetch(self, fetch_original=False):
                 self.__sources.stage_and_cache()
             except (SourceCacheError, DirectoryError) as e:
                 raise ElementError(
-                    "Error trying to stage sources for {}: {}".format(self.name, e), reason="stage-sources-fail"
+                    "Error trying to stage sources for {}: {}".format(self.name, e),
+                    reason="stage-sources-fail",
                 )
 
     # _calculate_cache_key():
@@ -2671,7 +2773,11 @@ def __load_sources(self, load_element):
     #
     def __get_dependency_artifact_names(self):
         return [
-            os.path.join(dep.project_name, _get_normal_name(dep.name), dep._get_cache_key())
+            os.path.join(
+                dep.project_name,
+                _get_normal_name(dep.name),
+                dep._get_cache_key(),
+            )
             for dep in self._dependencies(_Scope.BUILD)
         ]
 
@@ -3154,7 +3260,8 @@ def __init_splits(self):
         splits = bstdata.get_mapping("split-rules")
         self.__splits = {
             domain: re.compile(
-                "^(?:" + "|".join([utils._glob2re(r) for r in rules.as_str_list()]) + ")$", re.MULTILINE | re.DOTALL
+                "^(?:" + "|".join([utils._glob2re(r) for r in rules.as_str_list()]) + ")$",
+                re.MULTILINE | re.DOTALL,
             )
             for domain, rules in splits.items()
         }
@@ -3317,9 +3424,15 @@ def __update_cache_keys(self):
             # encode the dependency's weak cache key instead of it's name.
             #
             dependencies = [
-                [e.project_name, e.name, e._get_cache_key(strength=_KeyStrength.WEAK)]
-                if self.BST_STRICT_REBUILD or e in self.__strict_dependencies
-                else [e.project_name, e.name]
+                (
+                    [
+                        e.project_name,
+                        e.name,
+                        e._get_cache_key(strength=_KeyStrength.WEAK),
+                    ]
+                    if self.BST_STRICT_REBUILD or e in self.__strict_dependencies
+                    else [e.project_name, e.name]
+                )
                 for e in self._dependencies(_Scope.BUILD)
             ]
             self.__weak_cache_key = self._calculate_cache_key(dependencies)

src/buildstream/plugin.py

@@ -160,6 +160,7 @@
 _STR_BYTES_PATH = Union[str, bytes, "os.PathLike[str]", "os.PathLike[bytes]"]
 _CMD = Union[_STR_BYTES_PATH, Sequence[_STR_BYTES_PATH]]
 
+
 # _background_job_wrapper()
 #
 # Wrapper for running jobs in the background, transparently for users