Store sandbox state in Artifact proto

This is stored only when also storing the build tree. It is used to
configure the sandbox for `bst shell --use-buildtree`.

This is required for elements that create subsandboxes as part of
`configure_sandbox()`.

Author: juergbi

src/buildstream/_artifact.py

@@ -200,6 +200,7 @@ def get_extract_key(self):
     #    variables (Variables): The element's Variables
     #    environment (dict): dict of the element's environment variables
     #    sandboxconfig (SandboxConfig): The element's SandboxConfig
+    #    buildsandbox (Sandbox): The element's configured build sandbox
     #
     def cache(
         self,
@@ -213,6 +214,7 @@ def cache(
         variables,
         environment,
         sandboxconfig,
+        buildsandbox,
     ):
 
         context = self._context
@@ -317,6 +319,19 @@ def cache(
             rootvdir._import_files_internal(buildrootvdir, properties=properties, collect_result=False)
             artifact.buildroot.CopyFrom(rootvdir._get_digest())
 
+        if buildsandbox is not None:
+            sandbox_env = buildsandbox._get_configured_environment()
+            if sandbox_env:
+                for key, value in sorted(sandbox_env.items()):
+                    artifact.buildsandbox.environment.add(name=key, value=value)
+
+            artifact.buildsandbox.working_directory = buildsandbox._get_work_directory()
+
+            for subsandbox in buildsandbox._get_subsandboxes():
+                vdir = subsandbox.get_virtual_directory()
+                digest = artifact.buildsandbox.subsandbox_digests.add()
+                digest.CopyFrom(vdir._get_digest())
+
         os.makedirs(os.path.dirname(os.path.join(self._artifactdir, element.get_artifact_name())), exist_ok=True)
         keys = utils._deduplicate([self._cache_key, self._weak_cache_key])
         for key in keys:
@@ -681,6 +696,21 @@ def pull(self, *, pull_buildtrees):
 
         return True
 
+    def configure_sandbox(self, sandbox):
+        artifact = self._get_proto()
+
+        if artifact.buildsandbox and artifact.buildsandbox.environment:
+            env = {}
+            for env_var in artifact.buildsandbox.environment:
+                env[env_var.name] = env_var.value
+        else:
+            env = self.load_environment()
+
+        sandbox.set_environment(env)
+
+        if artifact.buildsandbox and artifact.buildsandbox.working_directory:
+            sandbox.set_work_directory(artifact.buildsandbox.working_directory)
+
     #  load_proto()
     #
     # Returns:

src/buildstream/_artifactcache.py

@@ -303,6 +303,10 @@ def _push_artifact_blobs(self, artifact, artifact_digest, remote):
                 except FileNotFoundError:
                     pass
 
+            if artifact_proto.buildsandbox:
+                for subsandbox_digest in artifact_proto.buildsandbox.subsandbox_digests:
+                    self.cas._send_directory(remote, subsandbox_digest)
+
             digests = [artifact_digest, artifact_proto.low_diversity_meta, artifact_proto.high_diversity_meta]
 
             if str(artifact_proto.public_data):
@@ -361,6 +365,9 @@ def _push_artifact_proto(self, element, artifact, artifact_digest, remote):
             referenced_directories.append(artifact_proto.sources)
         if artifact_proto.buildroot:
             referenced_directories.append(artifact_proto.buildroot)
+        if artifact_proto.buildsandbox:
+            for subsandbox_digest in artifact_proto.buildsandbox.subsandbox_digests:
+                referenced_directories.append(subsandbox_digest)
 
         referenced_blobs = [artifact_proto.low_diversity_meta, artifact_proto.high_diversity_meta] + [
             log_file.digest for log_file in artifact_proto.logs
@@ -419,6 +426,9 @@ def _pull_artifact_storage(self, element, key, artifact_digest, remote, pull_bui
                     self.cas.fetch_directory(remote, artifact.buildtree)
                 if str(artifact.buildroot):
                     self.cas.fetch_directory(remote, artifact.buildroot)
+                if artifact.buildsandbox:
+                    for subsandbox_digest in artifact.buildsandbox.subsandbox_digests:
+                        self.cas.fetch_directory(remote, subsandbox_digest)
 
             digests = [artifact.low_diversity_meta, artifact.high_diversity_meta]
             if str(artifact.public_data):

src/buildstream/element.py

@@ -1440,18 +1440,25 @@ def _prepare_sandbox(self, scope, shell=False, integrate=True, usebuildtree=Fals
         # pylint: disable-next=contextmanager-generator-missing-cleanup
         with self.__sandbox(config=self.__sandbox_config, allow_remote=False) as sandbox:
 
-            # Configure always comes first, and we need it.
-            self.__configure_sandbox(sandbox)
-
             if usebuildtree:
+                # Configure the sandbox from artifact metadata
+                self.__artifact.configure_sandbox(sandbox)
+
                 # Use the cached buildroot directly
                 buildrootvdir = self.__artifact.get_buildroot()
                 sandbox_vroot = sandbox.get_virtual_directory()
                 sandbox_vroot._import_files_internal(buildrootvdir, collect_result=False)
             elif shell and scope == _Scope.BUILD:
+                self.__configure_sandbox(sandbox)
                 # Stage what we need
                 self.__stage(sandbox)
             else:
+                # Runtime shell or `bst artifact checkout`
+
+                # Don't call `configure_sandbox()` as that may attempt to construct subsandboxes
+                # with build dependencies and we're not setting up a build sandbox.
+                sandbox.set_environment(self.get_environment())
+
                 # Stage deps in the sandbox root
                 with self.timed_activity("Staging dependencies", silent_nested=True), self.__collect_overlaps(sandbox):
                     self._stage_dependency_artifacts(sandbox, scope)
@@ -1787,6 +1794,7 @@ def _cache_artifact(self, sandbox, collect):
                 variables=self.__variables,
                 environment=self.__environment,
                 sandboxconfig=self.__sandbox_config,
+                buildsandbox=sandbox if buildrootvdir else None,
             )
 
         if collect is not None and collectvdir is None: