Merge branch 'apache:master' into master

Author: kotborealis

.github/compose/ci.buildgrid.yml

@@ -3,13 +3,9 @@
 #
 # Spins-up a unnamed and unauthenticated grid:
 #  - Controller + CAS + AC at http://localhost:50051
-#  - Ref. + CAS at: http://localhost:50052
 #
 # BuildStream configuration snippet:
 #
-#    artifacts:
-#      url: http://localhost:50052
-#      push: true
 #    remote-execution:
 #      execution-service:
 #        url: http://localhost:50051
@@ -25,6 +21,26 @@
 version: "3.2"
 
 services:
+  database:
+    image: registry.gitlab.com/buildgrid/buildgrid.hub.docker.com/buildgrid-postgres:nightly
+    environment:
+      POSTGRES_USER: bgd
+      POSTGRES_PASSWORD: insecure
+      POSTGRES_DB: bgd
+    volumes:
+      - type: volume
+        source: db
+        target: /var/lib/postgresql
+    networks:
+      - grid
+    ports:
+      - "5432:5432"
+    healthcheck:
+      test: ["CMD", "pg_isready", "-U", "bgd"]
+      interval: 1s
+      timeout: 5s
+      retries: 10
+
   controller:
     image: registry.gitlab.com/buildgrid/buildgrid.hub.docker.com/buildgrid:nightly
     command: [
@@ -34,6 +50,9 @@ services:
       - 50051:50051
     networks:
       - grid
+    depends_on:
+      database:
+        condition: service_healthy
 
   bot:
     image: registry.gitlab.com/buildgrid/buildgrid.hub.docker.com/buildbox:nightly
@@ -49,19 +68,10 @@ services:
     networks:
       - grid
 
-  storage:
-    image: registry.gitlab.com/buildgrid/buildgrid.hub.docker.com/buildgrid:nightly
-    command: [
-      "bgd", "server", "start", "-v",
-      "/etc/buildgrid/artifacts.yml"]
-    ports:
-      - 50052:50052
-    networks:
-      - grid
-
 networks:
   grid:
     driver: bridge
 
 volumes:
   cache:
+  db:

.github/compose/ci.docker-compose.yml

@@ -58,9 +58,7 @@ services:
     command: tox -vvvvv -- --color=yes --remote-execution
     environment:
       TOXENV: ${CI_TOXENV_MAIN}
-      ARTIFACT_CACHE_SERVICE: http://localhost:50052
       REMOTE_EXECUTION_SERVICE: http://localhost:50051
-      SOURCE_CACHE_SERVICE: http://localhost:50052
 
     # We need to use host networking mode in order to be able to
     # properly resolve services exposed by adjacent containers.

src/buildstream/_artifact.py

@@ -325,6 +325,9 @@ def cache(
                 for key, value in sorted(sandbox_env.items()):
                     artifact.buildsandbox.environment.add(name=key, value=value)
 
+            for directory in buildsandbox._get_marked_directories():
+                artifact.buildsandbox.marked_directories.append(directory)
+
             artifact.buildsandbox.working_directory = buildsandbox._get_work_directory()
 
             for subsandbox in buildsandbox._get_subsandboxes():
@@ -699,7 +702,7 @@ def pull(self, *, pull_buildtrees):
     def configure_sandbox(self, sandbox):
         artifact = self._get_proto()
 
-        if artifact.buildsandbox and artifact.buildsandbox.environment:
+        if artifact.HasField("buildsandbox") and artifact.buildsandbox.environment:
             env = {}
             for env_var in artifact.buildsandbox.environment:
                 env[env_var.name] = env_var.value
@@ -708,8 +711,12 @@ def configure_sandbox(self, sandbox):
 
         sandbox.set_environment(env)
 
-        if artifact.buildsandbox and artifact.buildsandbox.working_directory:
-            sandbox.set_work_directory(artifact.buildsandbox.working_directory)
+        if artifact.HasField("buildsandbox"):
+            for directory in artifact.buildsandbox.marked_directories:
+                sandbox.mark_directory(directory)
+
+            if artifact.buildsandbox.working_directory:
+                sandbox.set_work_directory(artifact.buildsandbox.working_directory)
 
     #  load_proto()
     #

src/buildstream/_artifactcache.py

@@ -288,28 +288,34 @@ def _push_artifact_blobs(self, artifact, artifact_digest, remote):
         artifact_proto = artifact._get_proto()
 
         try:
-            if str(artifact_proto.files):
+            if artifact_proto.HasField("files"):
                 self.cas._send_directory(remote, artifact_proto.files)
 
-            if str(artifact_proto.buildtree):
+            if artifact_proto.HasField("buildtree"):
                 try:
                     self.cas._send_directory(remote, artifact_proto.buildtree)
                 except FileNotFoundError:
                     pass
 
-            if str(artifact_proto.buildroot):
+            if artifact_proto.HasField("sources"):
+                try:
+                    self.cas._send_directory(remote, artifact_proto.sources)
+                except FileNotFoundError:
+                    pass
+
+            if artifact_proto.HasField("buildroot"):
                 try:
                     self.cas._send_directory(remote, artifact_proto.buildroot)
                 except FileNotFoundError:
                     pass
 
-            if artifact_proto.buildsandbox:
+            if artifact_proto.HasField("buildsandbox"):
                 for subsandbox_digest in artifact_proto.buildsandbox.subsandbox_digests:
                     self.cas._send_directory(remote, subsandbox_digest)
 
             digests = [artifact_digest, artifact_proto.low_diversity_meta, artifact_proto.high_diversity_meta]
 
-            if str(artifact_proto.public_data):
+            if artifact_proto.HasField("public_data"):
                 digests.append(artifact_proto.public_data)
 
             for log_file in artifact_proto.logs:
@@ -357,22 +363,25 @@ def _push_artifact_proto(self, element, artifact, artifact_digest, remote):
             raise ArtifactError("{}".format(e), temporary=True) from e
 
         referenced_directories = []
-        if artifact_proto.files:
+        if artifact_proto.HasField("files"):
             referenced_directories.append(artifact_proto.files)
-        if artifact_proto.buildtree:
+        if artifact_proto.HasField("buildtree"):
             referenced_directories.append(artifact_proto.buildtree)
-        if artifact_proto.sources:
+        if artifact_proto.HasField("sources"):
             referenced_directories.append(artifact_proto.sources)
-        if artifact_proto.buildroot:
+        if artifact_proto.HasField("buildroot"):
             referenced_directories.append(artifact_proto.buildroot)
-        if artifact_proto.buildsandbox:
+        if artifact_proto.HasField("buildsandbox"):
             for subsandbox_digest in artifact_proto.buildsandbox.subsandbox_digests:
                 referenced_directories.append(subsandbox_digest)
 
         referenced_blobs = [artifact_proto.low_diversity_meta, artifact_proto.high_diversity_meta] + [
             log_file.digest for log_file in artifact_proto.logs
         ]
 
+        if artifact_proto.HasField("public_data"):
+            referenced_blobs.append(artifact_proto.public_data)
+
         try:
             remote.push_blob(
                 uris,
@@ -418,20 +427,22 @@ def _pull_artifact_storage(self, element, key, artifact_digest, remote, pull_bui
             with utils.save_file_atomic(artifact_path, mode="wb") as f:
                 f.write(artifact.SerializeToString())
 
-            if str(artifact.files):
+            if artifact.HasField("files"):
                 self.cas.fetch_directory(remote, artifact.files)
 
             if pull_buildtrees:
-                if str(artifact.buildtree):
+                if artifact.HasField("buildtree"):
                     self.cas.fetch_directory(remote, artifact.buildtree)
-                if str(artifact.buildroot):
+                if artifact.HasField("sources"):
+                    self.cas.fetch_directory(remote, artifact.sources)
+                if artifact.HasField("buildroot"):
                     self.cas.fetch_directory(remote, artifact.buildroot)
-                if artifact.buildsandbox:
+                if artifact.HasField("buildsandbox"):
                     for subsandbox_digest in artifact.buildsandbox.subsandbox_digests:
                         self.cas.fetch_directory(remote, subsandbox_digest)
 
             digests = [artifact.low_diversity_meta, artifact.high_diversity_meta]
-            if str(artifact.public_data):
+            if artifact.HasField("public_data"):
                 digests.append(artifact.public_data)
 
             for log_digest in artifact.logs:

src/buildstream/_frontend/cli.py

@@ -1302,12 +1302,29 @@ def artifact():
     ),
     help="The dependencies we also want to show",
 )
+@click.option(
+    "--artifact-remote",
+    "artifact_remotes",
+    type=RemoteSpecType(RemoteSpecPurpose.PULL),
+    multiple=True,
+    help="A remote for downloading artifacts (Since: 2.7)",
+)
+@click.option(
+    "--ignore-project-artifact-remotes",
+    is_flag=True,
+    help="Ignore remote artifact cache servers recommended by projects (Since: 2.7)",
+)
 @click.argument("artifacts", type=click.Path(), nargs=-1)
 @click.pass_obj
-def artifact_show(app, deps, artifacts):
-    """show the cached state of artifacts"""
+def artifact_show(app, deps, artifact_remotes, ignore_project_artifact_remotes, artifacts):
+    """Show the cached state of artifacts"""
     with app.initialized():
-        targets = app.stream.artifact_show(artifacts, selection=deps)
+        targets = app.stream.artifact_show(
+            artifacts,
+            selection=deps,
+            artifact_remotes=artifact_remotes,
+            ignore_project_artifact_remotes=ignore_project_artifact_remotes,
+        )
         click.echo(app.logger.show_state_of_artifacts(targets))
         sys.exit(0)
 

src/buildstream/_loader/loadelement.pyx

@@ -202,7 +202,7 @@ cdef class Dependency:
     # Args:
     #    other (Dependency): The dependency to merge into this one
     #
-    cdef merge(self, Dependency other):
+    cpdef merge(self, Dependency other):
         self.dep_type = self.dep_type | other.dep_type
         self.strict = self.strict or other.strict
 

src/buildstream/_loader/loader.py

@@ -495,8 +495,8 @@ def _load_file(self, filename, provenance_node, *, load_subprojects=True):
         # The loader queue is a stack of tuples
         # [0] is the LoadElement instance
         # [1] is a stack of Dependency objects to load
-        # [2] is a list of dependency names used to warn when all deps are loaded
-        loader_queue = [(top_element, list(reversed(dependencies)), [])]
+        # [2] is a Dict[LoadElement, Dependency] of loaded dependencies
+        loader_queue = [(top_element, list(reversed(dependencies)), {})]
 
         # Load all dependency files for the new LoadElement
         while loader_queue:
@@ -505,8 +505,6 @@ def _load_file(self, filename, provenance_node, *, load_subprojects=True):
 
                 # Process the first dependency of the last loaded element
                 dep = current_element[1].pop()
-                # And record its name for checking later
-                current_element[2].append(dep.name)
 
                 if dep.junction:
                     loader = self.get_loader(dep.junction, dep.node)
@@ -529,7 +527,7 @@ def _load_file(self, filename, provenance_node, *, load_subprojects=True):
                         dep_element.mark_fully_loaded()
 
                         dep_deps = extract_depends_from_node(dep_element.node)
-                        loader_queue.append((dep_element, list(reversed(dep_deps)), []))
+                        loader_queue.append((dep_element, list(reversed(dep_deps)), {}))
 
                         # Pylint is not very happy about Cython and can't understand 'node' is a 'MappingNode'
                         if dep_element.node.get_str(Symbol.KIND) == "junction":  # pylint: disable=no-member
@@ -542,7 +540,15 @@ def _load_file(self, filename, provenance_node, *, load_subprojects=True):
                 # LoadElement on the dependency and append the dependency to the owning
                 # LoadElement dependency list.
                 dep.set_element(dep_element)
-                current_element[0].dependencies.append(dep)  # pylint: disable=no-member
+
+                dep_dict = current_element[2]
+                if dep.element in dep_dict:
+                    # Duplicate LoadElement in dependency list, this can happen if a dependency is
+                    # a link element that points to an element that is already a dependency.
+                    dep_dict[dep.element].merge(dep)
+                else:
+                    current_element[0].dependencies.append(dep)  # pylint: disable=no-member
+                    dep_dict[dep.element] = dep
             else:
                 # And pop the element off the queue
                 loader_queue.pop()

src/buildstream/_protos/buildstream/v2/artifact.proto

@@ -90,6 +90,7 @@ message Artifact {
     repeated build.bazel.remote.execution.v2.Command.EnvironmentVariable environment = 1;
     string working_directory = 2;
     repeated build.bazel.remote.execution.v2.Digest subsandbox_digests = 3;
+    repeated string marked_directories = 4;
   };
   SandboxState buildsandbox = 18;  // optional
 }

src/buildstream/_protos/buildstream/v2/artifact_pb2.py

@@ -28,14 +28,16 @@ class Artifact(_message.Message):
         digest: _remote_execution_pb2.Digest
         def __init__(self, name: _Optional[str] = ..., digest: _Optional[_Union[_remote_execution_pb2.Digest, _Mapping]] = ...) -> None: ...
     class SandboxState(_message.Message):
-        __slots__ = ("environment", "working_directory", "subsandbox_digests")
+        __slots__ = ("environment", "working_directory", "subsandbox_digests", "marked_directories")
         ENVIRONMENT_FIELD_NUMBER: _ClassVar[int]
         WORKING_DIRECTORY_FIELD_NUMBER: _ClassVar[int]
         SUBSANDBOX_DIGESTS_FIELD_NUMBER: _ClassVar[int]
+        MARKED_DIRECTORIES_FIELD_NUMBER: _ClassVar[int]
         environment: _containers.RepeatedCompositeFieldContainer[_remote_execution_pb2.Command.EnvironmentVariable]
         working_directory: str
         subsandbox_digests: _containers.RepeatedCompositeFieldContainer[_remote_execution_pb2.Digest]
-        def __init__(self, environment: _Optional[_Iterable[_Union[_remote_execution_pb2.Command.EnvironmentVariable, _Mapping]]] = ..., working_directory: _Optional[str] = ..., subsandbox_digests: _Optional[_Iterable[_Union[_remote_execution_pb2.Digest, _Mapping]]] = ...) -> None: ...
+        marked_directories: _containers.RepeatedScalarFieldContainer[str]
+        def __init__(self, environment: _Optional[_Iterable[_Union[_remote_execution_pb2.Command.EnvironmentVariable, _Mapping]]] = ..., working_directory: _Optional[str] = ..., subsandbox_digests: _Optional[_Iterable[_Union[_remote_execution_pb2.Digest, _Mapping]]] = ..., marked_directories: _Optional[_Iterable[str]] = ...) -> None: ...
     VERSION_FIELD_NUMBER: _ClassVar[int]
     BUILD_SUCCESS_FIELD_NUMBER: _ClassVar[int]
     BUILD_ERROR_FIELD_NUMBER: _ClassVar[int]