chore: pythonify BK definitions for docker and coverage

.buildkite/common.py

@@ -78,6 +78,10 @@ def group(label, command, instances, platforms, **kwargs):
     """
     # Use the 1st character of the group name (should be an emoji)
     label1 = label[0]
+    # if the emoji is in the form ":emoji:", pick the entire slug
+    if label.startswith(":") and ":" in label[1:]:
+        label1 = label[: label.index(":", 1) + 1]
+
     steps = []
     commands = command
     if isinstance(command, str):
@@ -275,7 +279,7 @@ def __init__(self, with_build_step=True, **kwargs):
         if with_build_step:
             build_cmds, self.shared_build = shared_build()
             self.build_group_per_arch(
-                "🏗️ Build", build_cmds, depends_on_build=False, set_key=True
+                "🏗️ Build", build_cmds, depends_on_build=False, set_key=self.shared_build
             )
         else:
             self.shared_build = None
@@ -313,9 +317,25 @@ def _adapt_group(self, group):
         for step in group["steps"]:
             step["command"] = prepend + step["command"]
             if self.shared_build is not None:
-                step["depends_on"] = self.build_key(
-                    get_arch_for_instance(step["agents"]["instance"])
-                )
+                if "depends_on" not in step:
+                    step["depends_on"] = []
+                elif isinstance(step["depends_on"], str):
+                    step["depends_on"] = [step["depends_on"]]
+                elif isinstance(step["depends_on"], list):
+                    pass
+                else:
+                    raise ValueError(
+                        f"depends_on should be a string or a list but is {type(step['depends_on'])}"
+                    )
+
+                step["depends_on"].append(self.shared_build)
+                step["depends_on"] = [
+                    self.build_key(
+                        dep, get_arch_for_instance(step["agents"]["instance"])
+                    )
+                    for dep in step["depends_on"]
+                ]
+
         return group
 
     def build_group(self, *args, **kwargs):
@@ -331,17 +351,17 @@ def build_group(self, *args, **kwargs):
             group(*args, **combined), depends_on_build=depends_on_build
         )
 
-    def build_key(self, arch):
+    def build_key(self, key, arch):
         """Return the Buildkite key for the build step, for the specified arch"""
-        return self.shared_build.replace("$(uname -m)", arch).replace(".tar.gz", "")
+        return key.replace("$(uname -m)", arch).replace(".tar.gz", "")
 
     def build_group_per_arch(self, label, *args, **kwargs):
         """
         Build a group, parametrizing over the architectures only.
 
         kwargs consumed by this method and not passed down to `group`:
         - `depends_on_build` (default: `True`): Whether the steps in this group depend on the artifacts from the shared compilation steps
-        - `set_key`: If True, causes the generated steps to have a "key" field
+        - `set_key`: If a string, causes the generated steps to have a "key" field replacing "$(uname -m)" with arch and removing trailing tar.gz
         """
         depends_on_build = kwargs.pop("depends_on_build", True)
         set_key = kwargs.pop("set_key", None)
@@ -350,7 +370,7 @@ def build_group_per_arch(self, label, *args, **kwargs):
         if set_key:
             for step in grp["steps"]:
                 step["key"] = self.build_key(
-                    get_arch_for_instance(step["agents"]["instance"])
+                    set_key, get_arch_for_instance(step["agents"]["instance"])
                 )
         return self.add_step(grp, depends_on_build=depends_on_build)
 

.buildkite/pipeline_coverage.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python3
+# Copyright 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Generate Buildkite pipelines dynamically"""
+
+from common import BKPipeline
+
+pipeline = BKPipeline(with_build_step=False)
+
+pipeline.build_group(
+    ":coverage: Coverage",
+    pipeline.devtool_test(
+        devtool_opts="--no-build",
+        pytest_opts="integration_tests/build/test_coverage.py",
+    ),
+)
+print(pipeline.to_json())

.buildkite/pipeline_docker_popular.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+# Copyright 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""
+Buildkite pipeline for testing popular Docker containers
+"""
+
+from common import BKPipeline, random_str
+
+pipeline = BKPipeline()
+
+ROOTFS_TAR = f"rootfs_$(uname -m)_{random_str(k=8)}.tar.gz"
+
+pipeline.build_group_per_arch(
+    ":ship: Rootfs build",
+    [
+        "sudo yum install -y systemd-container",
+        "cd tools/test-popular-containers",
+        "sudo ./build_rootfs.sh",
+        f'tar czf "{ROOTFS_TAR}" *.ext4 *.id_rsa',
+        f'buildkite-agent artifact upload "{ROOTFS_TAR}"',
+    ],
+    depends_on_build=False,
+    set_key=ROOTFS_TAR,
+)
+
+pipeline.build_group(
+    ":whale: Docker Popular Containers",
+    [
+        "./tools/devtool download_ci_artifacts",
+        f'buildkite-agent artifact download "{ROOTFS_TAR}" .',
+        f'tar xzf "{ROOTFS_TAR}" -C tools/test-popular-containers',
+        './tools/devtool sh "cd ./tools/test-popular-containers; PYTHONPATH=../../tests ./test-docker-rootfs.py"',
+    ],
+    depends_on=ROOTFS_TAR,
+)
+
+print(pipeline.to_json())

tools/test-popular-containers/build_rootfs.sh

@@ -24,6 +24,7 @@ function make_rootfs {
         ssh-keygen -f id_rsa -N ""
     fi
     cp id_rsa $rootfs.id_rsa
+    chmod a+r $rootfs.id_rsa
 
     truncate -s "$SIZE" "$IMG"
     mkfs.ext4 -F "$IMG" -d $LABEL