chore: pythonify BK definitions for docker and coverage

We used to manage these definitions in the BK with no versioning or
source control, let's move them to python as all the others so that they
stay in sync with the latest and greatest updates to the supported
platforms and instance types.

Signed-off-by: Riccardo Mancini <[email protected]>

Author: Manciukic

.buildkite/common.py

@@ -78,6 +78,10 @@ def group(label, command, instances, platforms, **kwargs):
     """
     # Use the 1st character of the group name (should be an emoji)
     label1 = label[0]
+    # if the emoji is in the form ":emoji:", pick the entire slug
+    if label.startswith(":") and ":" in label[1:]:
+        label1 = label[: label.index(":", 1) + 1]
+
     steps = []
     commands = command
     if isinstance(command, str):
@@ -275,7 +279,7 @@ def __init__(self, with_build_step=True, **kwargs):
         if with_build_step:
             build_cmds, self.shared_build = shared_build()
             self.build_group_per_arch(
-                "🏗️ Build", build_cmds, depends_on_build=False, set_key=True
+                "🏗️ Build", build_cmds, depends_on_build=False, set_key=self.shared_build
             )
         else:
             self.shared_build = None
@@ -313,9 +317,25 @@ def _adapt_group(self, group):
         for step in group["steps"]:
             step["command"] = prepend + step["command"]
             if self.shared_build is not None:
-                step["depends_on"] = self.build_key(
-                    get_arch_for_instance(step["agents"]["instance"])
-                )
+                if "depends_on" not in step:
+                    step["depends_on"] = []
+                elif isinstance(step["depends_on"], str):
+                    step["depends_on"] = [step["depends_on"]]
+                elif isinstance(step["depends_on"], list):
+                    pass
+                else:
+                    raise ValueError(
+                        f"depends_on should be a string or a list but is {type(step['depends_on'])}"
+                    )
+
+                step["depends_on"].append(self.shared_build)
+                step["depends_on"] = [
+                    self.build_key(
+                        dep, get_arch_for_instance(step["agents"]["instance"])
+                    )
+                    for dep in step["depends_on"]
+                ]
+
         return group
 
     def build_group(self, *args, **kwargs):
@@ -331,17 +351,17 @@ def build_group(self, *args, **kwargs):
             group(*args, **combined), depends_on_build=depends_on_build
         )
 
-    def build_key(self, arch):
+    def build_key(self, key, arch):
         """Return the Buildkite key for the build step, for the specified arch"""
-        return self.shared_build.replace("$(uname -m)", arch).replace(".tar.gz", "")
+        return key.replace("$(uname -m)", arch).replace(".tar.gz", "")
 
     def build_group_per_arch(self, label, *args, **kwargs):
         """
         Build a group, parametrizing over the architectures only.
 
         kwargs consumed by this method and not passed down to `group`:
         - `depends_on_build` (default: `True`): Whether the steps in this group depend on the artifacts from the shared compilation steps
-        - `set_key`: If True, causes the generated steps to have a "key" field
+        - `set_key`: If a string, causes the generated steps to have a "key" field replacing "$(uname -m)" with arch and removing trailing tar.gz
         """
         depends_on_build = kwargs.pop("depends_on_build", True)
         set_key = kwargs.pop("set_key", None)
@@ -350,7 +370,7 @@ def build_group_per_arch(self, label, *args, **kwargs):
         if set_key:
             for step in grp["steps"]:
                 step["key"] = self.build_key(
-                    get_arch_for_instance(step["agents"]["instance"])
+                    set_key, get_arch_for_instance(step["agents"]["instance"])
                 )
         return self.add_step(grp, depends_on_build=depends_on_build)
 

.buildkite/pipeline_coverage.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python3
+# Copyright 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Generate Buildkite pipelines dynamically"""
+
+from common import BKPipeline
+
+pipeline = BKPipeline(with_build_step=False)
+
+pipeline.build_group(
+    ":coverage: Coverage",
+    pipeline.devtool_test(
+        devtool_opts="--no-build",
+        pytest_opts="integration_tests/build/test_coverage.py",
+    ),
+)
+print(pipeline.to_json())

.buildkite/pipeline_docker_popular.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+# Copyright 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""
+Buildkite pipeline for testing popular Docker containers
+"""
+
+from common import BKPipeline, random_str
+
+pipeline = BKPipeline()
+
+ROOTFS_TAR = f"rootfs_$(uname -m)_{random_str(k=8)}.tar.gz"
+
+pipeline.build_group_per_arch(
+    ":ship: Rootfs build",
+    [
+        "sudo yum install -y systemd-container",
+        "cd tools/test-popular-containers",
+        "sudo ./build_rootfs.sh",
+        f'tar czf "{ROOTFS_TAR}" *.ext4 *.id_rsa',
+        f'buildkite-agent artifact upload "{ROOTFS_TAR}"',
+    ],
+    depends_on_build=False,
+    set_key=ROOTFS_TAR,
+)
+
+pipeline.build_group(
+    ":whale: Docker Popular Containers",
+    [
+        "./tools/devtool download_ci_artifacts",
+        f'buildkite-agent artifact download "{ROOTFS_TAR}" .',
+        f'tar xzf "{ROOTFS_TAR}" -C tools/test-popular-containers',
+        './tools/devtool sh "cd ./tools/test-popular-containers; PYTHONPATH=../../tests ./test-docker-rootfs.py"',
+    ],
+    depends_on=ROOTFS_TAR,
+)
+
+print(pipeline.to_json())

tools/test-popular-containers/build_rootfs.sh

@@ -24,6 +24,7 @@ function make_rootfs {
         ssh-keygen -f id_rsa -N ""
     fi
     cp id_rsa $rootfs.id_rsa
+    chmod a+r $rootfs.id_rsa
 
     truncate -s "$SIZE" "$IMG"
     mkfs.ext4 -F "$IMG" -d $LABEL