diff --git a/.buildkite/common.py b/.buildkite/common.py index 6860608e341..a84e73f41da 100644 --- a/.buildkite/common.py +++ b/.buildkite/common.py @@ -78,6 +78,10 @@ def group(label, command, instances, platforms, **kwargs): """ # Use the 1st character of the group name (should be an emoji) label1 = label[0] + # if the emoji is in the form ":emoji:", pick the entire slug + if label.startswith(":") and ":" in label[1:]: + label1 = label[: label.index(":", 1) + 1] + steps = [] commands = command if isinstance(command, str): @@ -275,7 +279,7 @@ def __init__(self, with_build_step=True, **kwargs): if with_build_step: build_cmds, self.shared_build = shared_build() self.build_group_per_arch( - "🏗️ Build", build_cmds, depends_on_build=False, set_key=True + "🏗️ Build", build_cmds, depends_on_build=False, set_key=self.shared_build ) else: self.shared_build = None @@ -313,9 +317,25 @@ def _adapt_group(self, group): for step in group["steps"]: step["command"] = prepend + step["command"] if self.shared_build is not None: - step["depends_on"] = self.build_key( - get_arch_for_instance(step["agents"]["instance"]) - ) + if "depends_on" not in step: + step["depends_on"] = [] + elif isinstance(step["depends_on"], str): + step["depends_on"] = [step["depends_on"]] + elif isinstance(step["depends_on"], list): + pass + else: + raise ValueError( + f"depends_on should be a string or a list but is {type(step['depends_on'])}" + ) + + step["depends_on"].append(self.shared_build) + step["depends_on"] = [ + self.build_key( + dep, get_arch_for_instance(step["agents"]["instance"]) + ) + for dep in step["depends_on"] + ] + return group def build_group(self, *args, **kwargs): @@ -331,9 +351,9 @@ def build_group(self, *args, **kwargs): group(*args, **combined), depends_on_build=depends_on_build ) - def build_key(self, arch): + def build_key(self, key, arch): """Return the Buildkite key for the build step, for the specified arch""" - return self.shared_build.replace("$(uname -m)", arch).replace(".tar.gz", "") + return key.replace("$(uname -m)", arch).replace(".tar.gz", "") def build_group_per_arch(self, label, *args, **kwargs): """ @@ -341,7 +361,7 @@ def build_group_per_arch(self, label, *args, **kwargs): kwargs consumed by this method and not passed down to `group`: - `depends_on_build` (default: `True`): Whether the steps in this group depend on the artifacts from the shared compilation steps - - `set_key`: If True, causes the generated steps to have a "key" field + - `set_key`: If a string, causes the generated steps to have a "key" field replacing "$(uname -m)" with arch and removing trailing tar.gz """ depends_on_build = kwargs.pop("depends_on_build", True) set_key = kwargs.pop("set_key", None) @@ -350,7 +370,7 @@ def build_group_per_arch(self, label, *args, **kwargs): if set_key: for step in grp["steps"]: step["key"] = self.build_key( - get_arch_for_instance(step["agents"]["instance"]) + set_key, get_arch_for_instance(step["agents"]["instance"]) ) return self.add_step(grp, depends_on_build=depends_on_build) diff --git a/.buildkite/pipeline_coverage.py b/.buildkite/pipeline_coverage.py new file mode 100755 index 00000000000..8ca73591c00 --- /dev/null +++ b/.buildkite/pipeline_coverage.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 +# Copyright 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Generate Buildkite pipelines dynamically""" + +from common import BKPipeline + +pipeline = BKPipeline(with_build_step=False) + +pipeline.build_group( + ":coverage: Coverage", + pipeline.devtool_test( + devtool_opts="--no-build", + pytest_opts="integration_tests/build/test_coverage.py", + ), +) +print(pipeline.to_json()) diff --git a/.buildkite/pipeline_docker_popular.py b/.buildkite/pipeline_docker_popular.py new file mode 100755 index 00000000000..2bcbe2eed56 --- /dev/null +++ b/.buildkite/pipeline_docker_popular.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 +# Copyright 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +""" +Buildkite pipeline for testing popular Docker containers +""" + +from common import BKPipeline, random_str + +pipeline = BKPipeline() + +ROOTFS_TAR = f"rootfs_$(uname -m)_{random_str(k=8)}.tar.gz" + +pipeline.build_group_per_arch( + ":ship: Rootfs build", + [ + "sudo yum install -y systemd-container", + "cd tools/test-popular-containers", + "sudo ./build_rootfs.sh", + f'tar czf "{ROOTFS_TAR}" *.ext4 *.id_rsa', + f'buildkite-agent artifact upload "{ROOTFS_TAR}"', + ], + depends_on_build=False, + set_key=ROOTFS_TAR, +) + +pipeline.build_group( + ":whale: Docker Popular Containers", + [ + "./tools/devtool download_ci_artifacts", + f'buildkite-agent artifact download "{ROOTFS_TAR}" .', + f'tar xzf "{ROOTFS_TAR}" -C tools/test-popular-containers', + './tools/devtool sh "cd ./tools/test-popular-containers; PYTHONPATH=../../tests ./test-docker-rootfs.py"', + ], + depends_on=ROOTFS_TAR, +) + +print(pipeline.to_json()) diff --git a/tools/test-popular-containers/build_rootfs.sh b/tools/test-popular-containers/build_rootfs.sh index 91131829e38..22c28b9cd88 100755 --- a/tools/test-popular-containers/build_rootfs.sh +++ b/tools/test-popular-containers/build_rootfs.sh @@ -24,6 +24,7 @@ function make_rootfs { ssh-keygen -f id_rsa -N "" fi cp id_rsa $rootfs.id_rsa + chmod a+r $rootfs.id_rsa truncate -s "$SIZE" "$IMG" mkfs.ext4 -F "$IMG" -d $LABEL