From 579d6e48b447f5c2076668c9ce6cb273e8bc916e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pablo=20Barb=C3=A1chano?= Date: Tue, 17 Sep 2024 08:55:42 +0200 Subject: [PATCH 1/2] buildkite: fix agents passed as a list instead of dict MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 19e90517fa9f12b740ba41393dbd6a8352c270f7 assumed the `agents` is a dict, which was true in all cases except in one pipeline. Fixes: 19e90517fa9f12b740ba41393dbd6a8352c270f7 Signed-off-by: Pablo Barbáchano --- .buildkite/pipeline_cpu_template.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.buildkite/pipeline_cpu_template.py b/.buildkite/pipeline_cpu_template.py index 2a182ed6e4b..ac6f223d95a 100755 --- a/.buildkite/pipeline_cpu_template.py +++ b/.buildkite/pipeline_cpu_template.py @@ -117,11 +117,11 @@ def group_snapshot_restore(test_step): BkStep.COMMAND: restore_commands, BkStep.LABEL: restore_label, BkStep.TIMEOUT: test_step["restore"][BkStep.TIMEOUT], - "agents": [ - f"instance={restore_instance}", - f"kv={restore_kv}", - f"os={restore_os}", - ], + "agents": { + "instance": restore_instance, + "kv": restore_kv, + "os": restore_os, + }, } ) From 84b2c8d686f6efad577b0600e8c7c3401a70c312 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pablo=20Barb=C3=A1chano?= Date: Tue, 17 Sep 2024 09:33:46 +0200 Subject: [PATCH 2/2] buildkite: make build steps unique MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If we have more than one build step in a pipeline, Buildkite complains that they are not unique. fatal: Failed to upload and process pipeline: Pipeline upload rejected: The key "build_x86_64" has already been used by another step in this build We had a similar issue with shared build tarballs, so just reuse the tarball name as key. Fixes: 19e90517fa9f12b740ba41393dbd6a8352c270f7 Signed-off-by: Pablo Barbáchano --- .buildkite/common.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/.buildkite/common.py b/.buildkite/common.py index 7119b79ee76..7937738d81f 100644 --- a/.buildkite/common.py +++ b/.buildkite/common.py @@ -259,7 +259,7 @@ def __init__(self, with_build_step=True, **kwargs): if with_build_step: build_cmds, self.shared_build = shared_build() self.build_group_per_arch( - "🏗️ Build", build_cmds, depends_on_build=False, key_prefix="build" + "🏗️ Build", build_cmds, depends_on_build=False, set_key=True ) else: self.shared_build = None @@ -297,8 +297,8 @@ def _adapt_group(self, group): for step in group["steps"]: step["command"] = prepend + step["command"] if self.shared_build is not None: - step["depends_on"] = ( - "build_" + DEFAULT_INSTANCES[step["agents"]["instance"]] + step["depends_on"] = self.build_key( + DEFAULT_INSTANCES[step["agents"]["instance"]] ) return group @@ -314,22 +314,26 @@ def build_group(self, *args, **kwargs): group(*args, **combined), depends_on_build=depends_on_build ) + def build_key(self, arch): + """Return the Buildkite key for the build step, for the specified arch""" + return self.shared_build.replace("$(uname -m)", arch).replace(".tar.gz", "") + def build_group_per_arch(self, label, *args, **kwargs): """ Build a group, parametrizing over the architectures only. kwargs consumed by this method and not passed down to `group`: - `depends_on_build` (default: `True`): Whether the steps in this group depend on the artifacts from the shared compilation steps - - `key_prefix`: If set, causes the generated steps to have a "key" field set to f"{key_prefix}_{$ARCH}". + - `set_key`: If True, causes the generated steps to have a "key" field """ depends_on_build = kwargs.pop("depends_on_build", True) - key_prefix = kwargs.pop("key_prefix", None) + set_key = kwargs.pop("set_key", None) combined = overlay_dict(self.per_arch, kwargs) grp = group(label, *args, **combined) - if key_prefix: + if set_key: for step in grp["steps"]: - step["key"] = ( - key_prefix + "_" + DEFAULT_INSTANCES[step["agents"]["instance"]] + step["key"] = self.build_key( + DEFAULT_INSTANCES[step["agents"]["instance"]] ) return self.add_step(grp, depends_on_build=depends_on_build)