Merge branch 'main' into 4534

Author: roypat

.buildkite/common.py

@@ -13,19 +13,18 @@
 import subprocess
 from pathlib import Path
 
-DEFAULT_INSTANCES = [
-    "c5n.metal",  # Intel Skylake
-    "m5n.metal",  # Intel Cascade Lake
-    "m6i.metal",  # Intel Icelake
-    "m6a.metal",  # AMD Milan
-    "m6g.metal",  # Graviton2
-    "m7g.metal",  # Graviton3
-]
+DEFAULT_INSTANCES = {
+    "c5n.metal": "x86_64",  # Intel Skylake
+    "m5n.metal": "x86_64",  # Intel Cascade Lake
+    "m6i.metal": "x86_64",  # Intel Icelake
+    "m6a.metal": "x86_64",  # AMD Milan
+    "m6g.metal": "aarch64",  # Graviton2
+    "m7g.metal": "aarch64",  # Graviton3
+}
 
 DEFAULT_PLATFORMS = [
     ("al2", "linux_5.10"),
-    # TODO: unpin 6.1 AMI once the bug is fixed
-    ("al2023", "linux_6.1-pinned"),
+    ("al2023", "linux_6.1"),
 ]
 
 
@@ -146,7 +145,7 @@ def __call__(self, parser, namespace, value, option_string=None):
     "--instances",
     required=False,
     nargs="+",
-    default=DEFAULT_INSTANCES,
+    default=DEFAULT_INSTANCES.keys(),
 )
 COMMON_PARSER.add_argument(
     "--platforms",
@@ -233,7 +232,7 @@ class BKPipeline:
 
     parser = COMMON_PARSER
 
-    def __init__(self, initial_steps=None, with_build_step=True, **kwargs):
+    def __init__(self, with_build_step=True, **kwargs):
         self.steps = []
         self.args = args = self.parser.parse_args()
         # Retry one time if agent was lost. This can happen if we terminate the
@@ -258,33 +257,22 @@ def __init__(self, initial_steps=None, with_build_step=True, **kwargs):
         # Build sharing
         if with_build_step:
             build_cmds, self.shared_build = shared_build()
-            step_build = group("🏗️ Build", build_cmds, **self.per_arch)
-            self.steps += [step_build, "wait"]
+            self.build_group_per_arch(
+                "🏗️ Build", build_cmds, depends_on_build=False, set_key=True
+            )
         else:
             self.shared_build = None
 
-        # If we run initial_steps before the "wait" step above, then a failure of the initial steps
-        # would result in the build not progressing past the "wait" step (as buildkite only proceeds past a wait step
-        # if everything before it passed). Thus put the initial steps after the "wait" step, but set `"depends_on": null`
-        # to start running them immediately (e.g. without waiting for the "wait" step to unblock).
-        #
-        # See also https://buildkite.com/docs/pipelines/dependencies#explicit-dependencies-in-uploaded-steps
-        if initial_steps:
-            for step in initial_steps:
-                step["depends_on"] = None
-
-            self.steps += initial_steps
-
-    def add_step(self, step, decorate=True):
+    def add_step(self, step, depends_on_build=True):
         """
         Add a step to the pipeline.
 
         https://buildkite.com/docs/pipelines/step-reference
 
         :param step: a Buildkite step
-        :param decorate: inject needed commands for sharing builds
+        :param depends_on_build: inject needed commands for sharing builds
         """
-        if decorate and isinstance(step, dict):
+        if depends_on_build and isinstance(step, dict):
             step = self._adapt_group(step)
         self.steps.append(step)
         return step
@@ -307,6 +295,10 @@ def _adapt_group(self, group):
 
         for step in group["steps"]:
             step["command"] = prepend + step["command"]
+            if self.shared_build is not None:
+                step["depends_on"] = self.build_key(
+                    DEFAULT_INSTANCES[step["agents"]["instance"]]
+                )
         return group
 
     def build_group(self, *args, **kwargs):
@@ -315,16 +307,34 @@ def build_group(self, *args, **kwargs):
 
         https://buildkite.com/docs/pipelines/group-step
         """
-        decorate = kwargs.pop("decorate", True)
+        depends_on_build = kwargs.pop("depends_on_build", True)
         combined = overlay_dict(self.per_instance, kwargs)
-        return self.add_step(group(*args, **combined), decorate=decorate)
+        return self.add_step(
+            group(*args, **combined), depends_on_build=depends_on_build
+        )
+
+    def build_key(self, arch):
+        """Return the Buildkite key for the build step, for the specified arch"""
+        return self.shared_build.replace("$(uname -m)", arch).replace(".tar.gz", "")
 
-    def build_group_per_arch(self, *args, **kwargs):
+    def build_group_per_arch(self, label, *args, **kwargs):
         """
         Build a group, parametrizing over the architectures only.
+
+        kwargs consumed by this method and not passed down to `group`:
+        - `depends_on_build` (default: `True`): Whether the steps in this group depend on the artifacts from the shared compilation steps
+        - `set_key`: If True, causes the generated steps to have a "key" field
         """
+        depends_on_build = kwargs.pop("depends_on_build", True)
+        set_key = kwargs.pop("set_key", None)
         combined = overlay_dict(self.per_arch, kwargs)
-        return self.add_step(group(*args, **combined))
+        grp = group(label, *args, **combined)
+        if set_key:
+            for step in grp["steps"]:
+                step["key"] = self.build_key(
+                    DEFAULT_INSTANCES[step["agents"]["instance"]]
+                )
+        return self.add_step(grp, depends_on_build=depends_on_build)
 
     def to_dict(self):
         """Render the pipeline as a dictionary."""

.buildkite/pipeline_cpu_template.py

@@ -34,7 +34,7 @@ class BkStep(str, Enum):
             "tools/devtool -y test --no-build -- -m no_block_pr integration_tests/functional/test_cpu_template_helper.py -k test_guest_cpu_config_change",
         ],
         BkStep.LABEL: "🖐️ fingerprint",
-        "instances": DEFAULT_INSTANCES,
+        "instances": DEFAULT_INSTANCES.keys(),
         "platforms": DEFAULT_PLATFORMS,
     },
     "cpuid_wrmsr": {
@@ -117,11 +117,11 @@ def group_snapshot_restore(test_step):
                 BkStep.COMMAND: restore_commands,
                 BkStep.LABEL: restore_label,
                 BkStep.TIMEOUT: test_step["restore"][BkStep.TIMEOUT],
-                "agents": [
-                    f"instance={restore_instance}",
-                    f"kv={restore_kv}",
-                    f"os={restore_os}",
-                ],
+                "agents": {
+                    "instance": restore_instance,
+                    "kv": restore_kv,
+                    "os": restore_os,
+                },
             }
         )
 

.buildkite/pipeline_pr.py

@@ -21,15 +21,17 @@
 pipeline = BKPipeline(
     priority=DEFAULT_PRIORITY,
     timeout_in_minutes=45,
-    initial_steps=[
-        {
-            "command": "./tools/devtool -y checkstyle",
-            "label": "🪶 Style",
-        },
-    ],
     with_build_step=not DOC_ONLY_CHANGE,
 )
 
+pipeline.add_step(
+    {
+        "command": "./tools/devtool -y checkstyle",
+        "label": "🪶 Style",
+    },
+    depends_on_build=False,
+)
+
 # run sanity build of devtool if Dockerfile is changed
 if any(x.parent.name == "devctr" for x in changed_files):
     pipeline.build_group_per_arch(
@@ -58,17 +60,17 @@
         platforms=[("al2", "linux_5.10")],
         timeout_in_minutes=300,
         **DEFAULTS_PERF,
-        decorate=False,
+        depends_on_build=False,
     )
     # modify Kani steps' label
     for step in kani_grp["steps"]:
         step["label"] = "🔍 Kani"
-    kani_grp["depends_on"] = None
 
 if run_all_tests(changed_files):
     pipeline.build_group(
         "📦 Build",
         pipeline.devtool_test(pytest_opts="integration_tests/build/"),
+        depends_on_build=False,
     )
 
     pipeline.build_group(

CHANGELOG.md

@@ -16,12 +16,20 @@ and this project adheres to
 
 ### Removed
 
+- [#4804](https://github.com/firecracker-microvm/firecracker/pull/4804): Drop
+  Support for guest kernel 4.14. Linux 4.14 reached end-of-life in
+  [January 2024](https://lore.kernel.org/lkml/2024011046-ecology-tiptoeing-ce50@gregkh/)
+  The minimum supported guest kernel now is 5.10.
+
 ### Fixed
 
 - [#4796](https://github.com/firecracker-microvm/firecracker/pull/4796): Fixed
   Vsock not notifying guest about `TRANSPORT_RESET_EVENT` event after snapshot
   restore. This resulted in guest waiting indefinitely on a connection which was
   reset during snapshot creation.
+- [#4790](https://github.com/firecracker-microvm/firecracker/pull/4790): v1.9.0
+  was missing most of the debugging information in the debuginfo file, due to a
+  change in the Cargo defaults. This has been corrected.
 
 ## \[1.9.0\]
 

Cargo.lock

@@ -27,3 +27,7 @@ panic = "abort"
 [profile.release]
 panic = "abort"
 lto = true
+strip = "none"
+
+[profile.bench]
+strip = "debuginfo"