DO NOT MERGE! Just testing

.buildkite/common.py

@@ -6,28 +6,35 @@
 """
 
 import argparse
+import ast
 import json
 import os
 import random
 import string
 import subprocess
 from pathlib import Path
 
-DEFAULT_INSTANCES = {
-    "c5n.metal": "x86_64",  # Intel Skylake
-    "m5n.metal": "x86_64",  # Intel Cascade Lake
-    "m6i.metal": "x86_64",  # Intel Icelake
-    "m6a.metal": "x86_64",  # AMD Milan
-    "m6g.metal": "aarch64",  # Graviton2
-    "m7g.metal": "aarch64",  # Graviton3
-}
+DEFAULT_INSTANCES = [
+    "c5n.metal",  # Intel Skylake
+    "m5n.metal",  # Intel Cascade Lake
+    "m6i.metal",  # Intel Icelake
+    "m6a.metal",  # AMD Milan
+    "m7a.metal-48xl",  # AMD Genoa
+    "m6g.metal",  # Graviton2
+    "m7g.metal",  # Graviton3
+]
 
 DEFAULT_PLATFORMS = [
     ("al2", "linux_5.10"),
     ("al2023", "linux_6.1"),
 ]
 
 
+def get_arch_for_instance(instance):
+    """Return instance architecture"""
+    return "x86_64" if instance[2] != "g" else "aarch64"
+
+
 def overlay_dict(base: dict, update: dict):
     """Overlay a dict over a base one"""
     base = base.copy()
@@ -133,7 +140,8 @@ def __call__(self, parser, namespace, value, option_string=None):
         res = getattr(namespace, self.dest, {})
         key_str, val = value.split("=", maxsplit=1)
         keys = key_str.split("/")
-        update = {keys[-1]: val}
+        # Interpret it as a literal iff it starts like one
+        update = {keys[-1]: ast.literal_eval(val) if val[0] in "[{'" else val}
         for key in list(reversed(keys))[1:]:
             update = {key: update}
         res = overlay_dict(res, update)
@@ -145,7 +153,7 @@ def __call__(self, parser, namespace, value, option_string=None):
     "--instances",
     required=False,
     nargs="+",
-    default=DEFAULT_INSTANCES.keys(),
+    default=DEFAULT_INSTANCES,
 )
 COMMON_PARSER.add_argument(
     "--platforms",
@@ -171,6 +179,12 @@ def __call__(self, parser, namespace, value, option_string=None):
     default=None,
     type=str,
 )
+COMMON_PARSER.add_argument(
+    "--no-kani",
+    help="Don't add kani step",
+    action="store_true",
+    default=False,
+)
 
 
 def random_str(k: int):
@@ -243,7 +257,7 @@ def __init__(self, with_build_step=True, **kwargs):
         self.per_instance = overlay_dict(per_instance, args.step_param)
         self.per_arch = self.per_instance.copy()
         self.per_arch["instances"] = ["m6i.metal", "m7g.metal"]
-        self.per_arch["platforms"] = [("al2", "linux_5.10")]
+        self.per_arch["platforms"] = [("al2023", "linux_6.1")]
         self.binary_dir = args.binary_dir
         # Build sharing
         if with_build_step:
@@ -288,7 +302,7 @@ def _adapt_group(self, group):
             step["command"] = prepend + step["command"]
             if self.shared_build is not None:
                 step["depends_on"] = self.build_key(
-                    DEFAULT_INSTANCES[step["agents"]["instance"]]
+                    get_arch_for_instance(step["agents"]["instance"])
                 )
         return group
 
@@ -323,7 +337,7 @@ def build_group_per_arch(self, label, *args, **kwargs):
         if set_key:
             for step in grp["steps"]:
                 step["key"] = self.build_key(
-                    DEFAULT_INSTANCES[step["agents"]["instance"]]
+                    get_arch_for_instance(step["agents"]["instance"])
                 )
         return self.add_step(grp, depends_on_build=depends_on_build)
 

.buildkite/pipeline_cpu_template.py

@@ -6,7 +6,7 @@
 
 from enum import Enum
 
-from common import DEFAULT_INSTANCES, DEFAULT_PLATFORMS, BKPipeline, group
+from common import DEFAULT_PLATFORMS, BKPipeline, group
 
 
 class BkStep(str, Enum):
@@ -26,16 +26,19 @@ class BkStep(str, Enum):
             "tools/devtool -y test --no-build -- -m nonci -n4 --dist worksteal integration_tests/functional/test_cpu_features_x86_64.py -k 'test_cpu_rdmsr' "
         ],
         BkStep.LABEL: "📖 rdmsr",
-        "instances": ["c5n.metal", "m5n.metal", "m6a.metal", "m6i.metal"],
-        "platforms": DEFAULT_PLATFORMS,
+        "instances": [
+            "c5n.metal",
+            "m5n.metal",
+            "m6i.metal",
+            "m6a.metal",
+            "m7a.metal-48xl",
+        ],
     },
     "fingerprint": {
         BkStep.COMMAND: [
             "tools/devtool -y test --no-build -- -m no_block_pr integration_tests/functional/test_cpu_template_helper.py -k test_guest_cpu_config_change",
         ],
         BkStep.LABEL: "🖐️ fingerprint",
-        "instances": DEFAULT_INSTANCES.keys(),
-        "platforms": DEFAULT_PLATFORMS,
     },
     "cpuid_wrmsr": {
         "snapshot": {
@@ -62,7 +65,13 @@ class BkStep(str, Enum):
             "c5n.metal": ["m5n.metal", "m6i.metal"],
             "m6i.metal": ["m5n.metal", "c5n.metal"],
         },
-        "instances": ["c5n.metal", "m5n.metal", "m6i.metal", "m6a.metal"],
+        "instances": [
+            "c5n.metal",
+            "m5n.metal",
+            "m6i.metal",
+            "m6a.metal",
+            "m7a.metal-48xl",
+        ],
     },
 }
 

.buildkite/pipeline_cross.py

@@ -18,13 +18,21 @@
     per_instance = pipeline.per_instance.copy()
     per_instance.pop("instances")
     per_instance.pop("platforms")
-    instances_x86_64 = ["c5n.metal", "m5n.metal", "m6i.metal", "m6a.metal"]
+    instances_x86_64 = [
+        "c5n.metal",
+        "m5n.metal",
+        "m6i.metal",
+        "m6a.metal",
+        "m7a.metal-48xl",
+    ]
     instances_aarch64 = ["m7g.metal"]
     commands = [
-        "./tools/devtool -y sh ./tools/create_snapshot_artifact/main.py",
-        "mkdir -pv snapshots/{instance}_{kv}",
-        "sudo chown -Rc $USER: snapshot_artifacts",
-        "mv -v snapshot_artifacts/* snapshots/{instance}_{kv}",
+        "./tools/devtool -y test --no-build -- -m nonci -n4 integration_tests/functional/test_snapshot_phase1.py",
+        # punch holes in mem snapshot tiles and tar them so they are preserved in S3
+        "find test_results/test_snapshot_phase1 -type f -name mem |xargs -P4 -t -n1 fallocate -d",
+        "mv -v test_results/test_snapshot_phase1 snapshot_artifacts",
+        "mkdir -pv snapshots",
+        "tar cSvf snapshots/{instance}_{kv}.tar snapshot_artifacts",
     ]
     pipeline.build_group(
         "📸 create snapshots",
@@ -80,10 +88,10 @@
         k_val = pytest_keyword_for_instance.get(dst_instance, "")
         step = {
             "command": [
-                f"buildkite-agent artifact download snapshots/{src_instance}_{src_kv}/* .",
-                f"mv -v snapshots/{src_instance}_{src_kv} snapshot_artifacts",
+                f"buildkite-agent artifact download snapshots/{src_instance}_{src_kv}.tar .",
+                f"tar xSvf snapshots/{src_instance}_{src_kv}.tar",
                 *pipeline.devtool_test(
-                    pytest_opts=f"-m nonci {k_val} integration_tests/functional/test_snapshot_restore_cross_kernel.py",
+                    pytest_opts=f"-m nonci -n8 --dist worksteal {k_val} integration_tests/functional/test_snapshot_restore_cross_kernel.py",
                 ),
             ],
             "label": f"🎬 {src_instance} {src_kv} ➡️ {dst_instance} {dst_kv}",

.buildkite/pipeline_perf.py

@@ -121,10 +121,7 @@
 # }
 # will pin steps running on instances "m6i.metal" with kernel version tagged "linux_6.1"
 # to a new kernel version tagged "linux_6.1-pinned"
-pins = {
-    # TODO: Unpin when performance instability on m6i/5.10 has gone.
-    "linux_5.10-pinned": {"instance": "m6i.metal", "kv": "linux_5.10"},
-}
+pins = {}
 
 
 def apply_pins(steps):

.buildkite/pipeline_pr.py

@@ -48,16 +48,18 @@
         "./tools/devtool -y make_release",
     )
 
-if not changed_files or any(
-    x.suffix in [".rs", ".toml", ".lock"] for x in changed_files
+if not pipeline.args.no_kani and (
+    not changed_files
+    or any(x.suffix in [".rs", ".toml", ".lock"] for x in changed_files)
+    or any(x.parent.name == "devctr" for x in changed_files)
 ):
     kani_grp = pipeline.build_group(
         "🔍 Kani",
         "./tools/devtool -y test --no-build -- ../tests/integration_tests/test_kani.py -n auto",
         # Kani step default
         # Kani runs fastest on m6a.metal
         instances=["m6a.metal", "m7g.metal"],
-        platforms=[("al2", "linux_5.10")],
+        platforms=[("al2023", "linux_6.1")],
         timeout_in_minutes=300,
         **DEFAULTS_PERF,
         depends_on_build=False,
@@ -76,7 +78,7 @@
     pipeline.build_group(
         "⚙ Functional and security 🔒",
         pipeline.devtool_test(
-            pytest_opts="-n 8 --dist worksteal integration_tests/{{functional,security}}",
+            pytest_opts="-n 16 --dist worksteal integration_tests/{{functional,security}}",
         ),
     )
 

.cargo/audit.toml

@@ -1 +1,7 @@
 [advisories]
+# The `paste` dependency is transitively included via `gdbstub`.
+# While the crate is archived/unmaintained, the author considers it feature-complete
+# and functionally stable. gdbstub will be update once they migrate 
+# to an alternative solution.
+# See https://github.com/daniel5151/gdbstub/issues/168
+ignore = ["RUSTSEC-2024-0436"]

.github/pull_request_template.md

@@ -15,16 +15,20 @@ Certificate of Origin and signing off your commits, please check
 
 ## PR Checklist
 
+- [ ] I have read and understand [CONTRIBUTING.md][3].
+- [ ] I have run `tools/devtool checkstyle` to verify that the PR passes the
+  automated style checks.
+- [ ] I have described what is done in these changes, why they are needed, and
+  how they are solving the problem in a clear and encompassing way.
+- [ ] I have updated any relevant documentation (both in code and in the docs)
+  in the PR.
+- [ ] I have mentioned all user-facing changes in `CHANGELOG.md`.
 - [ ] If a specific issue led to this PR, this PR closes the issue.
-- [ ] The description of changes is clear and encompassing.
-- [ ] Any required documentation changes (code and docs) are included in this
-  PR.
-- [ ] API changes follow the [Runbook for Firecracker API changes][2].
-- [ ] User-facing changes are mentioned in `CHANGELOG.md`.
-- [ ] All added/changed functionality is tested.
-- [ ] New `TODO`s link to an issue.
-- [ ] Commits meet
-  [contribution quality standards](https://github.com/firecracker-microvm/firecracker/blob/main/CONTRIBUTING.md#contribution-quality-standards).
+- [ ] When making API changes, I have followed the
+  [Runbook for Firecracker API changes][2].
+- [ ] I have tested all new and changed functionalities in unit tests and/or
+  integration tests.
+- [ ] I have linked an issue to every new `TODO`.
 
 ______________________________________________________________________
 

.gitlint

@@ -9,8 +9,8 @@ line-length=72
 
 [ignore-body-lines]
 # Ignore HTTP reference links
-# Ignore lines that start with 'Co-Authored-By' or with 'Signed-off-by'
-regex=(^\[.+\]: http.+)|(^Co-Authored-By)|(^Signed-off-by)
+# Ignore lines that start with 'Co-Authored-By', with 'Signed-off-by' or with 'Fixes'
+regex=(^\[.+\]: http.+)|(^Co-Authored-By)|(^Signed-off-by)|(^Fixes:)
 
 [ignore-by-author-name]
 # Ignore certain rules for commits of which the author name matches a regex