Merge remote-tracking branch 'upstream/main'

Author: longxiangqiao

.buildkite/common.py

@@ -13,21 +13,26 @@
 import subprocess
 from pathlib import Path
 
-DEFAULT_INSTANCES = {
-    "c5n.metal": "x86_64",  # Intel Skylake
-    "m5n.metal": "x86_64",  # Intel Cascade Lake
-    "m6i.metal": "x86_64",  # Intel Icelake
-    "m6a.metal": "x86_64",  # AMD Milan
-    "m6g.metal": "aarch64",  # Graviton2
-    "m7g.metal": "aarch64",  # Graviton3
-}
+DEFAULT_INSTANCES = [
+    "c5n.metal",  # Intel Skylake
+    "m5n.metal",  # Intel Cascade Lake
+    "m6i.metal",  # Intel Icelake
+    "m6a.metal",  # AMD Milan
+    "m6g.metal",  # Graviton2
+    "m7g.metal",  # Graviton3
+]
 
 DEFAULT_PLATFORMS = [
     ("al2", "linux_5.10"),
     ("al2023", "linux_6.1"),
 ]
 
 
+def get_arch_for_instance(instance):
+    """Return instance architecture"""
+    return "x86_64" if instance[2] != "g" else "aarch64"
+
+
 def overlay_dict(base: dict, update: dict):
     """Overlay a dict over a base one"""
     base = base.copy()
@@ -145,7 +150,7 @@ def __call__(self, parser, namespace, value, option_string=None):
     "--instances",
     required=False,
     nargs="+",
-    default=DEFAULT_INSTANCES.keys(),
+    default=DEFAULT_INSTANCES,
 )
 COMMON_PARSER.add_argument(
     "--platforms",
@@ -288,7 +293,7 @@ def _adapt_group(self, group):
             step["command"] = prepend + step["command"]
             if self.shared_build is not None:
                 step["depends_on"] = self.build_key(
-                    DEFAULT_INSTANCES[step["agents"]["instance"]]
+                    get_arch_for_instance(step["agents"]["instance"])
                 )
         return group
 
@@ -323,7 +328,7 @@ def build_group_per_arch(self, label, *args, **kwargs):
         if set_key:
             for step in grp["steps"]:
                 step["key"] = self.build_key(
-                    DEFAULT_INSTANCES[step["agents"]["instance"]]
+                    get_arch_for_instance(step["agents"]["instance"])
                 )
         return self.add_step(grp, depends_on_build=depends_on_build)
 

.buildkite/pipeline_cpu_template.py

@@ -23,7 +23,7 @@ class BkStep(str, Enum):
 cpu_template_test = {
     "rdmsr": {
         BkStep.COMMAND: [
-            "tools/devtool -y test --no-build -- -m nonci -n4 --dist worksteal integration_tests/functional/test_cpu_features.py -k 'test_cpu_rdmsr' "
+            "tools/devtool -y test --no-build -- -m nonci -n4 --dist worksteal integration_tests/functional/test_cpu_features_x86_64.py -k 'test_cpu_rdmsr' "
         ],
         BkStep.LABEL: "📖 rdmsr",
         "instances": ["c5n.metal", "m5n.metal", "m6a.metal", "m6i.metal"],
@@ -34,13 +34,13 @@ class BkStep(str, Enum):
             "tools/devtool -y test --no-build -- -m no_block_pr integration_tests/functional/test_cpu_template_helper.py -k test_guest_cpu_config_change",
         ],
         BkStep.LABEL: "🖐️ fingerprint",
-        "instances": DEFAULT_INSTANCES.keys(),
+        "instances": DEFAULT_INSTANCES,
         "platforms": DEFAULT_PLATFORMS,
     },
     "cpuid_wrmsr": {
         "snapshot": {
             BkStep.COMMAND: [
-                "tools/devtool -y test --no-build -- -m nonci -n4 --dist worksteal integration_tests/functional/test_cpu_features.py -k 'test_cpu_wrmsr_snapshot or test_cpu_cpuid_snapshot'",
+                "tools/devtool -y test --no-build -- -m nonci -n4 --dist worksteal integration_tests/functional/test_cpu_features_x86_64.py -k 'test_cpu_wrmsr_snapshot or test_cpu_cpuid_snapshot'",
                 "mkdir -pv tests/snapshot_artifacts_upload/{instance}_{os}_{kv}",
                 "sudo mv tests/snapshot_artifacts/* tests/snapshot_artifacts_upload/{instance}_{os}_{kv}",
             ],
@@ -52,7 +52,7 @@ class BkStep(str, Enum):
             BkStep.COMMAND: [
                 "buildkite-agent artifact download tests/snapshot_artifacts_upload/{instance}_{os}_{kv}/**/* .",
                 "mv tests/snapshot_artifacts_upload/{instance}_{os}_{kv} tests/snapshot_artifacts",
-                "tools/devtool -y test --no-build -- -m nonci -n4 --dist worksteal integration_tests/functional/test_cpu_features.py -k 'test_cpu_wrmsr_restore or test_cpu_cpuid_restore'",
+                "tools/devtool -y test --no-build -- -m nonci -n4 --dist worksteal integration_tests/functional/test_cpu_features_x86_64.py -k 'test_cpu_wrmsr_restore or test_cpu_cpuid_restore'",
             ],
             BkStep.LABEL: "📸 load snapshot artifacts created on {instance} {snapshot_os} {snapshot_kv} to {restore_instance} {restore_os} {restore_kv}",
             BkStep.TIMEOUT: 30,

.buildkite/pipeline_cross.py

@@ -21,10 +21,12 @@
     instances_x86_64 = ["c5n.metal", "m5n.metal", "m6i.metal", "m6a.metal"]
     instances_aarch64 = ["m7g.metal"]
     commands = [
-        "./tools/devtool -y sh ./tools/create_snapshot_artifact/main.py",
-        "mkdir -pv snapshots/{instance}_{kv}",
-        "sudo chown -Rc $USER: snapshot_artifacts",
-        "mv -v snapshot_artifacts/* snapshots/{instance}_{kv}",
+        "./tools/devtool -y test --no-build -- -m nonci -n4 integration_tests/functional/test_snapshot_phase1.py",
+        # punch holes in mem snapshot tiles and tar them so they are preserved in S3
+        "find test_results/test_snapshot_phase1 -type f -name mem |xargs -P4 -t -n1 fallocate -d",
+        "mv -v test_results/test_snapshot_phase1 snapshot_artifacts",
+        "mkdir -pv snapshots",
+        "tar cSvf snapshots/{instance}_{kv}.tar snapshot_artifacts",
     ]
     pipeline.build_group(
         "📸 create snapshots",
@@ -80,10 +82,10 @@
         k_val = pytest_keyword_for_instance.get(dst_instance, "")
         step = {
             "command": [
-                f"buildkite-agent artifact download snapshots/{src_instance}_{src_kv}/* .",
-                f"mv -v snapshots/{src_instance}_{src_kv} snapshot_artifacts",
+                f"buildkite-agent artifact download snapshots/{src_instance}_{src_kv}.tar .",
+                f"tar xSvf snapshots/{src_instance}_{src_kv}.tar",
                 *pipeline.devtool_test(
-                    pytest_opts=f"-m nonci {k_val} integration_tests/functional/test_snapshot_restore_cross_kernel.py",
+                    pytest_opts=f"-m nonci -n4 {k_val} integration_tests/functional/test_snapshot_restore_cross_kernel.py",
                 ),
             ],
             "label": f"🎬 {src_instance} {src_kv} ➡️ {dst_instance} {dst_kv}",

.github/pull_request_template.md

@@ -15,16 +15,20 @@ Certificate of Origin and signing off your commits, please check
 
 ## PR Checklist
 
+- [ ] I have read and understand [CONTRIBUTING.md][3].
+- [ ] I have run `tools/devtool checkstyle` to verify that the PR passes the
+  automated style checks.
+- [ ] I have described what is done in these changes, why they are needed, and
+  how they are solving the problem in a clear and encompassing way.
+- [ ] I have updated any relevant documentation (both in code and in the docs)
+  in the PR.
+- [ ] I have mentioned all user-facing changes in `CHANGELOG.md`.
 - [ ] If a specific issue led to this PR, this PR closes the issue.
-- [ ] The description of changes is clear and encompassing.
-- [ ] Any required documentation changes (code and docs) are included in this
-  PR.
-- [ ] API changes follow the [Runbook for Firecracker API changes][2].
-- [ ] User-facing changes are mentioned in `CHANGELOG.md`.
-- [ ] All added/changed functionality is tested.
-- [ ] New `TODO`s link to an issue.
-- [ ] Commits meet
-  [contribution quality standards](https://github.com/firecracker-microvm/firecracker/blob/main/CONTRIBUTING.md#contribution-quality-standards).
+- [ ] When making API changes, I have followed the
+  [Runbook for Firecracker API changes][2].
+- [ ] I have tested all new and changed functionalities in unit tests and/or
+  integration tests.
+- [ ] I have linked an issue to every new `TODO`.
 
 ______________________________________________________________________
 

CHANGELOG.md

@@ -10,6 +10,33 @@ and this project adheres to
 
 ### Added
 
+### Changed
+
+- [#4913](https://github.com/firecracker-microvm/firecracker/pull/4913): Removed
+  unnecessary fields (`max_connections` and `max_pending_resets`) from the
+  snapshot format, bumping the snapshot version to 5.0.0. Users need to
+  regenerate snapshots.
+
+### Deprecated
+
+### Removed
+
+### Fixed
+
+- [#4921](https://github.com/firecracker-microvm/firecracker/pull/4921): Fixed
+  swagger `CpuConfig` definition to include missing aarch64-specific fields.
+
+## \[1.10.1\]
+
+### Changed
+
+- [#4907](https://github.com/firecracker-microvm/firecracker/pull/4907): Bumped
+  the snapshot version to 4.0.0, so users need to regenerate snapshots.
+
+## \[1.10.0\]
+
+### Added
+
 - [#4834](https://github.com/firecracker-microvm/firecracker/pull/4834): Add
   `VIRTIO_NET_F_RX_MRGBUF` support to the `virtio-net` device. When this feature
   is negotiated, guest `virtio-net` driver can perform more efficient memory
@@ -33,10 +60,6 @@ and this project adheres to
 
 ### Changed
 
-- [#4875](https://github.com/firecracker-microvm/firecracker/pull/4875):
-  Increase default queue size for the `virtio-net` device from 256 to 512. This
-  decreases wait time between guest and vmm threads for network packets
-  processing and allows for more throughput.
 - [#4844](https://github.com/firecracker-microvm/firecracker/pull/4844): Upgrade
   `virtio-net` device to use `readv` syscall to avoid unnecessary memory copies
   on RX path, increasing the RX performance.