test: Add tests for multi-vCPU setup

docs/cpu_templates/cpu-templates.md

@@ -180,7 +180,7 @@ curl --unix-socket /tmp/firecracker.socket -i  \
   -H 'Content-Type: application/json'      \
   -d '{
         "kvm_capabilities": ["171", "172"],
-        "vcpu_features": [{ "index": 0, "bitmap": "0b1100000" }]
+        "vcpu_features": [{ "index": 0, "bitmap": "0b11xxxxx" }]
         "reg_modifiers": [
           {
             "addr": "0x603000000013c020",

docs/cpu_templates/schema.json

@@ -26,7 +26,7 @@
                   "bitmap": {
                         "description": "Bitmap for modifying the 32 bit field in kvm_vcpu_init::features. Must be in the format `0b[01x]{1,32}`. Corresponding bits will be cleared (`0`), set (`1`) or left intact (`x`). (`_`) can be used as a separator.",
                         "type": "string",
-                        "examples": ["0b1100000"]
+                        "examples": ["0b11xxxxx"]
                   }
                 }
             }

tests/framework/microvm.py

@@ -939,6 +939,7 @@ def make_snapshot(
             snapshot_type=snapshot_type,
             meta={
                 "kernel_file": str(self.kernel_file),
+                "vcpus_count": self.vcpus_count,
             },
         )
 

tests/integration_tests/functional/test_cpu_all.py

@@ -0,0 +1,43 @@
+# Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""
+Test all vCPUs are configured correctly and work properly.
+
+This test suite aims to catch bugs of Firecracker's vCPU configuration and
+CPU templates especially under multi-vCPU setup, by checking that all vCPUs
+are operating identically, except for the expected differences.
+"""
+
+import pytest
+
+# Use the maximum number of vCPUs supported by Firecracker
+MAX_VCPUS = 32
+
+
+@pytest.mark.parametrize("vcpu_count", [MAX_VCPUS])
+def test_all_vcpus_online(uvm_any):
+    """Check all vCPUs are online inside guest"""
+    assert (
+        uvm_any.ssh.check_output("cat /sys/devices/system/cpu/online").stdout.strip()
+        == f"0-{uvm_any.vcpus_count - 1}"
+    )
+
+
+@pytest.mark.parametrize("vcpu_count", [MAX_VCPUS])
+def test_all_vcpus_have_same_features(uvm_any):
+    """
+    Check all vCPUs have the same features inside guest.
+
+    This test ensures Firecracker or CPU templates don't configure CPU features
+    differently between vCPUs.
+
+    Note that whether the shown CPU features are expected or not should be
+    tested in (arch-specific) test_cpu_features_*.py only for vCPU 0. Thus, we
+    only test the equivalence of all CPUs in the same guest.
+    """
+    # Get a feature set for each CPU and deduplicate them.
+    unique_feature_lists = uvm_any.ssh.check_output(
+        'grep -E "^(flags|Features)" /proc/cpuinfo | uniq'
+    ).stdout.splitlines()
+    assert len(unique_feature_lists) == 1