test: enable serial console in integration tests

If SSH to guest is failing, we don't really get much useful in terms of
logs in our ci artifacts. Enable serial console so that we get guest
dmesg always, at least if we have the API server around (which is the
case for the vast majority of tests).

Since serial console is enabled by default now, this means we can stop
explicitly passing modified bootargs in a bunch of tests now.

Disable it for the tests that interact with the serial console via
stdin/stdout (here we are not daemonizing, so we are capturing serial
output anyway).

Disable pylint's "too-many-statements" lint, because it started firing
in microvm.spawn()

Signed-off-by: Patrick Roy <[email protected]>

Author: roypat

tests/framework/http_api.py

@@ -132,3 +132,4 @@ def __init__(self, api_usocket_full_name, *, on_error=None):
         self.snapshot_load = Resource(self, "/snapshot/load")
         self.cpu_config = Resource(self, "/cpu-config")
         self.entropy = Resource(self, "/entropy")
+        self.serial = Resource(self, "/serial")

tests/framework/microvm.py

@@ -259,6 +259,7 @@ def __init__(
 
         self.api = None
         self.log_file = None
+        self.serial_out_path = None
         self.metrics_file = None
         self._spawned = False
         self._killed = False
@@ -624,6 +625,7 @@ def add_pre_cmd(self, pre_cmd):
     def spawn(
         self,
         log_file="fc.log",
+        serial_out_path="serial.log",
         log_level="Debug",
         log_show_level=False,
         log_show_origin=False,
@@ -654,6 +656,11 @@ def spawn(
             if log_show_origin:
                 self.jailer.extra_args["show-log-origin"] = None
 
+        if serial_out_path is not None:
+            self.serial_out_path = Path(self.path) / serial_out_path
+            self.serial_out_path.touch()
+            self.create_jailed_resource(self.serial_out_path)
+
         if metrics_path is not None:
             self.metrics_file = Path(self.path) / metrics_path
             self.metrics_file.touch()
@@ -718,13 +725,18 @@ def spawn(
         # Firecracker process itself at least came up by checking
         # for the startup log message. Otherwise, you're on your own kid.
         if "config-file" in self.jailer.extra_args and self.iface:
+            assert not serial_out_path
             self.wait_for_ssh_up()
         elif "no-api" not in self.jailer.extra_args:
             if self.log_file and log_level in ("Trace", "Debug", "Info"):
                 self.check_log_message("API server started.")
             else:
                 self._wait_for_api_socket()
+
+            if serial_out_path is not None:
+                self.api.serial.put(serial_out_path=serial_out_path)
         elif self.log_file and log_level in ("Trace", "Debug", "Info"):
+            assert not serial_out_path
             self.check_log_message("Running Firecracker")
 
     @retry(wait=wait_fixed(0.2), stop=stop_after_attempt(5), reraise=True)
@@ -795,12 +807,9 @@ def basic_config(
         The function checks the response status code and asserts that
         the response is within the interval [200, 300).
 
-        If boot_args is None, the default boot_args in Firecracker is
-            reboot=k panic=1 nomodule 8250.nr_uarts=0 i8042.noaux i8042.nomux
-            i8042.nopnp i8042.dumbkbd swiotlb=noforce
-
-        if PCI is disabled, Firecracker also passes to the guest pci=off
-
+        If boot_args is None, the default boot_args used in tests is
+            reboot=k panic=1 nomodule swiotlb=noforce console=ttyS0 [pci=off]
+        which differs from Firecracker's default only in the enabling of the serial console.
         Reference: file:../../src/vmm/src/vmm_config/boot_source.rs::DEFAULT_KERNEL_CMDLINE
         """
         self.api.machine_config.put(
@@ -824,6 +833,10 @@ def basic_config(
 
         if boot_args is not None:
             self.boot_args = boot_args
+        else:
+            self.boot_args = "reboot=k panic=1 nomodule swiotlb=noforce console=ttyS0"
+            if not self.pci_enabled:
+                self.boot_args += " pci=off"
         boot_source_args = {
             "kernel_image_path": self.create_jailed_resource(self.kernel_file),
             "boot_args": self.boot_args,
@@ -1313,9 +1326,9 @@ def open(self):
             time.sleep(0.2)
             attempt += 1
 
-        screen_log_fd = os.open(self._vm.screen_log, os.O_RDONLY)
+        serial_log_fd = os.open(self._vm.screen_log, os.O_RDONLY)
         self._poller = select.poll()
-        self._poller.register(screen_log_fd, select.POLLIN | select.POLLHUP)
+        self._poller.register(serial_log_fd, select.POLLIN | select.POLLHUP)
 
     def tx(self, input_string, end="\n"):
         # pylint: disable=invalid-name

tests/integration_tests/functional/test_api.py

@@ -1136,8 +1136,10 @@ def test_get_full_config_after_restoring_snapshot(microvm_factory, uvm_nano):
     expected_cfg["boot-source"] = {
         "kernel_image_path": uvm_nano.get_jailed_resource(uvm_nano.kernel_file),
         "initrd_path": None,
-        "boot_args": None,
+        "boot_args": "reboot=k panic=1 nomodule swiotlb=noforce console=ttyS0",
     }
+    if not uvm_nano.pci_enabled:
+        expected_cfg["boot-source"]["boot_args"] += " pci=off"
 
     # no ipv4_address or imds_compat specified during PUT /mmds/config so we expect the default
     expected_cfg["mmds-config"] = {

tests/integration_tests/functional/test_api_server.py

@@ -23,7 +23,7 @@ def test_api_socket_in_use(uvm_plain):
 
     sock = socket.socket(socket.AF_UNIX)
     sock.bind(microvm.jailer.api_socket_path())
-    microvm.spawn(log_level="warn")
+    microvm.spawn(log_level="warn", serial_out_path=None)
     msg = "Failed to open the API socket at: /run/firecracker.socket. Check that it is not already used."
     microvm.check_log_message(msg)
 

tests/integration_tests/functional/test_cmd_line_parameters.py

@@ -29,7 +29,7 @@ def test_describe_snapshot_all_versions(
         jailer_binary_path=firecracker_release.jailer,
     )
     # FIXME: Once only FC versions >= 1.12 are supported, drop log_level="warn"
-    vm.spawn(log_level="warn")
+    vm.spawn(log_level="warn", serial_out_path=None)
     vm.basic_config(track_dirty_pages=True)
     vm.start()
     snapshot = vm.snapshot_diff()

tests/integration_tests/functional/test_cmd_line_start.py

@@ -117,7 +117,7 @@ def test_config_start_with_api(uvm_plain, vm_config_file):
     """
     test_microvm = uvm_plain
     vm_config = _configure_vm_from_json(test_microvm, vm_config_file)
-    test_microvm.spawn()
+    test_microvm.spawn(serial_out_path=None)
 
     assert test_microvm.state == "Running"
 
@@ -134,7 +134,7 @@ def test_config_start_no_api(uvm_plain, vm_config_file):
     test_microvm = uvm_plain
     _configure_vm_from_json(test_microvm, vm_config_file)
     test_microvm.jailer.extra_args.update({"no-api": None})
-    test_microvm.spawn()
+    test_microvm.spawn(serial_out_path=None)
 
     # Get names of threads in Firecracker.
     cmd = f"ps -T --no-headers -p {test_microvm.firecracker_pid} | awk '{{print $5}}'"
@@ -165,7 +165,7 @@ def test_config_start_no_api_exit(uvm_plain, vm_config_file):
     _configure_network_interface(test_microvm)
     test_microvm.jailer.extra_args.update({"no-api": None})
 
-    test_microvm.spawn()  # Start Firecracker and MicroVM
+    test_microvm.spawn(serial_out_path=None)  # Start Firecracker and MicroVM
     test_microvm.ssh.run("reboot")  # Exit
 
     test_microvm.mark_killed()  # waits for process to terminate
@@ -189,7 +189,7 @@ def test_config_bad_machine_config(uvm_plain, vm_config_file):
     test_microvm = uvm_plain
     _configure_vm_from_json(test_microvm, vm_config_file)
     test_microvm.jailer.extra_args.update({"no-api": None})
-    test_microvm.spawn()
+    test_microvm.spawn(serial_out_path=None)
     test_microvm.check_log_message("Configuration for VMM from one single json failed")
 
     test_microvm.mark_killed()
@@ -215,7 +215,7 @@ def test_config_machine_config_params(uvm_plain, test_config):
     _configure_vm_from_json(test_microvm, vm_config_file)
     test_microvm.jailer.extra_args.update({"no-api": None})
 
-    test_microvm.spawn()
+    test_microvm.spawn(serial_out_path=None)
 
     should_fail = False
     if cpu_template_used and "C3" not in SUPPORTED_CPU_TEMPLATES:
@@ -247,7 +247,7 @@ def test_config_start_with_limit(uvm_plain, vm_config_file):
 
     _configure_vm_from_json(test_microvm, vm_config_file)
     test_microvm.jailer.extra_args.update({"http-api-max-payload-size": "250"})
-    test_microvm.spawn()
+    test_microvm.spawn(serial_out_path=None)
 
     assert test_microvm.state == "Running"
 
@@ -277,7 +277,7 @@ def test_config_with_default_limit(uvm_plain, vm_config_file):
     test_microvm = uvm_plain
 
     _configure_vm_from_json(test_microvm, vm_config_file)
-    test_microvm.spawn()
+    test_microvm.spawn(serial_out_path=None)
 
     assert test_microvm.state == "Running"
 
@@ -311,7 +311,7 @@ def test_start_with_metadata(uvm_plain):
     metadata_file = DIR / "metadata.json"
     _add_metadata_file(test_microvm, metadata_file)
 
-    test_microvm.spawn()
+    test_microvm.spawn(serial_out_path=None)
 
     test_microvm.check_log_message("Successfully added metadata to mmds from file")
 
@@ -332,7 +332,7 @@ def test_start_with_metadata_limit(uvm_plain):
     metadata_file = DIR / "metadata.json"
     _add_metadata_file(test_microvm, metadata_file)
 
-    test_microvm.spawn()
+    test_microvm.spawn(serial_out_path=None)
 
     test_microvm.check_log_message(
         "Populating MMDS from file failed: The MMDS patch request doesn't fit."
@@ -352,7 +352,7 @@ def test_start_with_metadata_default_limit(uvm_plain):
 
     _add_metadata_file(test_microvm, metadata_file)
 
-    test_microvm.spawn()
+    test_microvm.spawn(serial_out_path=None)
 
     test_microvm.check_log_message(
         "Populating MMDS from file failed: The MMDS patch request doesn't fit."
@@ -372,7 +372,7 @@ def test_start_with_missing_metadata(uvm_plain):
     test_microvm.metadata_file = vm_metadata_path
 
     try:
-        test_microvm.spawn()
+        test_microvm.spawn(serial_out_path=None)
     except:  # pylint: disable=bare-except
         pass
     finally:
@@ -395,7 +395,7 @@ def test_start_with_invalid_metadata(uvm_plain):
     test_microvm.metadata_file = vm_metadata_path
 
     try:
-        test_microvm.spawn()
+        test_microvm.spawn(serial_out_path=None)
     except:  # pylint: disable=bare-except
         pass
     finally:
@@ -418,7 +418,7 @@ def test_config_start_and_mmds_with_api(uvm_plain, vm_config_file):
     _configure_network_interface(test_microvm)
 
     # Network namespace has already been created.
-    test_microvm.spawn()
+    test_microvm.spawn(serial_out_path=None)
 
     data_store = {
         "latest": {
@@ -480,7 +480,7 @@ def test_with_config_and_metadata_no_api(uvm_plain, vm_config_file, metadata_fil
     _add_metadata_file(test_microvm, metadata_file)
     _configure_network_interface(test_microvm)
     test_microvm.jailer.extra_args.update({"no-api": None})
-    test_microvm.spawn()
+    test_microvm.spawn(serial_out_path=None)
 
     # Get MMDS version and IPv4 address configured from the file.
     version, ipv4_address, imds_compat = _get_optional_fields_from_file(vm_config_file)

tests/integration_tests/functional/test_kernel_cmdline.py

@@ -14,7 +14,7 @@ def test_init_params(uvm_plain):
     """
     vm = uvm_plain
     vm.help.enable_console()
-    vm.spawn()
+    vm.spawn(serial_out_path=None)
     vm.memory_monitor = None
 
     # We will override the init with /bin/cat so that we try to read the

tests/integration_tests/functional/test_max_devices.py

@@ -18,9 +18,9 @@ def max_devices(uvm):
     match platform.machine():
         case "aarch64":
             # On aarch64, IRQs are available from 32 to 127. We always use one IRQ each for
-            # the VMGenID and RTC devices, so the maximum number of devices supported
-            # at the same time is 94.
-            return 94
+            # the VMGenID, RTC and serial devices, so the maximum number of devices supported
+            # at the same time is 93.
+            return 93
         case "x86_64":
             # IRQs are available from 5 to 23. We always use one IRQ for VMGenID device, so
             # the maximum number of devices supported at the same time is 18.

tests/integration_tests/functional/test_serial_io.py

@@ -51,11 +51,10 @@ def test_serial_after_snapshot(uvm_plain, microvm_factory):
     """
     microvm = uvm_plain
     microvm.help.enable_console()
-    microvm.spawn()
+    microvm.spawn(serial_out_path=None)
     microvm.basic_config(
         vcpu_count=2,
         mem_size_mib=256,
-        boot_args="console=ttyS0 reboot=k panic=1 swiotlb=noforce",
     )
     serial = Serial(microvm)
     serial.open()
@@ -72,7 +71,7 @@ def test_serial_after_snapshot(uvm_plain, microvm_factory):
     # Load microVM clone from snapshot.
     vm = microvm_factory.build()
     vm.help.enable_console()
-    vm.spawn()
+    vm.spawn(serial_out_path=None)
     vm.restore_from_snapshot(snapshot, resume=True)
     serial = Serial(vm)
     serial.open()
@@ -92,16 +91,14 @@ def test_serial_console_login(uvm_plain_any):
     """
     microvm = uvm_plain_any
     microvm.help.enable_console()
-    microvm.spawn()
+    microvm.spawn(serial_out_path=None)
 
     # We don't need to monitor the memory for this test because we are
     # just rebooting and the process dies before pmap gets the RSS.
     microvm.memory_monitor = None
 
     # Set up the microVM with 1 vCPU and a serial console.
-    microvm.basic_config(
-        vcpu_count=1, boot_args="console=ttyS0 reboot=k panic=1 swiotlb=noforce"
-    )
+    microvm.basic_config(vcpu_count=1)
 
     microvm.start()
 
@@ -146,7 +143,6 @@ def test_serial_dos(uvm_plain_any):
     # Set up the microVM with 1 vCPU and a serial console.
     microvm.basic_config(
         vcpu_count=1,
-        boot_args="console=ttyS0 reboot=k panic=1 swiotlb=noforce",
     )
     microvm.add_net_iface()
     microvm.start()
@@ -174,13 +170,12 @@ def test_serial_block(uvm_plain_any):
     """
     test_microvm = uvm_plain_any
     test_microvm.help.enable_console()
-    test_microvm.spawn()
+    test_microvm.spawn(serial_out_path=None)
     # Set up the microVM with 1 vCPU so we make sure the vCPU thread
     # responsible for the SSH connection will also run the serial.
     test_microvm.basic_config(
         vcpu_count=1,
         mem_size_mib=512,
-        boot_args="console=ttyS0 reboot=k panic=1 swiotlb=noforce",
     )
     test_microvm.add_net_iface()
     test_microvm.start()

tests/integration_tests/performance/test_initrd.py

@@ -29,7 +29,7 @@ def test_microvm_initrd_with_serial(uvm_with_initrd, huge_pages):
     """
     vm = uvm_with_initrd
     vm.help.enable_console()
-    vm.spawn()
+    vm.spawn(serial_out_path=None)
     vm.memory_monitor = None
 
     vm.basic_config(