Add option to redirect guest serial console output to file

CHANGELOG.md

@@ -47,6 +47,12 @@ and this project adheres to
   requests to `/mmds/config` to enforce MMDS to always respond plain text
   contents in the IMDS format regardless of the `Accept` header in requests.
   Users need to regenerate snapshots.
+- [#5350](https://github.com/firecracker-microvm/firecracker/pull/5350): Added a
+  `/serial` endpoint, which allows setting `serial_out_path` to the path of a
+  pre-created file into which Firecracker should redirect output from the
+  guest's serial console. Not configuring it means Firecracker will continue to
+  print serial output to stdout. Similarly to the logger, this configuration is
+  not persisted across snapshots.
 
 ### Changed
 

src/firecracker/src/api_server/parsed_request.rs

@@ -27,6 +27,7 @@ use super::request::net::{parse_patch_net, parse_put_net};
 use super::request::snapshot::{parse_patch_vm_state, parse_put_snapshot};
 use super::request::version::parse_get_version;
 use super::request::vsock::parse_put_vsock;
+use crate::api_server::request::serial::parse_put_serial;
 
 #[derive(Debug)]
 pub(crate) enum RequestAction {
@@ -90,6 +91,7 @@ impl TryFrom<&Request> for ParsedRequest {
             (Method::Put, "cpu-config", Some(body)) => parse_put_cpu_config(body),
             (Method::Put, "drives", Some(body)) => parse_put_drive(body, path_tokens.next()),
             (Method::Put, "logger", Some(body)) => parse_put_logger(body),
+            (Method::Put, "serial", Some(body)) => parse_put_serial(body),
             (Method::Put, "machine-config", Some(body)) => parse_put_machine_config(body),
             (Method::Put, "metrics", Some(body)) => parse_put_metrics(body),
             (Method::Put, "mmds", Some(body)) => parse_put_mmds(body, path_tokens.next()),

src/firecracker/src/api_server/request/mod.rs

@@ -13,6 +13,7 @@ pub mod machine_configuration;
 pub mod metrics;
 pub mod mmds;
 pub mod net;
+pub mod serial;
 pub mod snapshot;
 pub mod version;
 pub mod vsock;

src/firecracker/src/api_server/request/serial.rs

@@ -0,0 +1,39 @@
+// Copyright 2025 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+use micro_http::Body;
+use vmm::logger::{IncMetric, METRICS};
+use vmm::rpc_interface::VmmAction;
+use vmm::vmm_config::serial::SerialConfig;
+
+use crate::api_server::parsed_request::{ParsedRequest, RequestError};
+
+pub(crate) fn parse_put_serial(body: &Body) -> Result<ParsedRequest, RequestError> {
+    METRICS.put_api_requests.serial_count.inc();
+    let res = serde_json::from_slice::<SerialConfig>(body.raw());
+    let config = res.inspect_err(|_| {
+        METRICS.put_api_requests.serial_fails.inc();
+    })?;
+    Ok(ParsedRequest::new_sync(VmmAction::ConfigureSerial(config)))
+}
+
+#[cfg(test)]
+mod tests {
+    use std::path::PathBuf;
+
+    use super::*;
+    use crate::api_server::parsed_request::tests::vmm_action_from_request;
+
+    #[test]
+    fn test_parse_put_serial_request() {
+        let body = r#"{"serial_out_path": "serial"}"#;
+
+        let expected_config = SerialConfig {
+            serial_out_path: Some(PathBuf::from("serial")),
+        };
+        assert_eq!(
+            vmm_action_from_request(parse_put_serial(&Body::new(body)).unwrap()),
+            VmmAction::ConfigureSerial(expected_config)
+        );
+    }
+}

src/firecracker/swagger/firecracker.yaml

@@ -506,6 +506,27 @@ paths:
           schema:
             $ref: "#/definitions/Error"
 
+  /serial:
+    put:
+      summary: Configures the serial console
+      operationId: putSerialDevice
+      description:
+        Configure the serial console, which the guest can write its kernel logs to. Has no effect if
+        the serial console is not also enabled on the guest kernel command line
+      parameters:
+        - name: body
+          in: body
+          description: Serial console properties
+          required: true
+          schema:
+            $ref: "#/definitions/SerialDevice"
+      responses:
+        204:
+          description: Serial device configured
+        default:
+          description: Internal server error
+          schema:
+            $ref: "#/definitions/Error"
 
   /network-interfaces/{iface_id}:
     put:
@@ -1334,6 +1355,15 @@ definitions:
       rate_limiter:
         $ref: "#/definitions/RateLimiter"
 
+  SerialDevice:
+    type: object
+    description:
+      The configuration of the serial device
+    properties:
+      output_path:
+        type: string
+        description: Path to a file or named pipe on the host to which serial output should be written.
+
   FirecrackerVersion:
     type: object
     description:

src/vmm/src/arch/aarch64/fdt.rs

@@ -561,7 +561,7 @@ mod tests {
         cmdline.insert("console", "/dev/tty0").unwrap();
 
         device_manager
-            .attach_legacy_devices_aarch64(&vm, &mut event_manager, &mut cmdline)
+            .attach_legacy_devices_aarch64(&vm, &mut event_manager, &mut cmdline, None)
             .unwrap();
         let dummy = Arc::new(Mutex::new(DummyDevice::new()));
         device_manager

src/vmm/src/builder.rs

@@ -170,7 +170,12 @@ pub fn build_microvm_for_boot(
     let (mut vcpus, vcpus_exit_evt) = vm.create_vcpus(vm_resources.machine_config.vcpu_count)?;
     vm.register_memory_regions(guest_memory)?;
 
-    let mut device_manager = DeviceManager::new(event_manager, &vcpus_exit_evt, &vm)?;
+    let mut device_manager = DeviceManager::new(
+        event_manager,
+        &vcpus_exit_evt,
+        &vm,
+        vm_resources.serial_out_path.as_ref(),
+    )?;
 
     let vm = Arc::new(vm);
 
@@ -248,7 +253,12 @@ pub fn build_microvm_for_boot(
     }
 
     #[cfg(target_arch = "aarch64")]
-    device_manager.attach_legacy_devices_aarch64(&vm, event_manager, &mut boot_cmdline)?;
+    device_manager.attach_legacy_devices_aarch64(
+        &vm,
+        event_manager,
+        &mut boot_cmdline,
+        vm_resources.serial_out_path.as_ref(),
+    )?;
 
     device_manager.attach_vmgenid_device(vm.guest_memory(), &vm)?;
 
@@ -272,7 +282,6 @@ pub fn build_microvm_for_boot(
     )?;
 
     let vmm = Vmm {
-        events_observer: Some(std::io::stdin()),
         instance_info: instance_info.clone(),
         shutdown_exit_code: None,
         kvm,
@@ -473,7 +482,6 @@ pub fn build_microvm_from_snapshot(
         DeviceManager::restore(device_ctor_args, &microvm_state.device_states)?;
 
     let mut vmm = Vmm {
-        events_observer: Some(std::io::stdin()),
         instance_info: instance_info.clone(),
         shutdown_exit_code: None,
         kvm,
@@ -722,7 +730,6 @@ pub(crate) mod tests {
         let (_, vcpus_exit_evt) = vm.create_vcpus(1).unwrap();
 
         Vmm {
-            events_observer: Some(std::io::stdin()),
             instance_info: InstanceInfo::default(),
             shutdown_exit_code: None,
             kvm,

src/vmm/src/device_manager/legacy.rs

@@ -104,7 +104,7 @@ impl PortIODeviceManager {
                 SerialEventsWrapper {
                     buffer_ready_event_fd: None,
                 },
-                SerialOut::Sink(std::io::sink()),
+                SerialOut::Sink,
             ),
             input: None,
         }));
@@ -114,7 +114,7 @@ impl PortIODeviceManager {
                 SerialEventsWrapper {
                     buffer_ready_event_fd: None,
                 },
-                SerialOut::Sink(std::io::sink()),
+                SerialOut::Sink,
             ),
             input: None,
         }));
@@ -249,7 +249,7 @@ mod tests {
                     SerialEventsWrapper {
                         buffer_ready_event_fd: None,
                     },
-                    SerialOut::Sink(std::io::sink()),
+                    SerialOut::Sink,
                 ),
                 input: None,
             })),

src/vmm/src/device_manager/mod.rs

@@ -7,6 +7,8 @@
 
 use std::convert::Infallible;
 use std::fmt::Debug;
+use std::os::unix::prelude::OpenOptionsExt;
+use std::path::PathBuf;
 use std::sync::{Arc, Mutex};
 
 use acpi::ACPIDeviceManager;
@@ -125,11 +127,25 @@ impl DeviceManager {
     /// Sets up the serial device.
     fn setup_serial_device(
         event_manager: &mut EventManager,
+        output: Option<&PathBuf>,
     ) -> Result<Arc<Mutex<SerialDevice>>, std::io::Error> {
-        let serial = Arc::new(Mutex::new(SerialDevice::new(
-            Some(std::io::stdin()),
-            SerialOut::Stdout(std::io::stdout()),
-        )?));
+        let (serial_in, serial_out) = match output {
+            Some(ref path) => (
+                None,
+                std::fs::OpenOptions::new()
+                    .custom_flags(libc::O_NONBLOCK)
+                    .write(true)
+                    .open(path)
+                    .map(SerialOut::File)?,
+            ),
+            None => {
+                Self::set_stdout_nonblocking();
+
+                (Some(std::io::stdin()), SerialOut::Stdout(std::io::stdout()))
+            }
+        };
+
+        let serial = Arc::new(Mutex::new(SerialDevice::new(serial_in, serial_out)?));
         event_manager.add_subscriber(serial.clone());
         Ok(serial)
     }
@@ -139,11 +155,10 @@ impl DeviceManager {
         event_manager: &mut EventManager,
         vcpus_exit_evt: &EventFd,
         vm: &Vm,
+        serial_output: Option<&PathBuf>,
     ) -> Result<PortIODeviceManager, DeviceManagerCreateError> {
-        Self::set_stdout_nonblocking();
-
         // Create serial device
-        let serial = Self::setup_serial_device(event_manager)?;
+        let serial = Self::setup_serial_device(event_manager, serial_output)?;
         let reset_evt = vcpus_exit_evt
             .try_clone()
             .map_err(DeviceManagerCreateError::EventFd)?;
@@ -161,9 +176,11 @@ impl DeviceManager {
         event_manager: &mut EventManager,
         vcpus_exit_evt: &EventFd,
         vm: &Vm,
+        serial_output: Option<&PathBuf>,
     ) -> Result<Self, DeviceManagerCreateError> {
         #[cfg(target_arch = "x86_64")]
-        let legacy_devices = Self::create_legacy_devices(event_manager, vcpus_exit_evt, vm)?;
+        let legacy_devices =
+            Self::create_legacy_devices(event_manager, vcpus_exit_evt, vm, serial_output)?;
 
         Ok(DeviceManager {
             mmio_devices: MMIODeviceManager::new(),
@@ -243,6 +260,7 @@ impl DeviceManager {
         vm: &Vm,
         event_manager: &mut EventManager,
         cmdline: &mut Cmdline,
+        serial_out_path: Option<&PathBuf>,
     ) -> Result<(), AttachDeviceError> {
         // Serial device setup.
         let cmdline_contains_console = cmdline
@@ -253,9 +271,7 @@ impl DeviceManager {
             .contains("console=");
 
         if cmdline_contains_console {
-            // Make stdout non-blocking.
-            Self::set_stdout_nonblocking();
-            let serial = Self::setup_serial_device(event_manager)?;
+            let serial = Self::setup_serial_device(event_manager, serial_out_path)?;
             self.mmio_devices.register_mmio_serial(vm, serial, None)?;
             self.mmio_devices.add_mmio_serial_to_cmdline(cmdline)?;
         }
@@ -453,6 +469,7 @@ impl<'a> Persist<'a> for DeviceManager {
             constructor_args.event_manager,
             constructor_args.vcpus_exit_evt,
             constructor_args.vm,
+            constructor_args.vm_resources.serial_out_path.as_ref(),
         )?;
 
         // Restore MMIO devices
@@ -555,7 +572,7 @@ pub(crate) mod tests {
         #[cfg(target_arch = "x86_64")]
         let legacy_devices = PortIODeviceManager::new(
             Arc::new(Mutex::new(
-                SerialDevice::new(None, SerialOut::Sink(std::io::sink())).unwrap(),
+                SerialDevice::new(None, SerialOut::Sink).unwrap(),
             )),
             Arc::new(Mutex::new(
                 I8042Device::new(EventFd::new(libc::EFD_NONBLOCK).unwrap()).unwrap(),
@@ -582,15 +599,15 @@ pub(crate) mod tests {
         let mut cmdline = Cmdline::new(4096).unwrap();
         let mut event_manager = EventManager::new().unwrap();
         vmm.device_manager
-            .attach_legacy_devices_aarch64(&vmm.vm, &mut event_manager, &mut cmdline)
+            .attach_legacy_devices_aarch64(&vmm.vm, &mut event_manager, &mut cmdline, None)
             .unwrap();
         assert!(vmm.device_manager.mmio_devices.rtc.is_some());
         assert!(vmm.device_manager.mmio_devices.serial.is_none());
 
         let mut vmm = default_vmm();
         cmdline.insert("console", "/dev/blah").unwrap();
         vmm.device_manager
-            .attach_legacy_devices_aarch64(&vmm.vm, &mut event_manager, &mut cmdline)
+            .attach_legacy_devices_aarch64(&vmm.vm, &mut event_manager, &mut cmdline, None)
             .unwrap();
         assert!(vmm.device_manager.mmio_devices.rtc.is_some());
         assert!(vmm.device_manager.mmio_devices.serial.is_some());

src/vmm/src/device_manager/persist.rs

@@ -16,9 +16,7 @@ use super::mmio::*;
 use crate::arch::DeviceType;
 use crate::devices::acpi::vmgenid::{VMGenIDState, VMGenIdConstructorArgs, VmGenId, VmGenIdError};
 #[cfg(target_arch = "aarch64")]
-use crate::devices::legacy::serial::SerialOut;
-#[cfg(target_arch = "aarch64")]
-use crate::devices::legacy::{RTCDevice, SerialDevice};
+use crate::devices::legacy::RTCDevice;
 use crate::devices::virtio::balloon::persist::{BalloonConstructorArgs, BalloonState};
 use crate::devices::virtio::balloon::{Balloon, BalloonError};
 use crate::devices::virtio::block::BlockError;
@@ -358,13 +356,10 @@ impl<'a> Persist<'a> for MMIODeviceManager {
         {
             for state in &state.legacy_devices {
                 if state.type_ == DeviceType::Serial {
-                    let serial = Arc::new(Mutex::new(SerialDevice::new(
-                        Some(std::io::stdin()),
-                        SerialOut::Stdout(std::io::stdout()),
-                    )?));
-                    constructor_args
-                        .event_manager
-                        .add_subscriber(serial.clone());
+                    let serial = crate::DeviceManager::setup_serial_device(
+                        constructor_args.event_manager,
+                        constructor_args.vm_resources.serial_out_path.as_ref(),
+                    )?;
 
                     dev_manager.register_mmio_serial(vm, serial, Some(state.device_info))?;
                 }