Post Migration Announce

net_util/src/lib.rs

@@ -101,7 +101,7 @@ fn create_unix_socket() -> Result<net::UdpSocket> {
     Ok(unsafe { net::UdpSocket::from_raw_fd(sock) })
 }
 
-fn vnet_hdr_len() -> usize {
+pub fn vnet_hdr_len() -> usize {
     std::mem::size_of::<virtio_net_hdr_v1>()
 }
 

virtio-devices/src/device.rs

@@ -166,6 +166,12 @@ pub trait VirtioDevice: Send {
     /// Set the access platform trait to let the device perform address
     /// translations if needed.
     fn set_access_platform(&mut self, _access_platform: Arc<dyn AccessPlatform>) {}
+
+    /// Some devices can announce their location after a live migration to
+    /// speed up normal execution.
+    fn post_migration_announcer(&mut self) -> std::option::Option<Box<dyn PostMigrationAnnouncer>> {
+        None
+    }
 }
 
 /// Trait to define address translation for devices managed by virtio-iommu
@@ -338,3 +344,11 @@ impl Pausable for VirtioCommon {
         Ok(())
     }
 }
+
+/// A PostMigrationAnnouncer is used to inform other devices about the new
+/// location of a VM after a live migration.
+pub trait PostMigrationAnnouncer: Send + Sync {
+    // Sending the announces is done on a best-effort basis, so we ignore
+    // errors.
+    fn announce_once(&self);
+}

virtio-devices/src/lib.rs

@@ -43,7 +43,7 @@ pub use self::block::{Block, BlockState};
 pub use self::console::{Console, ConsoleResizer, Endpoint};
 pub use self::device::{
     DmaRemapping, VirtioCommon, VirtioDevice, VirtioInterrupt, VirtioInterruptType,
-    VirtioSharedMemoryList,
+    VirtioSharedMemoryList, PostMigrationAnnouncer
 };
 pub use self::epoll_helper::{
     EPOLL_HELPER_EVENT_LAST, EpollHelper, EpollHelperError, EpollHelperHandler,

virtio-devices/src/net.rs

@@ -20,8 +20,9 @@ use log::{debug, error, info, trace};
 #[cfg(not(fuzzing))]
 use net_util::virtio_features_to_tap_offload;
 use net_util::{
-    CtrlQueue, MacAddr, NetCounters, NetQueuePair, OpenTapError, RxVirtio, Tap, TapError, TxVirtio,
-    VirtioNetConfig, build_net_config_space, build_net_config_space_with_mq, open_tap,
+    CtrlQueue, MAC_ADDR_LEN, MacAddr, NetCounters, NetQueuePair, OpenTapError, RxVirtio, Tap,
+    TapError, TxVirtio, VirtioNetConfig, build_net_config_space, build_net_config_space_with_mq,
+    open_tap, vnet_hdr_len,
 };
 use seccompiler::SeccompAction;
 use serde::{Deserialize, Serialize};
@@ -40,6 +41,7 @@ use super::{
     EpollHelperHandler, Error as DeviceError, RateLimiterConfig, VirtioCommon, VirtioDevice,
     VirtioDeviceType, VirtioInterruptType,
 };
+use crate::device::PostMigrationAnnouncer;
 use crate::seccomp_filters::Thread;
 use crate::thread_helper::spawn_virtio_thread;
 use crate::{GuestMemoryMmap, VirtioInterrupt};
@@ -655,6 +657,38 @@ impl Net {
     pub fn wait_for_epoll_threads(&mut self) {
         self.common.wait_for_epoll_threads();
     }
+
+    fn build_rarp_announce(&self) -> [u8; 60] {
+        const ETH_P_RARP: u16 = 0x8035; // Ethertype RARP
+        const ARP_HTYPE_ETH: u16 = 0x1; // Hardware type Ethernet
+        const ARP_PTYPE_IP: u16 = 0x0800; // Protocol type IPv4
+        const ARP_OP_REQUEST_REV: u16 = 0x0003; // RARP Request opcode
+
+        const IPV4_ADDR_LENGTH: usize = 4; // Size of an IPv4 address
+
+        let mut buf = [0u8; 60];
+
+        // Ethernet header
+        buf[0..6].copy_from_slice(&[0xff; MAC_ADDR_LEN]); // This is a broadcast
+        buf[6..12].copy_from_slice(&self.config.mac); // Src is this NIC
+        buf[12..14].copy_from_slice(&ETH_P_RARP.to_be_bytes()); // This is a RARP packet
+
+        // ARP Header
+        buf[14..16].copy_from_slice(&ARP_HTYPE_ETH.to_be_bytes());
+        buf[16..18].copy_from_slice(&ARP_PTYPE_IP.to_be_bytes());
+        buf[18] = MAC_ADDR_LEN as u8; // Hardware address length (ethernet)
+        buf[19] = IPV4_ADDR_LENGTH as u8; // Protocol address length (IPv4)
+        // This is a "fake RARP" packet, we don't want to perform a real RARP lookup.
+        // Thus the content of the next fields is largely irrelevant. Setting sha = tha
+        // is fine according to RFC 903.
+        buf[20..22].copy_from_slice(&ARP_OP_REQUEST_REV.to_be_bytes());
+        buf[22..28].copy_from_slice(&self.config.mac); // Source hardware address
+        buf[28..32].copy_from_slice(&[0x00; IPV4_ADDR_LENGTH]); // Source protocol address
+        buf[32..38].copy_from_slice(&self.config.mac); // Target hardware address
+        buf[38..42].copy_from_slice(&[0x00; IPV4_ADDR_LENGTH]); // Target protocol address
+
+        buf
+    }
 }
 
 impl Drop for Net {
@@ -870,6 +904,13 @@ impl VirtioDevice for Net {
     fn set_access_platform(&mut self, access_platform: Arc<dyn AccessPlatform>) {
         self.common.set_access_platform(access_platform);
     }
+
+    fn post_migration_announcer(&mut self) -> std::option::Option<Box<dyn PostMigrationAnnouncer>> {
+        Some(Box::new(TapRarpAnnouncer::new(
+            self.build_rarp_announce(),
+            self.taps.clone(),
+        )))
+    }
 }
 
 impl Pausable for Net {
@@ -898,3 +939,32 @@ impl Snapshottable for Net {
 }
 impl Transportable for Net {}
 impl Migratable for Net {}
+
+pub struct TapRarpAnnouncer {
+    announce: [u8; 60],
+    taps: Vec<Tap>,
+}
+
+impl TapRarpAnnouncer {
+    pub fn new(announce: [u8; 60], taps: Vec<Tap>) -> Self {
+        Self { announce, taps }
+    }
+}
+
+impl PostMigrationAnnouncer for TapRarpAnnouncer {
+    fn announce_once(&self) {
+        // We have to add a virtio-net header to the announce.
+        let mut buf = vec![0u8; vnet_hdr_len() + self.announce.len()];
+        buf[vnet_hdr_len()..].copy_from_slice(&self.announce);
+
+        for tap in &self.taps {
+            let _ = unsafe {
+                libc::write(
+                    tap.as_raw_fd(),
+                    buf.as_ptr() as *const libc::c_void,
+                    buf.len(),
+                )
+            };
+        }
+    }
+}

vmm/src/device_manager.rs

@@ -16,10 +16,11 @@ use std::num::Wrapping;
 use std::os::unix::fs::OpenOptionsExt;
 use std::os::unix::io::{AsRawFd, FromRawFd};
 use std::path::{Path, PathBuf};
-use std::result;
 use std::sync::{Arc, Mutex};
+use std::time::Duration;
 #[cfg(not(target_arch = "riscv64"))]
 use std::time::Instant;
+use std::{result, thread};
 
 use acpi_tables::sdt::GenericAddress;
 use acpi_tables::{Aml, aml};
@@ -90,8 +91,8 @@ use vfio_ioctls::{VfioContainer, VfioDevice, VfioDeviceFd};
 use virtio_devices::transport::{VirtioPciDevice, VirtioPciDeviceActivator, VirtioTransport};
 use virtio_devices::vhost_user::VhostUserConfig;
 use virtio_devices::{
-    AccessPlatformMapping, ActivateError, Block, Endpoint, IommuMapping, VdpaDmaMapping,
-    VirtioMemMappingSource,
+    AccessPlatformMapping, ActivateError, Block, Endpoint, IommuMapping, PostMigrationAnnouncer,
+    VdpaDmaMapping, VirtioMemMappingSource,
 };
 use vm_allocator::{AddressAllocator, SystemAllocator};
 use vm_device::dma_mapping::ExternalDmaMapping;
@@ -5063,6 +5064,47 @@ impl DeviceManager {
             self.vfio_container = None;
         }
     }
+
+    // Calls the PostMigrationAnnouncers of each device that has one.
+    pub fn post_migration_announce(&self) {
+        let announcers: Vec<Box<dyn PostMigrationAnnouncer>> = self
+            .virtio_devices
+            .iter()
+            .map(|dev| dev.virtio_device.lock().unwrap().post_migration_announcer())
+            .flatten()
+            .collect();
+
+        announcers.iter().for_each(|a| a.announce_once());
+        schedule_post_migration_announces(announcers, 4, 50, 100, 450);
+    }
+}
+
+// We could make this announcer configurable.
+fn schedule_post_migration_announces(
+    announcers: Vec<Box<dyn PostMigrationAnnouncer>>,
+    rounds: u32,
+    initial_ms: u64,
+    step_ms: u64,
+    max_ms: u64,
+) {
+    if announcers.is_empty() || rounds == 0 {
+        return;
+    }
+
+    let _ = thread::Builder::new()
+        .name("post-migration-announcers".to_string())
+        .spawn(move || {
+            for round in 0..rounds {
+                // The first announce is done synchronous, thus we sleep at the
+                // start of the loop.
+
+                let delay = (initial_ms + (round as u64) * step_ms).min(max_ms);
+                let delay = Duration::from_millis(delay);
+                thread::sleep(delay);
+
+                announcers.iter().for_each(|a| a.announce_once());
+            }
+        });
 }
 
 #[cfg(feature = "ivshmem")]

vmm/src/lib.rs

@@ -1847,6 +1847,7 @@ impl Vmm {
                     // The unwrap is safe, because the state machine makes sure we called
                     // vm_receive_state before, which creates the VM.
                     let vm = self.vm.vm_mut().unwrap();
+                    vm.announce();
                     vm.resume()?;
                     Ok(Completed)
                 }

vmm/src/vm.rs

@@ -2757,6 +2757,18 @@ impl Vm {
             .nmi()
             .map_err(|_| Error::ErrorNmi);
     }
+
+    /// Announces this VMs location to the network. This is done by sending
+    /// reverse ARP packets for every NIC.
+    ///
+    /// Should be done during a live migration just before the VM resumes its
+    /// execution. That way the network learns the new location of the VM.
+    pub fn announce(&self) {
+        self.device_manager
+            .lock()
+            .unwrap()
+            .post_migration_announce()
+    }
 }
 
 impl Pausable for Vm {