Introduce hugepage support

Init hugepages on the host and enhance vm-service api to support
enabling hugepage on the VM as well.

Signed-off-by: Guvenc Gulce <[email protected]>

Author: guvenc

cli/src/vm_commands.rs

@@ -51,6 +51,9 @@ pub enum VmCommand {
             help = "PCI device BDF to passthrough for networking (e.g., 0000:03:00.0)"
         )]
         pci_device: Vec<String>,
+
+        #[arg(long)]
+        hugepages: bool,
     },
     Start {
         #[arg(required = true)]
@@ -115,7 +118,19 @@ pub async fn handle_vm_command(args: VmArgs) -> Result<()> {
             memory,
             vm_id,
             pci_device,
-        } => create_vm(&mut client, image_ref, vcpus, memory, vm_id, pci_device).await?,
+            hugepages,
+        } => {
+            create_vm(
+                &mut client,
+                image_ref,
+                vcpus,
+                memory,
+                vm_id,
+                pci_device,
+                hugepages,
+            )
+            .await?
+        }
         VmCommand::Start { vm_id } => start_vm(&mut client, vm_id).await?,
         VmCommand::Info { vm_id } => get_vm_info(&mut client, vm_id).await?,
         VmCommand::List => list_vms(&mut client).await?,
@@ -142,6 +157,7 @@ async fn create_vm(
     memory: u64,
     vm_id: Option<String>,
     pci_devices: Vec<String>,
+    hugepages: bool,
 ) -> Result<()> {
     println!("Requesting VM creation with image: {image_ref}...");
 
@@ -162,7 +178,10 @@ async fn create_vm(
                 boot_vcpus: vcpus,
                 max_vcpus: vcpus,
             }),
-            memory: Some(MemoryConfig { size_mib: memory }),
+            memory: Some(MemoryConfig {
+                size_mib: memory,
+                hugepages,
+            }),
             image_ref,
             net: net_configs,
             ..Default::default()

feos/services/vm-service/src/dispatcher.rs

@@ -193,8 +193,7 @@ impl VmServiceDispatcher {
                     Ok(true) => {
                         if let Err(e) = self.status_channel_tx.send(event_to_forward) {
                             debug!(
-                                "VM_DISPATCHER: Failed to forward successful VM status event for {}: {e}",
-                                vm_id
+                                "VM_DISPATCHER: Failed to forward successful VM status event for {vm_id}: {e}"
                             );
                         }
                     }
@@ -211,10 +210,7 @@ impl VmServiceDispatcher {
                 }
             }
             Err(e) => {
-                error!(
-                    "DB_UPDATE: Failed to decode VmStateChangedEvent for VM {}: {e}",
-                    vm_id
-                );
+                error!("DB_UPDATE: Failed to decode VmStateChangedEvent for VM {vm_id}: {e}");
             }
         }
     }

feos/services/vm-service/src/vmm/ch_adapter.rs

@@ -120,6 +120,7 @@ impl CloudHypervisorAdapter {
             ch_vm_config.memory = Some(models::MemoryConfig {
                 size: mem.size_mib as i64 * 1024 * 1024,
                 shared: Some(true),
+                hugepages: Some(mem.hugepages),
                 ..Default::default()
             });
         }

feos/src/lib.rs

@@ -6,6 +6,7 @@ use feos_proto::{
 };
 use feos_utils::filesystem::mount_virtual_filesystems;
 use feos_utils::host::info::is_running_on_vm;
+use feos_utils::host::memory::configure_hugepages;
 use feos_utils::network::{configure_network_devices, configure_sriov};
 use host_service::{
     api::HostApiHandler, dispatcher::HostServiceDispatcher, Command as HostCommand, RestartSignal,
@@ -32,6 +33,7 @@ use vm_service::{
 };
 
 const VFS_NUM: u32 = 125;
+const HUGEPAGES_NUM: u32 = 1024;
 
 pub async fn run_server(restarted_after_upgrade: bool) -> Result<()> {
     println!(
@@ -63,6 +65,11 @@ pub async fn run_server(restarted_after_upgrade: bool) -> Result<()> {
             info!("MAIN: Mounting virtual filesystems...");
             mount_virtual_filesystems();
 
+            info!("MAIN: Configuring hugepages...");
+            if let Err(e) = configure_hugepages(HUGEPAGES_NUM).await {
+                warn!("Failed to configure hugepages: {e}");
+            }
+
             let is_on_vm = is_running_on_vm().await.unwrap_or_else(|e| {
                 error!("Error checking VM status: {e}");
                 false // Default to false in case of error

feos/tests/integration_tests.rs

@@ -229,7 +229,10 @@ async fn test_create_and_start_vm() -> Result<()> {
             boot_vcpus: 2,
             max_vcpus: 2,
         }),
-        memory: Some(MemoryConfig { size_mib: 2048 }),
+        memory: Some(MemoryConfig {
+            size_mib: 2048,
+            hugepages: false,
+        }),
         image_ref,
         disks: vec![],
         net: vec![],
@@ -332,7 +335,10 @@ async fn test_vm_healthcheck_and_crash_recovery() -> Result<()> {
             boot_vcpus: 1,
             max_vcpus: 1,
         }),
-        memory: Some(MemoryConfig { size_mib: 1024 }),
+        memory: Some(MemoryConfig {
+            size_mib: 1024,
+            hugepages: false,
+        }),
         image_ref,
         disks: vec![],
         net: vec![],

feos/utils/src/host/memory.rs

@@ -0,0 +1,59 @@
+use log::{info, warn};
+use nix::mount::{mount, MsFlags};
+use std::io;
+use tokio::fs;
+
+const HUGEPAGE_FS_TYPE: &[u8] = b"hugetlbfs";
+const HUGEPAGE_MOUNT_POINT: &str = "/dev/hugepages";
+
+pub async fn configure_hugepages(num_pages: u32) -> io::Result<()> {
+    let nr_hugepages_path = "/sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages";
+
+    info!("Attempting to allocate {num_pages} hugepages...");
+    fs::write(nr_hugepages_path, num_pages.to_string()).await?;
+    info!("Successfully wrote to {nr_hugepages_path}");
+
+    let allocated_pages_str = fs::read_to_string(nr_hugepages_path).await?;
+    let allocated_pages = allocated_pages_str.trim().parse::<u32>().unwrap_or(0);
+
+    if allocated_pages < num_pages {
+        warn!(
+            "System only allocated {allocated_pages} of the requested {num_pages} hugepages. This might happen due to memory fragmentation."
+        );
+    } else {
+        info!("System successfully allocated {allocated_pages} hugepages.");
+    }
+
+    if !is_mounted(HUGEPAGE_MOUNT_POINT).await {
+        info!("Mounting hugetlbfs at {HUGEPAGE_MOUNT_POINT}...");
+        fs::create_dir_all(HUGEPAGE_MOUNT_POINT).await?;
+        mount_hugetlbfs()?;
+        info!("Successfully mounted hugetlbfs.");
+    } else {
+        info!("hugetlbfs is already mounted at {HUGEPAGE_MOUNT_POINT}.");
+    }
+
+    Ok(())
+}
+
+fn mount_hugetlbfs() -> Result<(), io::Error> {
+    const NONE: Option<&'static [u8]> = None;
+    mount(
+        Some(b"none".as_ref()),
+        HUGEPAGE_MOUNT_POINT,
+        Some(HUGEPAGE_FS_TYPE),
+        MsFlags::empty(),
+        NONE,
+    )
+    .map_err(|e| io::Error::other(format!("Failed to mount hugetlbfs: {e}")))
+}
+
+async fn is_mounted(path: &str) -> bool {
+    let Ok(mounts) = fs::read_to_string("/proc/mounts").await else {
+        return false;
+    };
+    mounts.lines().any(|line| {
+        let parts: Vec<&str> = line.split_whitespace().collect();
+        parts.get(1) == Some(&path)
+    })
+}

feos/utils/src/host/mod.rs

@@ -1,2 +1,3 @@
 pub mod info;
+pub mod memory;
 pub mod power;

proto/v1/vm.proto

@@ -135,6 +135,7 @@ message CpuConfig {
 
 message MemoryConfig {
   uint64 size_mib = 1; // Memory size in Megabytes (MiB).
+  bool hugepages = 2;
 }
 
 message DiskConfig {