Implement PCI device management via vhotplug

Signed-off-by: Yuri Nesterov <yuriy.nesterov@unikie.com>

Author: nesteroff

flake.lock

@@ -180,6 +180,16 @@
       inputs.nixpkgs.follows = "nixpkgs";
     };
 
+    # Hot-plugging USB devices into virtual machines
+    vhotplug = {
+      url = "github:tiiuae/vhotplug";
+      inputs = {
+        nixpkgs.follows = "nixpkgs";
+        flake-parts.follows = "flake-parts";
+        treefmt-nix.follows = "treefmt-nix";
+      };
+    };
+
     # A UI for the one true VPN: Wireguard
     wireguard-gui = {
       url = "github:tiiuae/wireguard-gui";

flake.nix

@@ -11,31 +11,22 @@ let
     mkOption
     mkIf
     types
-    flatten
-    concatMapStringsSep
     ;
   cfg = config.ghaf.services.kill-switch;
 
-  netvmName = "net-vm";
-  audiovmName = "audio-vm";
-  inherit (config.microvm) stateDir;
   supportedDevices = [
     "mic"
     "net"
     "cam"
   ];
 
-  # Flatten the list of devices
   audioPciDevices =
     if config.ghaf.virtualization.microvm.audiovm.enable then
-      flatten (map (device: "${device.vendorId}:${device.productId}") config.ghaf.common.hardware.audio)
+      config.ghaf.common.hardware.audio
     else
       [ ];
   netPciDevices =
-    if config.ghaf.virtualization.microvm.netvm.enable then
-      flatten (map (device: "${device.vendorId}:${device.productId}") config.ghaf.common.hardware.nics)
-    else
-      [ ];
+    if config.ghaf.virtualization.microvm.netvm.enable then config.ghaf.common.hardware.nics else [ ];
   camUsbDevices = builtins.filter (
     d: lib.hasPrefix "cam" d.name
   ) config.ghaf.hardware.definition.usb.devices;
@@ -44,7 +35,6 @@ let
     name = "ghaf-killswitch";
     runtimeInputs = with pkgs; [
       coreutils
-      hotplug
       vhotplug
     ];
 
@@ -95,28 +85,25 @@ let
           fi
       fi
 
-      find_vm_devices() {
-
-        case "$device" in
-        mic)
-          vm_name="${audiovmName}"
-          ${concatMapStringsSep "\n" (pciDevice: ''
-            devices+=("${pciDevice}")
-          '') audioPciDevices}
-        ;;
-        net)
-          vm_name="${netvmName}"
-          ${concatMapStringsSep "\n" (pciDevice: ''
-            devices+=("${pciDevice}")
-          '') netPciDevices}
-        ;;
-        esac
-      }
-
       block_devices() {
-        if [[ "$device" == "net" || "$device" == "mic" ]]; then
-          echo "Blocking $device device ..."
-          hotplug --detach-pci "''${devices[@]}" --data-path "$state_path" --socket-path "$socket_path"
+        if [[ "$device" == "net" ]]; then
+          echo "Blocking net device ..."
+          ${lib.concatStringsSep "\n" (
+            map (d: ''
+              vhotplugcli pci detach \
+                ${if d.vendorId == null then "" else "--vid ${d.vendorId}"} \
+                ${if d.productId == null then "" else "--did ${d.productId}"}
+            '') netPciDevices
+          )}
+        elif [[ "$device" == "mic" ]]; then
+          echo "Blocking mic device ..."
+          ${lib.concatStringsSep "\n" (
+            map (d: ''
+              vhotplugcli pci detach \
+                ${if d.vendorId == null then "" else "--vid ${d.vendorId}"} \
+                ${if d.productId == null then "" else "--did ${d.productId}"}
+            '') audioPciDevices
+          )}
         else
           ${lib.concatStringsSep "\n" (
             map (d: ''
@@ -135,14 +122,24 @@ let
       }
 
       unblock_devices() {
-        if [[ "$device" == "net" || "$device" == "mic" ]]; then
-          echo "Unblocking $device device ..."
-          if ! hotplug --attach-pci --data-path "$state_path" --socket-path "$socket_path"; then
-            echo "Failed to attach PCI devices. Check systemctl status microvm@$vm_name.service"
-            # Recovery from failed attach; restart the VM
-            echo "Fallback: restarting $vm_name..."
-            systemctl restart microvm@"$vm_name".service
-          fi
+        if [[ "$device" == "net" ]]; then
+          echo "Unblocking net device ..."
+          ${lib.concatStringsSep "\n" (
+            map (d: ''
+              vhotplugcli pci attach \
+                ${if d.vendorId == null then "" else "--vid ${d.vendorId}"} \
+                ${if d.productId == null then "" else "--did ${d.productId}"}
+            '') netPciDevices
+          )}
+        elif [[ "$device" == "mic" ]]; then
+        echo "Unblocking mic device ..."
+          ${lib.concatStringsSep "\n" (
+            map (d: ''
+              vhotplugcli pci attach \
+                ${if d.vendorId == null then "" else "--vid ${d.vendorId}"} \
+                ${if d.productId == null then "" else "--did ${d.productId}"}
+            '') audioPciDevices
+          )}
         else
           ${lib.concatStringsSep "\n" (
             map (d: ''
@@ -168,15 +165,6 @@ let
           echo "$device is not supported"
           exit 1
         fi
-
-        # Find VM name and the devices
-        vm_name=""
-        declare -a devices
-        find_vm_devices
-
-        # Set socket and state path
-        socket_path="${stateDir}/$vm_name/$vm_name.sock"
-        state_path="${stateDir}/$vm_name/state"
       fi
 
       case "$cmd" in

modules/common/services/killswitch.nix

@@ -37,8 +37,11 @@ let
   # is used to determine suspend actions for PCI devices at runtime
   pciDevices = flatten (
     map (device: "${device.vendorId}:${device.productId}") (
-      config.ghaf.common.hardware.nics ++ config.ghaf.common.hardware.audio
+      lib.filter (d: d.vendorId != null && d.productId != null) (
+        config.ghaf.common.hardware.nics ++ config.ghaf.common.hardware.audio
+      )
     )
+
   );
 
   # List of VMs that are running a fake suspend
@@ -82,7 +85,7 @@ let
       pkgs.systemd
       pkgs.coreutils
       pkgs.grpcurl
-      pkgs.hotplug
+      pkgs.vhotplug
       pkgs.wait-for-unit
     ];
     text = ''
@@ -141,21 +144,15 @@ let
           ;;
 
         pci-suspend)
-          socket_path="${config.microvm.stateDir}/$vm_name/$vm_name.sock"
-          state_path="${config.microvm.stateDir}/$vm_name/pci-state"
           case "$action" in
             suspend)
-              declare -a pci_devices
-              ${concatMapStringsSep "\n" (pciDevice: ''
-                pci_devices+=("${pciDevice}")
-              '') pciDevices}
-
               echo "Suspending PCI devices for $vm_name..."
-              hotplug --detach-pci "''${pci_devices[@]}" --data-path "$state_path" --socket-path "$socket_path"
+              vhotplugcli pci suspend --vm "$vm_name"
+
               ;;
             resume)
               echo "Resuming PCI devices for $vm_name..."
-              if ! hotplug --attach-pci --data-path "$state_path" --socket-path "$socket_path"; then
+              if ! vhotplugcli pci resume --vm "$vm_name"; then
                 echo "Failed to attach PCI devices for $vm_name. Please check the logs."
                 # Recovery from failed attach; restart the VM
                 echo "Fallback: restarting $vm_name..."

modules/common/services/power.nix

@@ -3,6 +3,7 @@
 {
   imports = [
     ./usb/external-devices.nix
+    ./usb/internal-devices.nix
     ./usb/vhotplug.nix
     ./usb/quirks.nix
     ./devices.nix