microvm-nix · esselius · Jul 9, 2025
diff --git a/lib/default.nix b/lib/default.nix
@@ -29,49 +29,6 @@ rec {
       lib.drop offset lib.strings.lowerChars
     ));
 
-  createVolumesScript = pkgs: volumes:
-    lib.optionalString (volumes != []) (
-      lib.optionalString (lib.any (v: v.autoCreate) volumes) ''
-        PATH=$PATH:${with pkgs.buildPackages; lib.makeBinPath [ coreutils util-linux e2fsprogs xfsprogs dosfstools btrfs-progs ]}
-      '' +
-      pkgs.lib.concatMapStringsSep "\n" (
-        { image
-        , label
-        , size ? throw "Specify a size for volume ${image} or use autoCreate = false"
-        , mkfsExtraArgs
-        , fsType ? defaultFsType
-        , autoCreate ? true
-        , ...
-        }: pkgs.lib.warnIf
-          (label != null && !autoCreate) "Volume is not automatically labeled unless autoCreate is true. Volume has to be labeled manually, otherwise it will not be identified"
-          (let labelOption =
-                 if autoCreate then
-                   (if builtins.elem fsType ["ext2" "ext3" "ext4" "xfs" "btrfs"] then "-L"
-                    else if fsType == "vfat" then "-n"
-                    else (pkgs.lib.warnIf (label != null)
-                      "Will not label volume ${label} with filesystem type ${fsType}. Open an issue on the microvm.nix project to request a fix."
-                      null))
-                 else null;
-               labelArgument =
-                 if (labelOption != null && label != null) then "${labelOption} '${label}'"
-                 else "";
-               mkfsExtraArgsString =
-                 if mkfsExtraArgs != null
-                 then lib.escapeShellArgs mkfsExtraArgs
-                 else " ";
-           in (lib.optionalString autoCreate ''
-
-              if [ ! -e '${image}' ]; then
-                touch '${image}'
-                # Mark NOCOW
-                chattr +C '${image}' || true
-                truncate -s ${toString size}M '${image}'
-                mkfs.${fsType} ${labelArgument} ${mkfsExtraArgsString} '${image}'
-              fi
-            ''))
-      ) volumes
-    );
-
   buildRunner = import ./runner.nix;
 
   makeMacvtap = { microvmConfig, hypervisorConfig }:

diff --git a/lib/runner.nix b/lib/runner.nix
@@ -6,9 +6,10 @@
 let
   inherit (pkgs) lib;
 
-  inherit (microvmConfig) hostName;
+  inherit (microvmConfig) hostName vmHostPackages;
 
-  inherit (import ./. { inherit lib; }) createVolumesScript makeMacvtap withDriveLetters extractOptValues extractParamValue;
+  inherit (import ./. { inherit lib; }) makeMacvtap withDriveLetters extractOptValues extractParamValue;
+  inherit (import ./volumes.nix { inherit lib microvmConfig; }) createVolumesScript;
   inherit (makeMacvtap {
     inherit microvmConfig hypervisorConfig;
   }) openMacvtapFds macvtapFds;
@@ -26,22 +27,12 @@ let
   execArg = lib.optionalString microvmConfig.prettyProcnames
     ''-a "microvm@${hostName}"'';
 
-  vmHostPackages =
-    if microvmConfig.cpu == null
-    then
-      # When cross-compiling for a target host, select packages for
-      # the target:
-      pkgs
-    else
-      # When cross-compiling for CPU emulation in qemu, select
-      # packages for the host:
-      pkgs.buildPackages;
 
   binScripts = microvmConfig.binScripts // {
     microvm-run = ''
       set -eou pipefail
       ${preStart}
-      ${createVolumesScript vmHostPackages microvmConfig.volumes}
+      ${createVolumesScript microvmConfig.volumes}
       ${lib.optionalString (hypervisorConfig.requiresMacvtapAsFds or false) openMacvtapFds}
 
       exec ${execArg} ${command}
@@ -63,11 +54,11 @@ let
   };
 
   binScriptPkgs = lib.mapAttrs (scriptName: lines:
-    pkgs.writeShellScript "microvm-${hostName}-${scriptName}" lines
+    vmHostPackages.writeShellScript "microvm-${hostName}-${scriptName}" lines
   ) binScripts;
 in
 
-pkgs.buildPackages.runCommand "microvm-${microvmConfig.hypervisor}-${hostName}"
+vmHostPackages.buildPackages.runCommand "microvm-${microvmConfig.hypervisor}-${hostName}"
 {
   # for `nix run`
   meta.mainProgram = "microvm-run";

diff --git a/lib/runners/qemu.nix b/lib/runners/qemu.nix
@@ -8,6 +8,7 @@
 let
   inherit (pkgs) lib;
   inherit (pkgs.stdenv) system;
+  inherit (microvmConfig) vmHostPackages;
 
   enableLibusb = pkg: pkg.overrideAttrs (oa: {
     configureFlags = oa.configureFlags ++ [
@@ -38,16 +39,11 @@ let
     ++ lib.optional microvmConfig.optimize.enable minimizeQemuClosureSize
   );
 
-  qemuPkg =
-    if microvmConfig.cpu == null
-    then
-      # When cross-compiling for a target host, select qemu for the target:
-      pkgs.qemu_kvm
-    else
-      # When cross-compiling for CPU emulation, select qemu for the host:
-      pkgs.buildPackages.qemu;
+  qemu = overrideQemu vmHostPackages.qemu;
 
-  qemu = overrideQemu qemuPkg;
+  aioEngine = if vmHostPackages.stdenv.hostPlatform.isLinux
+    then "io_uring"
+    else "threads";
 
   inherit (microvmConfig) hostName vcpu mem balloon initialBalloonMem deflateOnOOM hotplugMem hotpluggedMem user interfaces shares socket forwardPorts devices vsock graphics storeOnDisk kernel initrdPath storeDisk credentialFiles;
   inherit (microvmConfig.qemu) machine extraArgs serialConsole;
@@ -72,10 +68,10 @@ let
       else "host"
     ) ];
 
-  accel =
-    if microvmConfig.cpu == null
-    then "kvm:tcg"
-    else "tcg";
+  accel = if vmHostPackages.stdenv.hostPlatform.isLinux
+    then "kvm:tcg" else
+      if vmHostPackages.stdenv.hostPlatform.isDarwin
+      then "hvf:tcg" else "tcg";
 
   # PCI required by vfio-pci for PCI passthrough
   pciInDevices = lib.any ({ bus, ... }: bus == "pci") devices;
@@ -104,6 +100,9 @@ let
         inherit accel;
         gic-version = "max";
       };
+      aarch64-darwin = {
+        inherit accel;
+      };
     }.${system};
 
   machineConfig = builtins.concatStringsSep "," (
@@ -129,9 +128,10 @@ let
 
   canSandbox =
     # Don't let qemu sandbox itself if it is going to call qemu-bridge-helper
-    ! lib.any ({ type, ... }:
+    (! lib.any ({ type, ... }:
       type == "bridge"
-    ) microvmConfig.interfaces;
+    ) microvmConfig.interfaces) &&
+    (builtins.elem "--enable-seccomp" (qemu.configureFlags or []));
 
   tapMultiQueue = vcpu > 1;
 
@@ -196,7 +196,7 @@ lib.warnIf (mem == 2048) ''
     lib.optionals serialConsole [
       "-serial" "chardev:stdio"
     ] ++
-    lib.optionals (microvmConfig.cpu == null) [
+    lib.optionals (vmHostPackages.stdenv.hostPlatform.isLinux && microvmConfig.cpu == null) [
       "-enable-kvm"
     ] ++
     cpuArgs ++
@@ -209,7 +209,7 @@ lib.warnIf (mem == 2048) ''
       "-append" "${kernelConsole} reboot=t panic=-1 ${builtins.unsafeDiscardStringContext (toString microvmConfig.kernelParams)}"
     ] ++
     lib.optionals storeOnDisk [
-      "-drive" "id=store,format=raw,read-only=on,file=${storeDisk},if=none,aio=io_uring"
+      "-drive" "id=store,format=raw,read-only=on,file=${storeDisk},if=none,aio=${aioEngine}"
       "-device" "virtio-blk-${devType},drive=store${lib.optionalString (devType == "pci") ",disable-legacy=on"}"
     ] ++
     (if graphics.enable
@@ -233,7 +233,7 @@ lib.warnIf (mem == 2048) ''
     ] ++
     builtins.concatMap ({ image, letter, serial, direct, readOnly, ... }:
       [ "-drive"
-        "id=vd${letter},format=raw,file=${image},if=none,aio=io_uring,discard=unmap${
+        "id=vd${letter},format=raw,file=${image},if=none,aio=${aioEngine},discard=unmap${
           lib.optionalString (direct != null) ",cache=none"
         },read-only=${if readOnly then "on" else "off"}"
         "-device"
@@ -243,10 +243,10 @@ lib.warnIf (mem == 2048) ''
       ]
     ) volumes ++
     lib.optionals (shares != []) (
-      [
-        "-object" "memory-backend-memfd,id=mem,size=${toString mem}M,share=on"
+      (lib.optionals vmHostPackages.stdenv.hostPlatform.isLinux [
         "-numa" "node,memdev=mem"
-      ] ++
+        "-object" "memory-backend-memfd,id=mem,size=${toString mem}M,share=on"
+      ]) ++
       builtins.concatMap ({ proto, index, socket, source, tag, securityModel, ... }: {
         "virtiofs" = [
           "-chardev" "socket,id=fs${toString index},path=${socket}"
@@ -372,7 +372,7 @@ lib.warnIf (mem == 2048) ''
            # wait for exit
           cat
         ) | \
-        ${pkgs.socat}/bin/socat STDIO UNIX:${socket},shut-none
+        ${vmHostPackages.socat}/bin/socat STDIO UNIX:${socket},shut-none
     ''
     else throw "Cannot shutdown without socket";
 
@@ -384,9 +384,9 @@ lib.warnIf (mem == 2048) ''
         ${writeQmp { execute = "qmp_capabilities"; }}
         ${writeQmp { execute = "balloon"; arguments.value = 987; }}
       ) | sed -e s/987/$VALUE/ | \
-        ${pkgs.socat}/bin/socat STDIO UNIX:${socket},shut-none | \
+        ${vmHostPackages.socat}/bin/socat STDIO UNIX:${socket},shut-none | \
         tail -n 1 | \
-        ${pkgs.jq}/bin/jq -r .data.actual \
+        ${vmHostPackages.jq}/bin/jq -r .data.actual \
       )
       echo $(( $SIZE / 1024 / 1024 ))
     ''

diff --git a/lib/volumes.nix b/lib/volumes.nix
@@ -0,0 +1,79 @@
+{ lib, microvmConfig }:
+let
+  inherit (microvmConfig) vmHostPackages;
+  inherit (import ../../lib {
+    inherit lib;
+  }) defaultFsType;
+
+  fsTypeToUtil = fs: with vmHostPackages; {
+      "ext2" = e2fsprogs;
+      "ext3" = e2fsprogs;
+      "ext4" = e2fsprogs;
+      "xfs" = xfsprogs;
+      "btrfs" = btrfs-progs;
+      "vfat" = dosfstools;
+    }.${fs} or (throw "Do not know how to handle ${fs}");
+  collectFsTypes = volumes: map (v: v.fsType) volumes;
+  collectFsUtils = volumes: map (fsType: fsTypeToUtil fsType) (collectFsTypes volumes);
+in
+{
+  createVolumesScript =
+    volumes:
+    lib.optionalString (volumes != [ ]) (
+      lib.optionalString (lib.any (v: v.autoCreate) volumes) ''
+        PATH=$PATH:${lib.makeBinPath ([ vmHostPackages.coreutils ] ++ (collectFsUtils volumes))}
+      ''
+      + lib.concatMapStringsSep "\n" (
+        {
+          image,
+          label,
+          size ? throw "Specify a size for volume ${image} or use autoCreate = false",
+          mkfsExtraArgs,
+          fsType ? defaultFsType,
+          autoCreate ? true,
+          ...
+        }:
+        lib.warnIf (label != null && !autoCreate)
+          "Volume is not automatically labeled unless autoCreate is true. Volume has to be labeled manually, otherwise it will not be identified"
+          (
+            let
+              labelOption =
+                if autoCreate then
+                  (
+                    if
+                      builtins.elem fsType [
+                        "ext2"
+                        "ext3"
+                        "ext4"
+                        "xfs"
+                        "btrfs"
+                      ]
+                    then
+                      "-L"
+                    else if fsType == "vfat" then
+                      "-n"
+                    else
+                      (lib.warnIf (label != null)
+                        "Will not label volume ${label} with filesystem type ${fsType}. Open an issue on the microvm.nix project to request a fix."
+                        null
+                      )
+                  )
+                else
+                  null;
+              labelArgument = lib.optionalString (labelOption != null && label != null) "${labelOption} '${label}'";
+              mkfsExtraArgsString = if mkfsExtraArgs != null then lib.escapeShellArgs mkfsExtraArgs else " ";
+            in
+            (lib.optionalString autoCreate ''
+
+              if [ ! -e '${image}' ]; then
+                touch '${image}'
+                # Mark NOCOW
+                chattr +C '${image}' || true
+                truncate -s ${toString size}M '${image}'
+                mkfs.${fsType} ${labelArgument} ${mkfsExtraArgsString} '${image}'
+              fi
+            '')
+          )
+      ) volumes
+    );
+}
diff --git a/nixos-modules/microvm/options.nix b/nixos-modules/microvm/options.nix
@@ -4,6 +4,7 @@ let
     inherit lib;
   };
 
+  cfg = config.microvm;
   hostName = config.networking.hostName or "$HOSTNAME";
   kernelAtLeast = lib.versionAtLeast config.boot.kernelPackages.kernel.version;
 in
@@ -483,6 +484,13 @@ in
       '';
     };
 
+    vmHostPackages = mkOption {
+      description = "If set, overrides the default host package.";
+      example = "nixpkgs.legacyPackages.aarch64-darwin.pkgs";
+      type = types.nullOr types.pkgs;
+      default = if cfg.cpu == null then pkgs else pkgs.buildPackages;
+    };
+
     qemu.machine = mkOption {
       type = types.str;
       description = ''