diff --git a/lib/default.nix b/lib/default.nix index efa88964..25223679 100644 --- a/lib/default.nix +++ b/lib/default.nix @@ -29,49 +29,6 @@ rec { lib.drop offset lib.strings.lowerChars )); - createVolumesScript = pkgs: volumes: - lib.optionalString (volumes != []) ( - lib.optionalString (lib.any (v: v.autoCreate) volumes) '' - PATH=$PATH:${with pkgs.buildPackages; lib.makeBinPath [ coreutils util-linux e2fsprogs xfsprogs dosfstools btrfs-progs ]} - '' + - pkgs.lib.concatMapStringsSep "\n" ( - { image - , label - , size ? throw "Specify a size for volume ${image} or use autoCreate = false" - , mkfsExtraArgs - , fsType ? defaultFsType - , autoCreate ? true - , ... - }: pkgs.lib.warnIf - (label != null && !autoCreate) "Volume is not automatically labeled unless autoCreate is true. Volume has to be labeled manually, otherwise it will not be identified" - (let labelOption = - if autoCreate then - (if builtins.elem fsType ["ext2" "ext3" "ext4" "xfs" "btrfs"] then "-L" - else if fsType == "vfat" then "-n" - else (pkgs.lib.warnIf (label != null) - "Will not label volume ${label} with filesystem type ${fsType}. Open an issue on the microvm.nix project to request a fix." - null)) - else null; - labelArgument = - if (labelOption != null && label != null) then "${labelOption} '${label}'" - else ""; - mkfsExtraArgsString = - if mkfsExtraArgs != null - then lib.escapeShellArgs mkfsExtraArgs - else " "; - in (lib.optionalString autoCreate '' - - if [ ! -e '${image}' ]; then - touch '${image}' - # Mark NOCOW - chattr +C '${image}' || true - truncate -s ${toString size}M '${image}' - mkfs.${fsType} ${labelArgument} ${mkfsExtraArgsString} '${image}' - fi - '')) - ) volumes - ); - buildRunner = import ./runner.nix; makeMacvtap = { microvmConfig, hypervisorConfig }: diff --git a/lib/runner.nix b/lib/runner.nix index cc4b0d5d..c3031054 100644 --- a/lib/runner.nix +++ b/lib/runner.nix @@ -6,9 +6,10 @@ let inherit (pkgs) lib; - inherit (microvmConfig) hostName; + inherit (microvmConfig) hostName vmHostPackages; - inherit (import ./. { inherit lib; }) createVolumesScript makeMacvtap withDriveLetters extractOptValues extractParamValue; + inherit (import ./. { inherit lib; }) makeMacvtap withDriveLetters extractOptValues extractParamValue; + inherit (import ./volumes.nix { inherit lib microvmConfig; }) createVolumesScript; inherit (makeMacvtap { inherit microvmConfig hypervisorConfig; }) openMacvtapFds macvtapFds; @@ -26,22 +27,12 @@ let execArg = lib.optionalString microvmConfig.prettyProcnames ''-a "microvm@${hostName}"''; - vmHostPackages = - if microvmConfig.cpu == null - then - # When cross-compiling for a target host, select packages for - # the target: - pkgs - else - # When cross-compiling for CPU emulation in qemu, select - # packages for the host: - pkgs.buildPackages; binScripts = microvmConfig.binScripts // { microvm-run = '' set -eou pipefail ${preStart} - ${createVolumesScript vmHostPackages microvmConfig.volumes} + ${createVolumesScript microvmConfig.volumes} ${lib.optionalString (hypervisorConfig.requiresMacvtapAsFds or false) openMacvtapFds} exec ${execArg} ${command} @@ -63,11 +54,11 @@ let }; binScriptPkgs = lib.mapAttrs (scriptName: lines: - pkgs.writeShellScript "microvm-${hostName}-${scriptName}" lines + vmHostPackages.writeShellScript "microvm-${hostName}-${scriptName}" lines ) binScripts; in -pkgs.buildPackages.runCommand "microvm-${microvmConfig.hypervisor}-${hostName}" +vmHostPackages.buildPackages.runCommand "microvm-${microvmConfig.hypervisor}-${hostName}" { # for `nix run` meta.mainProgram = "microvm-run"; diff --git a/lib/runners/qemu.nix b/lib/runners/qemu.nix index cf246f2b..8a931b56 100644 --- a/lib/runners/qemu.nix +++ b/lib/runners/qemu.nix @@ -8,6 +8,7 @@ let inherit (pkgs) lib; inherit (pkgs.stdenv) system; + inherit (microvmConfig) vmHostPackages; enableLibusb = pkg: pkg.overrideAttrs (oa: { configureFlags = oa.configureFlags ++ [ @@ -38,16 +39,11 @@ let ++ lib.optional microvmConfig.optimize.enable minimizeQemuClosureSize ); - qemuPkg = - if microvmConfig.cpu == null - then - # When cross-compiling for a target host, select qemu for the target: - pkgs.qemu_kvm - else - # When cross-compiling for CPU emulation, select qemu for the host: - pkgs.buildPackages.qemu; + qemu = overrideQemu vmHostPackages.qemu; - qemu = overrideQemu qemuPkg; + aioEngine = if vmHostPackages.stdenv.hostPlatform.isLinux + then "io_uring" + else "threads"; inherit (microvmConfig) hostName vcpu mem balloon initialBalloonMem deflateOnOOM hotplugMem hotpluggedMem user interfaces shares socket forwardPorts devices vsock graphics storeOnDisk kernel initrdPath storeDisk credentialFiles; inherit (microvmConfig.qemu) machine extraArgs serialConsole; @@ -72,10 +68,10 @@ let else "host" ) ]; - accel = - if microvmConfig.cpu == null - then "kvm:tcg" - else "tcg"; + accel = if vmHostPackages.stdenv.hostPlatform.isLinux + then "kvm:tcg" else + if vmHostPackages.stdenv.hostPlatform.isDarwin + then "hvf:tcg" else "tcg"; # PCI required by vfio-pci for PCI passthrough pciInDevices = lib.any ({ bus, ... }: bus == "pci") devices; @@ -104,6 +100,9 @@ let inherit accel; gic-version = "max"; }; + aarch64-darwin = { + inherit accel; + }; }.${system}; machineConfig = builtins.concatStringsSep "," ( @@ -129,9 +128,10 @@ let canSandbox = # Don't let qemu sandbox itself if it is going to call qemu-bridge-helper - ! lib.any ({ type, ... }: + (! lib.any ({ type, ... }: type == "bridge" - ) microvmConfig.interfaces; + ) microvmConfig.interfaces) && + (builtins.elem "--enable-seccomp" (qemu.configureFlags or [])); tapMultiQueue = vcpu > 1; @@ -196,7 +196,7 @@ lib.warnIf (mem == 2048) '' lib.optionals serialConsole [ "-serial" "chardev:stdio" ] ++ - lib.optionals (microvmConfig.cpu == null) [ + lib.optionals (vmHostPackages.stdenv.hostPlatform.isLinux && microvmConfig.cpu == null) [ "-enable-kvm" ] ++ cpuArgs ++ @@ -209,7 +209,7 @@ lib.warnIf (mem == 2048) '' "-append" "${kernelConsole} reboot=t panic=-1 ${builtins.unsafeDiscardStringContext (toString microvmConfig.kernelParams)}" ] ++ lib.optionals storeOnDisk [ - "-drive" "id=store,format=raw,read-only=on,file=${storeDisk},if=none,aio=io_uring" + "-drive" "id=store,format=raw,read-only=on,file=${storeDisk},if=none,aio=${aioEngine}" "-device" "virtio-blk-${devType},drive=store${lib.optionalString (devType == "pci") ",disable-legacy=on"}" ] ++ (if graphics.enable @@ -233,7 +233,7 @@ lib.warnIf (mem == 2048) '' ] ++ builtins.concatMap ({ image, letter, serial, direct, readOnly, ... }: [ "-drive" - "id=vd${letter},format=raw,file=${image},if=none,aio=io_uring,discard=unmap${ + "id=vd${letter},format=raw,file=${image},if=none,aio=${aioEngine},discard=unmap${ lib.optionalString (direct != null) ",cache=none" },read-only=${if readOnly then "on" else "off"}" "-device" @@ -243,10 +243,10 @@ lib.warnIf (mem == 2048) '' ] ) volumes ++ lib.optionals (shares != []) ( - [ - "-object" "memory-backend-memfd,id=mem,size=${toString mem}M,share=on" + (lib.optionals vmHostPackages.stdenv.hostPlatform.isLinux [ "-numa" "node,memdev=mem" - ] ++ + "-object" "memory-backend-memfd,id=mem,size=${toString mem}M,share=on" + ]) ++ builtins.concatMap ({ proto, index, socket, source, tag, securityModel, ... }: { "virtiofs" = [ "-chardev" "socket,id=fs${toString index},path=${socket}" @@ -372,7 +372,7 @@ lib.warnIf (mem == 2048) '' # wait for exit cat ) | \ - ${pkgs.socat}/bin/socat STDIO UNIX:${socket},shut-none + ${vmHostPackages.socat}/bin/socat STDIO UNIX:${socket},shut-none '' else throw "Cannot shutdown without socket"; @@ -384,9 +384,9 @@ lib.warnIf (mem == 2048) '' ${writeQmp { execute = "qmp_capabilities"; }} ${writeQmp { execute = "balloon"; arguments.value = 987; }} ) | sed -e s/987/$VALUE/ | \ - ${pkgs.socat}/bin/socat STDIO UNIX:${socket},shut-none | \ + ${vmHostPackages.socat}/bin/socat STDIO UNIX:${socket},shut-none | \ tail -n 1 | \ - ${pkgs.jq}/bin/jq -r .data.actual \ + ${vmHostPackages.jq}/bin/jq -r .data.actual \ ) echo $(( $SIZE / 1024 / 1024 )) '' diff --git a/lib/volumes.nix b/lib/volumes.nix new file mode 100644 index 00000000..fc629cc5 --- /dev/null +++ b/lib/volumes.nix @@ -0,0 +1,79 @@ +{ lib, microvmConfig }: +let + inherit (microvmConfig) vmHostPackages; + inherit (import ../../lib { + inherit lib; + }) defaultFsType; + + fsTypeToUtil = fs: with vmHostPackages; { + "ext2" = e2fsprogs; + "ext3" = e2fsprogs; + "ext4" = e2fsprogs; + "xfs" = xfsprogs; + "btrfs" = btrfs-progs; + "vfat" = dosfstools; + }.${fs} or (throw "Do not know how to handle ${fs}"); + collectFsTypes = volumes: map (v: v.fsType) volumes; + collectFsUtils = volumes: map (fsType: fsTypeToUtil fsType) (collectFsTypes volumes); +in +{ + createVolumesScript = + volumes: + lib.optionalString (volumes != [ ]) ( + lib.optionalString (lib.any (v: v.autoCreate) volumes) '' + PATH=$PATH:${lib.makeBinPath ([ vmHostPackages.coreutils ] ++ (collectFsUtils volumes))} + '' + + lib.concatMapStringsSep "\n" ( + { + image, + label, + size ? throw "Specify a size for volume ${image} or use autoCreate = false", + mkfsExtraArgs, + fsType ? defaultFsType, + autoCreate ? true, + ... + }: + lib.warnIf (label != null && !autoCreate) + "Volume is not automatically labeled unless autoCreate is true. Volume has to be labeled manually, otherwise it will not be identified" + ( + let + labelOption = + if autoCreate then + ( + if + builtins.elem fsType [ + "ext2" + "ext3" + "ext4" + "xfs" + "btrfs" + ] + then + "-L" + else if fsType == "vfat" then + "-n" + else + (lib.warnIf (label != null) + "Will not label volume ${label} with filesystem type ${fsType}. Open an issue on the microvm.nix project to request a fix." + null + ) + ) + else + null; + labelArgument = lib.optionalString (labelOption != null && label != null) "${labelOption} '${label}'"; + mkfsExtraArgsString = if mkfsExtraArgs != null then lib.escapeShellArgs mkfsExtraArgs else " "; + in + (lib.optionalString autoCreate '' + + if [ ! -e '${image}' ]; then + touch '${image}' + # Mark NOCOW + chattr +C '${image}' || true + truncate -s ${toString size}M '${image}' + mkfs.${fsType} ${labelArgument} ${mkfsExtraArgsString} '${image}' + fi + '') + ) + ) volumes + ); +} diff --git a/nixos-modules/microvm/options.nix b/nixos-modules/microvm/options.nix index 5f208914..6c3be2e0 100644 --- a/nixos-modules/microvm/options.nix +++ b/nixos-modules/microvm/options.nix @@ -4,6 +4,7 @@ let inherit lib; }; + cfg = config.microvm; hostName = config.networking.hostName or "$HOSTNAME"; kernelAtLeast = lib.versionAtLeast config.boot.kernelPackages.kernel.version; in @@ -483,6 +484,13 @@ in ''; }; + vmHostPackages = mkOption { + description = "If set, overrides the default host package."; + example = "nixpkgs.legacyPackages.aarch64-darwin.pkgs"; + type = types.nullOr types.pkgs; + default = if cfg.cpu == null then pkgs else pkgs.buildPackages; + }; + qemu.machine = mkOption { type = types.str; description = ''