Split balloon and hot plug memory options, remove balloonMem, add balloon, initialBalloonMem, hotplugMem, hotpluggedMem options, no longer add balloonMem to normal memory

- ballooning and hot plug memory are totally different things.
- balloonMem didn't make much sense as all except one vmm only support enabling balloning.
  For cloud-hypervisor there is a new initialBalloonMem option.
- For cloud-hypervisor hot plug memory there is a new and separate hotpluggedMem option to control the initial size.
- balloon is described as way to remove memory and decrease the configured memory.
  Adding it to the normal memory doesn't make much sense in that regard.

Author: SuperSandro2000

lib/runners/alioth.nix

@@ -7,13 +7,21 @@ let
   inherit (pkgs) lib;
   inherit (microvmConfig)
     user
-    vcpu mem interfaces volumes shares devices vsock
+    vcpu mem balloon initialBalloonMem hotplugMem hotpluggedMem interfaces volumes shares devices vsock
     kernel initrdPath
     storeDisk storeOnDisk;
 in {
   command =
     if user != null
     then throw "alioth will not change user"
+    else if balloon
+    then throw "balloon not implemented for alioth"
+    else if initialBalloonMem != 0
+    then throw "initialBalloonMem not implemented for alioth"
+    else if hotplugMem != 0
+    then throw "alioth does not support hotplugMem"
+    else if hotpluggedMem != 0
+    then throw "alioth does not support hotpluggedMem"
     else builtins.concatStringsSep " " (
       [
         "${pkgs.alioth}/bin/alioth" "run"

lib/runners/cloud-hypervisor.nix

@@ -5,7 +5,7 @@
 
 let
   inherit (pkgs) lib;
-  inherit (microvmConfig) vcpu mem balloonMem deflateOnOOM user interfaces volumes shares socket devices hugepageMem graphics storeDisk storeOnDisk kernel initrdPath;
+  inherit (microvmConfig) vcpu mem balloon initialBalloonMem deflateOnOOM hotplugMem hotpluggedMem user interfaces volumes shares socket devices hugepageMem graphics storeDisk storeOnDisk kernel initrdPath;
   inherit (microvmConfig.cloud-hypervisor) extraArgs;
 
   kernelPath = {
@@ -20,8 +20,7 @@ let
     then "console=ttyAMA0"
     else "";
 
-  # balloon
-  useBallooning = balloonMem > 0;
+  useHotPlugMemory = hotplugMem > 0;
 
   useVirtiofs = builtins.any ({ proto, ... }: proto == "virtiofs") shares;
 
@@ -41,19 +40,19 @@ let
     shared = if useVirtiofs || graphics.enable then "on" else "off";
   }
   # add ballooning options and override 'size' key
-  // lib.optionalAttrs useBallooning {
-    size = "${toString (mem + balloonMem)}M";
+  // lib.optionalAttrs useHotPlugMemory {
+    size = "${toString (mem + hotplugMem)}M";
     hotplug_method = "virtio-mem";
-    hotplug_size = "${toString balloonMem}M";
-    hotplugged_size = "${toString balloonMem}M";
+    hotplug_size = "${toString hotplugMem}M";
+    hotplugged_size = "${toString hotpluggedMem}M";
   }
   # enable hugepages (shared option is ignored by CHV)
   // lib.optionalAttrs hugepageMem {
     hugepages = "on";
   });
 
   balloonOps = opsMapped ({
-    size = "${toString balloonMem}M";
+    size = "${toString initialBalloonMem}M";
     free_page_reporting = "on";
   }
   # enable deflating memory balloon on out-of-memory
@@ -158,7 +157,7 @@ in {
         "--gpu" "socket=${graphics.socket}"
       ]
       ++
-      lib.optionals useBallooning [ "--balloon" balloonOps ]
+      lib.optionals balloon [ "--balloon" balloonOps ]
       ++
       arg "--disk" (
         lib.optional storeOnDisk (opsMapped ({

lib/runners/crosvm.nix

@@ -7,7 +7,7 @@ let
   inherit (pkgs) lib;
   inherit (pkgs.stdenv) system;
   inherit (microvmConfig)
-    vcpu mem balloonMem user volumes shares
+    vcpu mem balloon initialBalloonMem hotplugMem hotpluggedMem user volumes shares
     socket devices vsock graphics
     kernel initrdPath storeDisk storeOnDisk;
   inherit (microvmConfig.crosvm) pivotRoot extraArgs;
@@ -46,15 +46,23 @@ in {
   command =
     if user != null
     then throw "crosvm will not change user"
+    else if initialBalloonMem != 0
+    then throw "crosvm does not support initialBalloonMem"
+    else if hotplugMem != 0
+    then throw "crosvm does not support hotplugMem"
+    else if hotpluggedMem != 0
+    then throw "crosvm does not support hotpluggedMem"
     else lib.escapeShellArgs (
       [
         "${pkgs.crosvm}/bin/crosvm" "run"
-        "-m" (toString (mem + balloonMem))
+        "-m" (toString mem)
         "-c" (toString vcpu)
         "--serial" "type=stdout,console=true,stdin=true"
         "-p" "console=ttyS0 reboot=k panic=1 ${builtins.unsafeDiscardStringContext (toString microvmConfig.kernelParams)}"
       ]
       ++
+      lib.optional (!balloon) "--no-balloon"
+      ++
       lib.optionals storeOnDisk [
         "-r" storeDisk
       ]

lib/runners/firecracker.nix

@@ -7,7 +7,7 @@ let
   inherit (pkgs) lib system;
   inherit (microvmConfig)
     hostName user socket preStart
-    vcpu mem
+    vcpu mem balloon initialBalloonMem hotplugMem hotpluggedMem
     interfaces volumes shares devices
     kernel initrdPath
     storeDisk;
@@ -72,6 +72,14 @@ in {
     then throw "9p/virtiofs shares not implemented for Firecracker"
     else if devices != []
     then throw "devices passthrough not implemented for Firecracker"
+    else if balloon
+    then throw "balloon not implemented for Firecracker"
+    else if initialBalloonMem != 0
+    then throw "initialBalloonMem not implemented for Firecracker"
+    else if hotplugMem != 0
+    then throw "hotplugMem not implemented for Firecracker"
+    else if hotpluggedMem != 0
+    then throw "hotpluggedMem not implemented for Firecracker"
     else lib.escapeShellArgs [
       "${pkgs.firecracker}/bin/firecracker"
       "--config-file" configFile

lib/runners/kvmtool.nix

@@ -7,7 +7,7 @@ let
   inherit (pkgs) lib;
   inherit (microvmConfig)
     hostName preStart user
-    vcpu mem balloonMem interfaces volumes shares devices vsock
+    vcpu mem balloon initialBalloonMem hotplugMem hotpluggedMem interfaces volumes shares devices vsock
     kernel initrdPath
     storeDisk storeOnDisk;
 in {
@@ -19,11 +19,17 @@ in {
   command =
     if user != null
     then throw "kvmtool will not change user"
+    else if initialBalloonMem != 0
+    then throw "kvmtool does not support initialBalloonMem"
+    else if hotplugMem != 0
+    then throw "kvmtool does not support hotplugMem"
+    else if hotpluggedMem != 0
+    then throw "kvmtool does not support hotpluggedMem"
     else builtins.concatStringsSep " " (
       [
         "${pkgs.kvmtool}/bin/lkvm" "run"
         "--name" (lib.escapeShellArg hostName)
-        "-m" (toString (mem + balloonMem))
+        "-m" (toString mem)
         "-c" (toString vcpu)
         "--console" "serial"
         "--rng"
@@ -36,9 +42,9 @@ in {
         "-d" (lib.escapeShellArg "${storeDisk},ro")
       ]
       ++
-      lib.optionals (balloonMem > 0) [ "--balloon" ]
+      lib.optionals balloon [ "--balloon" ]
       ++
-      builtins.concatMap ({ image, serial, direct, readOnly, ... }:
+      builtins.concatMap ({ serial, direct, readOnly, ... }:
         lib.warnIf (serial != null) ''
           Volume serial is not supported for kvmtool
         ''

lib/runners/qemu.nix

@@ -39,7 +39,7 @@ let
   qemu = overrideQemu (if microvmConfig.cpu == null then
     pkgs.qemu_kvm else pkgs.buildPackages.qemu_full);
 
-  inherit (microvmConfig) hostName cpu vcpu mem balloonMem deflateOnOOM user interfaces shares socket forwardPorts devices vsock graphics storeOnDisk kernel initrdPath storeDisk;
+  inherit (microvmConfig) hostName vcpu mem balloon initialBalloonMem deflateOnOOM hotplugMem hotpluggedMem user interfaces shares socket forwardPorts devices vsock graphics storeOnDisk kernel initrdPath storeDisk;
   inherit (microvmConfig.qemu) machine extraArgs serialConsole;
 
   inherit (import ../. { inherit (pkgs) lib; }) withDriveLetters;
@@ -156,12 +156,18 @@ lib.warnIf (mem == 2048) ''
 {
   inherit tapMultiQueue;
 
-  command = lib.escapeShellArgs (
+  command = if initialBalloonMem != 0
+  then throw "qemu does not support initialBalloonMem"
+  else if hotplugMem != 0
+  then throw "qemu does not support hotplugMem"
+  else if hotpluggedMem != 0
+  then throw "qemu does not support hotpluggedMem"
+  else lib.escapeShellArgs (
     [
       "${qemu}/bin/qemu-system-${arch}"
       "-name" hostName
       "-M" machineConfig
-      "-m" (toString (mem + balloonMem))
+      "-m" (toString mem)
       "-smp" (toString vcpu)
       "-nodefaults" "-no-user-config"
       # qemu just hangs after shutdown, allow to exit by rebooting
@@ -208,7 +214,7 @@ lib.warnIf (mem == 2048) ''
     ] ++
     lib.optionals (user != null) [ "-user" user ] ++
     lib.optionals (socket != null) [ "-qmp" "unix:${socket},server,nowait" ] ++
-    lib.optionals (balloonMem > 0) [
+    lib.optionals balloon [
 	 "-device" ("virtio-balloon,free-page-reporting=on,id=balloon0" + lib.optionalString (deflateOnOOM) ",deflate-on-oom=on")
     ] ++
     builtins.concatMap ({ image, letter, serial, direct, readOnly, ... }:
@@ -224,7 +230,7 @@ lib.warnIf (mem == 2048) ''
     ) volumes ++
     lib.optionals (shares != []) (
       [
-        "-object" "memory-backend-memfd,id=mem,size=${toString (mem + balloonMem)}M,share=on"
+        "-object" "memory-backend-memfd,id=mem,size=${toString mem}M,share=on"
         "-numa" "node,memdev=mem"
       ] ++
       builtins.concatMap ({ proto, index, socket, source, tag, securityModel, ... }: {

lib/runners/stratovirt.nix

@@ -8,7 +8,7 @@ let
 
   inherit (microvmConfig)
     hostName
-    vcpu mem interfaces shares socket forwardPorts devices
+    vcpu mem balloon initialBalloonMem hotplugMem hotpluggedMem interfaces shares socket forwardPorts devices
     kernel initrdPath
     storeOnDisk storeDisk;
 
@@ -70,7 +70,15 @@ let
 in {
   inherit tapMultiQueue;
 
-  command = lib.escapeShellArgs (
+  command = if balloon
+    then throw "balloon not implemented for stratovirt"
+    else if initialBalloonMem != 0
+    then throw "initialBalloonMem not implemented for stratovirt"
+    else if hotplugMem != 0
+    then throw "stratovirt does not support hotplugMem"
+    else if hotpluggedMem != 0
+    then throw "stratovirt does not support hotpluggedMem"
+    else lib.escapeShellArgs (
     [
       "${pkgs.expect}/bin/unbuffer"
       "${pkgs.stratovirt}/bin/stratovirt"

nixos-modules/microvm/options.nix

@@ -111,16 +111,47 @@ in
       '';
     };
 
-    balloonMem = mkOption {
+    hotplugMem = mkOption {
       description = ''
-        Amount of balloon memory in megabytes
+        Amount of hotplug memory in megabytes.
 
-        The way virtio-balloon works is that this is the memory size
-        that the host can request to be freed by the VM. Initial
-        booting of the VM allocates mem+balloonMem megabytes of RAM.
+        This describes the maximum amount of memory that can be dynamically added to the VM with virtio-mem.
       '';
       default = 0;
-      type = types.int;
+      type = types.ints.unsigned;
+    };
+
+    hotpluggedMem = mkOption {
+      description = ''
+        Amount of hotplugged memory in megabytes.
+
+        This basically describes the amount of hotplug memory the VM starts with.
+      '';
+      default = config.microvm.hotplugMem;
+      type = types.ints.unsigned;
+    };
+
+    balloon = mkOption {
+      description = ''
+        Whether to enable ballooning.
+
+        By "inflating" or increasing the balloon the host can reduce the VMs
+        memory amount and reclaim it for itself.
+        When "deflating" or decreasing the balloon the host can give the memory
+        back to the VM.
+
+        virtio-mem is recommended over ballooning if supported by the hypervisor.
+      '';
+      default = false;
+      type = types.bool;
+    };
+
+    initialBalloonMem = mkOption {
+      description = ''
+        Amount of initial balloon memory in megabytes.
+      '';
+      default = 0;
+      type = types.ints.unsigned;
     };
 
     deflateOnOOM = mkOption {
@@ -614,6 +645,10 @@ in
     };
   };
 
+  imports = [
+    (lib.mkRemovedOptionModule ["microvm" "balloonMem"] "The balloonMem option has been removed and replaced by the boolean option balloon")
+  ];
+
   config = lib.mkMerge [ {
     microvm.qemu.machine =
       lib.mkIf (pkgs.stdenv.system == "x86_64-linux") (