nixos/test-driver: allow assigning other vsock number ranges (#405508)

Author: tfc

nixos/doc/manual/development/running-nixos-tests-interactively.section.md

@@ -87,11 +87,33 @@ $ ssh vsock/3 -o User=root
 
 The socket numbers correspond to the node number of the test VM, but start
 at three instead of one because that's the lowest possible
-vsock number.
+vsock number. The exact SSH commands are also printed out when starting
+`nixos-test-driver`.
 
 On non-NixOS systems you'll probably need to enable
 the SSH config from {manpage}`systemd-ssh-proxy(1)` yourself.
 
+If starting VM fails with an error like
+
+```
+qemu-system-x86_64: -device vhost-vsock-pci,guest-cid=3: vhost-vsock: unable to set guest cid: Address already in use
+```
+
+it means that the vsock numbers for the VMs are already in use. This can happen
+if another interactive test with SSH backdoor enabled is running on the machine.
+
+In that case, you need to assign another range of vsock numbers. You can pick another
+offset with
+
+```nix
+{
+  sshBackdoor = {
+    enable = true;
+    vsockOffset = 23542;
+  };
+}
+```
+
 ## Port forwarding to NixOS test VMs {#sec-nixos-test-port-forwarding}
 
 If your test has only a single VM, you may use e.g.

nixos/doc/manual/redirects.json

@@ -1832,6 +1832,9 @@
   "test-opt-sshBackdoor.enable": [
     "index.html#test-opt-sshBackdoor.enable"
   ],
+  "test-opt-sshBackdoor.vsockOffset": [
+    "index.html#test-opt-sshBackdoor.vsockOffset"
+  ],
   "test-opt-defaults": [
     "index.html#test-opt-defaults"
   ],

nixos/lib/test-driver/src/test_driver/__init__.py

@@ -112,7 +112,7 @@ def main() -> None:
     arg_parser.add_argument(
         "--dump-vsocks",
         help="indicates that the interactive SSH backdoor is active and dumps information about it on start",
-        action="store_true",
+        type=int,
     )
 
     args = arg_parser.parse_args()
@@ -141,8 +141,8 @@ def main() -> None:
         if args.interactive:
             history_dir = os.getcwd()
             history_path = os.path.join(history_dir, ".nixos-test-history")
-            if args.dump_vsocks:
-                driver.dump_machine_ssh()
+            if offset := args.dump_vsocks:
+                driver.dump_machine_ssh(offset)
             ptpython.ipython.embed(
                 user_ns=driver.test_symbols(),
                 history_filename=history_path,

nixos/lib/test-driver/src/test_driver/driver.py

@@ -178,14 +178,14 @@ def subtest(name: str) -> Iterator[None]:
         )
         return {**general_symbols, **machine_symbols, **vlan_symbols}
 
-    def dump_machine_ssh(self) -> None:
+    def dump_machine_ssh(self, offset: int) -> None:
         print("SSH backdoor enabled, the machines can be accessed like this:")
         print(
             f"{Style.BRIGHT}Note:{Style.RESET_ALL} this requires {Style.BRIGHT}systemd-ssh-proxy(1){Style.RESET_ALL} to be enabled (default on NixOS 25.05 and newer)."
         )
         names = [machine.name for machine in self.machines]
         longest_name = len(max(names, key=len))
-        for num, name in enumerate(names, start=3):
+        for num, name in enumerate(names, start=offset + 1):
             spaces = " " * (longest_name - len(name) + 2)
             print(
                 f"    {name}:{spaces}{Style.BRIGHT}ssh -o User=root vsock/{num}{Style.RESET_ALL}"

nixos/lib/testing/nodes.nix

@@ -84,6 +84,22 @@ in
         type = types.bool;
         description = "Whether to turn on the VSOCK-based access to all VMs. This provides an unauthenticated access intended for debugging.";
       };
+      vsockOffset = mkOption {
+        default = 2;
+        type = types.ints.between 2 4294967296;
+        description = ''
+          This field is only relevant when multiple users run the (interactive)
+          driver outside the sandbox and with the SSH backdoor activated.
+          The typical symptom for this being a problem are error messages like this:
+          `vhost-vsock: unable to set guest cid: Address already in use`
+
+          This option allows to assign an offset to each vsock number to
+          resolve this.
+
+          This is a 32bit number. The lowest possible vsock number is `3`
+          (i.e. with the lowest node number being `1`, this is 2+1).
+        '';
+      };
     };
 
     node.type = mkOption {
@@ -182,17 +198,39 @@ in
     passthru.nodes = config.nodesCompat;
 
     extraDriverArgs = mkIf config.sshBackdoor.enable [
-      "--dump-vsocks"
+      "--dump-vsocks=${toString config.sshBackdoor.vsockOffset}"
     ];
 
     defaults = mkMerge [
       (mkIf config.node.pkgsReadOnly {
         nixpkgs.pkgs = config.node.pkgs;
         imports = [ ../../modules/misc/nixpkgs/read-only.nix ];
       })
-      (mkIf config.sshBackdoor.enable {
-        testing.sshBackdoor.enable = true;
-      })
+      (mkIf config.sshBackdoor.enable (
+        let
+          inherit (config.sshBackdoor) vsockOffset;
+        in
+        { config, ... }:
+        {
+          services.openssh = {
+            enable = true;
+            settings = {
+              PermitRootLogin = "yes";
+              PermitEmptyPasswords = "yes";
+            };
+          };
+
+          security.pam.services.sshd = {
+            allowNullPassword = true;
+          };
+
+          virtualisation.qemu.options = [
+            "-device vhost-vsock-pci,guest-cid=${
+              toString (config.virtualisation.test.nodeNumber + vsockOffset)
+            }"
+          ];
+        }
+      ))
     ];
 
   };

nixos/modules/testing/test-instrumentation.nix

@@ -86,11 +86,6 @@ in
       enables commands to be sent to test and debug stage 1. Use
       machine.switch_root() to leave stage 1 and proceed to stage 2
     '';
-
-    sshBackdoor = {
-      enable = mkEnableOption "vsock-based ssh backdoor for the VM";
-    };
-
   };
 
   config = {
@@ -104,18 +99,6 @@ in
       }
     ];
 
-    services.openssh = mkIf config.testing.sshBackdoor.enable {
-      enable = true;
-      settings = {
-        PermitRootLogin = "yes";
-        PermitEmptyPasswords = "yes";
-      };
-    };
-
-    security.pam.services.sshd = mkIf config.testing.sshBackdoor.enable {
-      allowNullPassword = true;
-    };
-
     systemd.services.backdoor = lib.mkMerge [
       backdoorService
       {
@@ -191,10 +174,6 @@ in
         #       we avoid defining attributes if not possible.
         # TODO: refactor such that test-instrumentation can import qemu-vm
         package = lib.mkDefault pkgs.qemu_test;
-
-        options = mkIf config.testing.sshBackdoor.enable [
-          "-device vhost-vsock-pci,guest-cid=${toString (config.virtualisation.test.nodeNumber + 2)}"
-        ];
       };
     };