nixos/k3s: use dynamic networking in multi node test (#381122)

Author: ck3d

nixos/tests/k3s/multi-node.nix

@@ -60,7 +60,7 @@ import ../make-test-python.nix (
 
     nodes = {
       server =
-        { pkgs, ... }:
+        { nodes, pkgs, ... }:
         {
           environment.systemPackages = with pkgs; [
             gzip
@@ -83,8 +83,12 @@ import ../make-test-python.nix (
               "--disable metrics-server"
               "--disable servicelb"
               "--disable traefik"
-              "--node-ip 192.168.1.1"
               "--pause-image test.local/pause:local"
+              "--node-ip ${nodes.server.networking.primaryIPAddress}"
+              # The interface selection logic of flannel would normally use eth0, as the nixos
+              # testing driver sets a default route via dev eth0. However, in test setups we
+              # have to use eth1 for inter-node communication.
+              "--flannel-iface eth1"
             ];
           };
           networking.firewall.allowedTCPPorts = [
@@ -93,19 +97,10 @@ import ../make-test-python.nix (
             6443
           ];
           networking.firewall.allowedUDPPorts = [ 8472 ];
-          networking.firewall.trustedInterfaces = [ "flannel.1" ];
-          networking.useDHCP = false;
-          networking.defaultGateway = "192.168.1.1";
-          networking.interfaces.eth1.ipv4.addresses = pkgs.lib.mkForce [
-            {
-              address = "192.168.1.1";
-              prefixLength = 24;
-            }
-          ];
         };
 
       server2 =
-        { pkgs, ... }:
+        { nodes, pkgs, ... }:
         {
           environment.systemPackages = with pkgs; [
             gzip
@@ -119,16 +114,17 @@ import ../make-test-python.nix (
             enable = true;
             package = k3s;
             images = [ pauseImage ];
-            serverAddr = "https://192.168.1.1:6443";
+            serverAddr = "https://${nodes.server.networking.primaryIPAddress}:6443";
             clusterInit = false;
             extraFlags = [
               "--disable coredns"
               "--disable local-storage"
               "--disable metrics-server"
               "--disable servicelb"
               "--disable traefik"
-              "--node-ip 192.168.1.3"
               "--pause-image test.local/pause:local"
+              "--node-ip ${nodes.server2.networking.primaryIPAddress}"
+              "--flannel-iface eth1"
             ];
           };
           networking.firewall.allowedTCPPorts = [
@@ -137,19 +133,10 @@ import ../make-test-python.nix (
             6443
           ];
           networking.firewall.allowedUDPPorts = [ 8472 ];
-          networking.firewall.trustedInterfaces = [ "flannel.1" ];
-          networking.useDHCP = false;
-          networking.defaultGateway = "192.168.1.3";
-          networking.interfaces.eth1.ipv4.addresses = pkgs.lib.mkForce [
-            {
-              address = "192.168.1.3";
-              prefixLength = 24;
-            }
-          ];
         };
 
       agent =
-        { pkgs, ... }:
+        { nodes, pkgs, ... }:
         {
           virtualisation.memorySize = 1024;
           virtualisation.diskSize = 2048;
@@ -159,23 +146,15 @@ import ../make-test-python.nix (
             role = "agent";
             package = k3s;
             images = [ pauseImage ];
-            serverAddr = "https://192.168.1.3:6443";
+            serverAddr = "https://${nodes.server2.networking.primaryIPAddress}:6443";
             extraFlags = [
               "--pause-image test.local/pause:local"
-              "--node-ip 192.168.1.2"
+              "--node-ip ${nodes.agent.networking.primaryIPAddress}"
+              "--flannel-iface eth1"
             ];
           };
           networking.firewall.allowedTCPPorts = [ 6443 ];
           networking.firewall.allowedUDPPorts = [ 8472 ];
-          networking.firewall.trustedInterfaces = [ "flannel.1" ];
-          networking.useDHCP = false;
-          networking.defaultGateway = "192.168.1.2";
-          networking.interfaces.eth1.ipv4.addresses = pkgs.lib.mkForce [
-            {
-              address = "192.168.1.2";
-              prefixLength = 24;
-            }
-          ];
         };
     };