Merge remote-tracking branch 'origin/1.14.1' into k0s-1-28

Author: sgalsaleh

.github/workflows/image-deps-updater.yaml

@@ -57,7 +57,6 @@ jobs:
           - k0s
           - openebs
           - velero
-          - embeddedclusteroperator
           - seaweedfs
     steps:
     - name: Checkout

.github/workflows/update-addons.yaml

@@ -46,7 +46,6 @@ jobs:
           - registry
           - seaweedfs
           - velero
-          - embeddedclusteroperator
           - adminconsole
     steps:
       - name: Check out repo

README.md

@@ -226,7 +226,23 @@ make ssh-node0
     make build run
     ```
 
-    Subsequent changes to the kotsadm web component are reflected in real-time, no manual steps required.
+    Subsequent changes to the kotsadm web component are reflected in real-time; no manual steps are required.
+    However, to add, remove, or upgrade a dependency / package:
+
+    * From a new terminal session, exec into the kotsadm-web container:
+        ```bash
+        make kotsadm-web-up-ec
+        ```
+
+    * Run the desired `yarn` commands. For example:
+        ```bash
+        yarn add <package>
+        ```
+
+    * When finished, exit the container:
+        ```bash
+        exit
+        ```
 
 1. To apply additional API changes, stop the current process with Ctrl+C, then run the following command:
     ```bash

cmd/embedded-cluster/join.go

@@ -144,6 +144,11 @@ var joinCommand = &cli.Command{
 			Usage:  "Enable high availability.",
 			Hidden: true,
 		},
+		&cli.StringFlag{
+			Name:  "network-interface",
+			Usage: "The network interface to use for the cluster",
+			Value: "",
+		},
 		&cli.BoolFlag{
 			Name:  "no-prompt",
 			Usage: "Disable interactive prompts.",
@@ -181,13 +186,6 @@ var joinCommand = &cli.Command{
 			return fmt.Errorf("usage: %s join <url> <token>", binName)
 		}
 
-		logrus.Debugf("getting network interface from join command")
-		jcmdAddress := strings.Split(c.Args().Get(0), ":")[0]
-		networkInterface, err := netutils.InterfaceNameForAddress(jcmdAddress)
-		if err != nil {
-			return fmt.Errorf("unable to get network interface for address %s: %w", jcmdAddress, err)
-		}
-
 		logrus.Debugf("fetching join token remotely")
 		jcmd, err := getJoinToken(c.Context, c.Args().Get(0), c.Args().Get(1))
 		if err != nil {
@@ -200,7 +198,7 @@ var joinCommand = &cli.Command{
 		}
 
 		setProxyEnv(jcmd.InstallationSpec.Proxy)
-		proxyOK, localIP, err := checkProxyConfigForLocalIP(jcmd.InstallationSpec.Proxy, networkInterface)
+		proxyOK, localIP, err := checkProxyConfigForLocalIP(jcmd.InstallationSpec.Proxy, c.String("network-interface"))
 		if err != nil {
 			return fmt.Errorf("failed to check proxy config for local IP: %w", err)
 		}
@@ -279,7 +277,7 @@ var joinCommand = &cli.Command{
 		}
 
 		logrus.Debugf("overriding network configuration")
-		if err := applyNetworkConfiguration(jcmd, networkInterface); err != nil {
+		if err := applyNetworkConfiguration(c, jcmd); err != nil {
 			err := fmt.Errorf("unable to apply network configuration: %w", err)
 			metrics.ReportJoinFailed(c.Context, jcmd.InstallationSpec.MetricsBaseURL, jcmd.ClusterID, err)
 		}
@@ -292,7 +290,7 @@ var joinCommand = &cli.Command{
 		}
 
 		logrus.Debugf("joining node to cluster")
-		if err := runK0sInstallCommand(jcmd.K0sJoinCommand, networkInterface); err != nil {
+		if err := runK0sInstallCommand(c, jcmd.K0sJoinCommand); err != nil {
 			err := fmt.Errorf("unable to join node to cluster: %w", err)
 			metrics.ReportJoinFailed(c.Context, jcmd.InstallationSpec.MetricsBaseURL, jcmd.ClusterID, err)
 			return err
@@ -340,10 +338,10 @@ var joinCommand = &cli.Command{
 	},
 }
 
-func applyNetworkConfiguration(jcmd *JoinCommandResponse, networkInterface string) error {
+func applyNetworkConfiguration(c *cli.Context, jcmd *JoinCommandResponse) error {
 	if jcmd.InstallationSpec.Network != nil {
 		clusterSpec := config.RenderK0sConfig()
-		address, err := netutils.FirstValidAddress(networkInterface)
+		address, err := netutils.FirstValidAddress(c.String("network-interface"))
 		if err != nil {
 			return fmt.Errorf("unable to find first valid address: %w", err)
 		}
@@ -488,14 +486,14 @@ func systemdUnitFileName() string {
 
 // runK0sInstallCommand runs the k0s install command as provided by the kots
 // adm api.
-func runK0sInstallCommand(fullcmd string, networkInterface string) error {
+func runK0sInstallCommand(c *cli.Context, fullcmd string) error {
 	args := strings.Split(fullcmd, " ")
 	args = append(args, "--token-file", "/etc/k0s/join-token")
 	if strings.Contains(fullcmd, "controller") {
 		args = append(args, "--disable-components", "konnectivity-server", "--enable-dynamic-config")
 	}
 
-	nodeIP, err := netutils.FirstValidAddress(networkInterface)
+	nodeIP, err := netutils.FirstValidAddress(c.String("network-interface"))
 	if err != nil {
 		return fmt.Errorf("unable to find first valid address: %w", err)
 	}

cmd/embedded-cluster/preflights.go

@@ -3,10 +3,8 @@ package main
 import (
 	"fmt"
 	"os"
-	"strings"
 
 	"github.com/replicatedhq/embedded-cluster/pkg/defaults"
-	"github.com/replicatedhq/embedded-cluster/pkg/netutils"
 	"github.com/replicatedhq/embedded-cluster/pkg/versions"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli/v2"
@@ -97,6 +95,11 @@ var joinRunPreflightsCommand = &cli.Command{
 			Usage:  "Path to the air gap bundle. If set, the installation will complete without internet access.",
 			Hidden: true,
 		},
+		&cli.StringFlag{
+			Name:  "network-interface",
+			Usage: "The network interface to use for the cluster",
+			Value: "",
+		},
 		&cli.BoolFlag{
 			Name:  "no-prompt",
 			Usage: "Disable interactive prompts.",
@@ -114,13 +117,6 @@ var joinRunPreflightsCommand = &cli.Command{
 			return fmt.Errorf("usage: %s join preflights <url> <token>", binName)
 		}
 
-		logrus.Debugf("getting network interface from join command")
-		jcmdAddress := strings.Split(c.Args().Get(0), ":")[0]
-		networkInterface, err := netutils.InterfaceNameForAddress(jcmdAddress)
-		if err != nil {
-			return fmt.Errorf("unable to get network interface for address %s: %w", jcmdAddress, err)
-		}
-
 		logrus.Debugf("fetching join token remotely")
 		jcmd, err := getJoinToken(c.Context, c.Args().Get(0), c.Args().Get(1))
 		if err != nil {
@@ -133,7 +129,7 @@ var joinRunPreflightsCommand = &cli.Command{
 		}
 
 		setProxyEnv(jcmd.InstallationSpec.Proxy)
-		proxyOK, localIP, err := checkProxyConfigForLocalIP(jcmd.InstallationSpec.Proxy, networkInterface)
+		proxyOK, localIP, err := checkProxyConfigForLocalIP(jcmd.InstallationSpec.Proxy, c.String("network-interface"))
 		if err != nil {
 			return fmt.Errorf("failed to check proxy config for local IP: %w", err)
 		}

cmd/embedded-cluster/reset.go

@@ -365,11 +365,6 @@ var resetCommand = &cli.Command{
 			Usage: "Disable interactive prompts",
 			Value: false,
 		},
-		&cli.BoolFlag{
-			Name:  "reboot",
-			Usage: "Reboot system after resetting the node",
-			Value: false,
-		},
 	},
 	Usage: fmt.Sprintf("Remove %s from the current node", binName),
 	Action: func(c *cli.Context) error {
@@ -378,7 +373,7 @@ var resetCommand = &cli.Command{
 		}
 
 		logrus.Info("This will remove this node from the cluster and completely reset it, removing all data stored on the node.")
-		logrus.Info("Do not reset another node until this is complete.")
+		logrus.Info("This action will cause the node to reboot. Do not reset another node until this is complete.")
 		if !c.Bool("force") && !c.Bool("no-prompt") && !prompts.New().Confirm("Do you want to continue?", false) {
 			return fmt.Errorf("Aborting")
 		}
@@ -449,10 +444,6 @@ var resetCommand = &cli.Command{
 			return err
 		}
 
-		if !c.Bool("reboot") {
-			logrus.Infof("Node has been reset. Please reboot to ensure transient configuration is also reset.")
-		}
-
 		if err := helpers.RemoveAll(defaults.PathToK0sConfig()); err != nil {
 			return fmt.Errorf("failed to remove k0s config: %w", err)
 		}
@@ -501,10 +492,8 @@ var resetCommand = &cli.Command{
 			return fmt.Errorf("failed to remove k0s binary: %w", err)
 		}
 
-		if c.Bool("reboot") {
-			if _, err := exec.Command("reboot").Output(); err != nil {
-				return err
-			}
+		if _, err := exec.Command("reboot").Output(); err != nil {
+			return err
 		}
 
 		return nil

e2e/cluster/cluster.go

@@ -359,22 +359,20 @@ func ConfigureProxy(in *Input) {
 	// them trust it.
 	for i := 0; i < in.Nodes; i++ {
 		name := fmt.Sprintf("node-%s-%02d", in.id, i)
-		for _, cmd := range [][]string{
-			{"ip", "route", "del", "default"},
-			{"ip", "route", "add", "default", "via", "10.0.0.254"},
-			{"mkdir", "-p", "/usr/local/share/ca-certificates/proxy"},
-		} {
-			RunCommandOnNode(in, cmd, name)
-		}
+		RunCommandOnNode(in, []string{"mkdir", "-p", "/usr/local/share/ca-certificates/proxy"}, name)
 
 		CopyFileToNode(in, name, File{
 			SourcePath: "/tmp/ca.crt",
 			DestPath:   "/usr/local/share/ca-certificates/proxy/ca.crt",
 			Mode:       0644,
 		})
 
-		cmd := []string{"update-ca-certificates"}
-		RunCommandOnNode(in, cmd, name)
+		for _, cmd := range [][]string{
+			{"update-ca-certificates"},
+			{"/usr/local/bin/default-route-through-proxy.sh"},
+		} {
+			RunCommandOnNode(in, cmd, name)
+		}
 	}
 }
 

e2e/install_test.go

@@ -624,6 +624,9 @@ func TestResetAndReinstall(t *testing.T) {
 		t.Fatalf("fail to reset the installation: %v", err)
 	}
 
+	t.Logf("%s: waiting for nodes to reboot", time.Now().Format(time.RFC3339))
+	time.Sleep(30 * time.Second)
+
 	t.Logf("%s: installing embedded-cluster on node 0 after reset", time.Now().Format(time.RFC3339))
 	line = []string{"single-node-install.sh", "ui"}
 	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
@@ -698,6 +701,9 @@ func TestResetAndReinstallAirgap(t *testing.T) {
 		t.Fatalf("fail to reset the installation: %v", err)
 	}
 
+	t.Logf("%s: waiting for nodes to reboot", time.Now().Format(time.RFC3339))
+	time.Sleep(30 * time.Second)
+
 	t.Logf("%s: installing embedded-cluster on node 0", time.Now().Format(time.RFC3339))
 	line = []string{"single-node-airgap-install.sh"}
 	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {

e2e/preflights_test.go

@@ -29,12 +29,24 @@ func TestPreflights(t *testing.T) {
 	})
 
 	_, stderr, err := container.Exec(cli,
-		"apt-get update && apt-get install -y apt-utils kmod",
+		"apt-get update && apt-get install -y apt-utils kmod netcat-traditional",
 	)
 	if err != nil {
 		t.Fatalf("failed to install deps: err=%v, stderr=%s", err, stderr)
 	}
 
+	if _, stderr, err = container.Exec(cli, "nohup netcat -l -p 10250 &"); err != nil {
+		t.Fatalf("failed to start netcat: err=%v, stderr=%s", err, stderr)
+	}
+
+	if _, stderr, err = container.Exec(cli, "nohup netcat -l 127.0.0.1 -p 50000 &"); err != nil {
+		t.Fatalf("failed to start netcat: err=%v, stderr=%s", err, stderr)
+	}
+
+	if _, stderr, err = container.Exec(cli, "nohup netcat -l -u -p 4789 &"); err != nil {
+		t.Fatalf("failed to start netcat: err=%v, stderr=%s", err, stderr)
+	}
+
 	runCmd := fmt.Sprintf("%s install run-preflights --no-prompt", container.GetECBinaryPath())
 	if os.Getenv("LICENSE_PATH") != "" {
 		runCmd = fmt.Sprintf("%s --license %s", runCmd, container.GetLicensePath())
@@ -93,10 +105,13 @@ func TestPreflights(t *testing.T) {
 			assert: func(t *testing.T, results *preflights.Output) {
 				expected := map[string]bool{
 					// TODO: work to remove these
-					"System Clock":                true,
-					"'devices' Cgroup Controller": true,
-					"API Access":                  true,
-					"Proxy Registry Access":       true,
+					"System Clock":                            true,
+					"'devices' Cgroup Controller":             true,
+					"API Access":                              true,
+					"Proxy Registry Access":                   true,
+					"Kubelet Port Availability":               true,
+					"Calico Communication Port Availability":  true,
+					"Local Artifact Mirror Port Availability": true,
 					// as long as fio ran successfully, we're good
 					"Filesystem Write Latency": true,
 				}
@@ -124,6 +139,26 @@ func TestPreflights(t *testing.T) {
 				}
 			},
 		},
+		{
+			name: "Should contain port failures",
+			assert: func(t *testing.T, results *preflights.Output) {
+				expected := map[string]bool{
+					"Kubelet Port Availability":               false,
+					"Calico Communication Port Availability":  false,
+					"Local Artifact Mirror Port Availability": false,
+				}
+				for _, res := range results.Fail {
+					if _, ok := expected[res.Title]; ok {
+						expected[res.Title] = true
+					}
+				}
+				for title, found := range expected {
+					if !found {
+						t.Errorf("expected port failure not found: %q", title)
+					}
+				}
+			},
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

e2e/restore_test.go

@@ -65,6 +65,9 @@ func TestSingleNodeDisasterRecovery(t *testing.T) {
 		t.Fatalf("fail to reset the installation: %v", err)
 	}
 
+	t.Logf("%s: waiting for nodes to reboot", time.Now().Format(time.RFC3339))
+	time.Sleep(30 * time.Second)
+
 	t.Logf("%s: restoring the installation", time.Now().Format(time.RFC3339))
 	line = append([]string{"restore-installation.exp"}, testArgs...)
 	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
@@ -156,6 +159,9 @@ func TestSingleNodeDisasterRecoveryWithProxy(t *testing.T) {
 		t.Fatalf("fail to reset the installation: %v", err)
 	}
 
+	t.Logf("%s: waiting for nodes to reboot", time.Now().Format(time.RFC3339))
+	time.Sleep(30 * time.Second)
+
 	t.Logf("%s: restoring the installation", time.Now().Format(time.RFC3339))
 	line = append([]string{"restore-installation.exp"}, testArgs...)
 	line = append(line, "--http-proxy", cluster.HTTPProxy)
@@ -229,6 +235,9 @@ func TestSingleNodeResumeDisasterRecovery(t *testing.T) {
 		t.Fatalf("fail to reset the installation: %v", err)
 	}
 
+	t.Logf("%s: waiting for nodes to reboot", time.Now().Format(time.RFC3339))
+	time.Sleep(30 * time.Second)
+
 	t.Logf("%s: restoring the installation", time.Now().Format(time.RFC3339))
 	line = append([]string{"resume-restore.exp"}, testArgs...)
 	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
@@ -329,6 +338,10 @@ func TestSingleNodeAirgapDisasterRecovery(t *testing.T) {
 	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
 		t.Fatalf("fail to reset the installation: %v", err)
 	}
+
+	t.Logf("%s: waiting for nodes to reboot", time.Now().Format(time.RFC3339))
+	time.Sleep(30 * time.Second)
+
 	installTestDependenciesDebian(t, tc, 0, true)
 	t.Logf("%s: restoring the installation", time.Now().Format(time.RFC3339))
 	testArgs = append(testArgs, "--pod-cidr", "10.128.0.0/20", "--service-cidr", "10.129.0.0/20")
@@ -463,7 +476,7 @@ func TestMultiNodeHADisasterRecovery(t *testing.T) {
 	}
 
 	// reset the cluster
-	line = []string{"reset-installation.sh", "--force", "--reboot"}
+	line = []string{"reset-installation.sh", "--force"}
 	t.Logf("%s: resetting the installation on node 2", time.Now().Format(time.RFC3339))
 	if _, _, err := RunCommandOnNode(t, tc, 2, line); err != nil {
 		t.Fatalf("fail to reset the installation: %v", err)
@@ -681,7 +694,7 @@ func TestMultiNodeAirgapHADisasterRecovery(t *testing.T) {
 	}
 
 	// reset the cluster
-	line = []string{"reset-installation.sh", "--force", "--reboot"}
+	line = []string{"reset-installation.sh", "--force"}
 	t.Logf("%s: resetting the installation on node 2", time.Now().Format(time.RFC3339))
 	if _, _, err := RunCommandOnNode(t, tc, 2, line); err != nil {
 		t.Fatalf("fail to reset the installation: %v", err)