feat: allow additional controller nodes to run workloads (#411)

* ensure goldpinger is running on all nodes

* fix version test

* fix repositories

* count goldpinger pods, not running goldpinger pods

* test nightly kotsadm image

* append --no-taints in join

* ?

Author: laverya

cmd/embedded-cluster/join.go

@@ -325,6 +325,10 @@ func runK0sInstallCommand(fullcmd string) error {
 	args = append(args, "--token-file", "/etc/k0s/join-token")
 	if strings.Contains(fullcmd, "controller") {
 		args = append(args, "--disable-components", "konnectivity-server", "--enable-dynamic-config")
+
+		if !strings.Contains(fullcmd, "--no-taints") {
+			args = append(args, "--no-taints")
+		}
 	}
 	if _, err := runCommand(args[0], args[1:]...); err != nil {
 		return err

e2e/kots-release-install/cluster-config.yaml

@@ -33,6 +33,8 @@ spec:
       repositories:
         - name: ingress-nginx
           url: https://kubernetes.github.io/ingress-nginx
+        - name: okgolove
+          url: https://okgolove.github.io/helm-charts/
       charts:
         - name: ingress-nginx
           chartname: ingress-nginx/ingress-nginx
@@ -45,3 +47,8 @@ spec:
                 nodePorts:
                   http: "80"
                   https: "443"
+        - chartname: okgolove/goldpinger
+          name: goldpinger
+          namespace: goldpinger
+          version: 6.1.2
+          order: 11

e2e/kots-release-upgrade/cluster-config.yaml

@@ -57,3 +57,9 @@ spec:
           namespace: goldpinger
           version: 6.1.2
           order: 11
+        - chartname: oci://registry-1.docker.io/bitnamicharts/memcached
+          name: memcached-vendor-chart
+          namespace: memcached
+          values: "abc: xyz"
+          order: 4
+          version: 6.6.4

e2e/scripts/check-postupgrade-state.sh

@@ -52,8 +52,8 @@ main() {
     echo "installations"
     kubectl get installations
 
-    # ensure that goldpinger exists
-    kubectl get ns goldpinger
+    # ensure that memcached exists
+    kubectl get ns memcached
 
     # ensure that new app pods exist
     kubectl get pods -n kotsadm -l app=second

e2e/scripts/single-node-install.sh

@@ -109,8 +109,8 @@ check_openebs_storage_class() {
 }
 
 ensure_app_not_upgraded() {
-    if kubectl get ns | grep -q goldpinger ; then
-        echo "found goldpinger ns"
+    if kubectl get ns | grep -q memcached ; then
+        echo "found memcached ns"
         return 1
     fi
     if kubectl get pods -n kotsadm -l app=second | grep -q second ; then

e2e/scripts/wait-for-ready-nodes.sh

@@ -18,6 +18,16 @@ main() {
         kubectl get nodes || true
     done
     echo "All nodes are ready"
+
+    echo "checking that goldpinger has run on all nodes"
+    kubectl get pods -n goldpinger
+    local goldpinger_running_count=
+    goldpinger_running_count=$(kubectl get pods --no-headers -n goldpinger | wc -l)
+    if [ "$goldpinger_running_count" -lt "$expected_nodes" ]; then
+        echo "goldpinger is running on $goldpinger_running_count nodes, expected $expected_nodes"
+        exit 1
+    fi
+
     exit 0
 }
 

e2e/version_test.go

@@ -69,7 +69,7 @@ func TestVersion(t *testing.T) {
 		}
 	}
 
-	expectedCharts := []string{"openebs", "embedded-cluster-operator", "admin-console", "ingress-nginx"}
+	expectedCharts := []string{"openebs", "embedded-cluster-operator", "admin-console", "ingress-nginx", "goldpinger"}
 	if len(parsed.Configs.Charts) != len(expectedCharts) {
 		t.Log(output)
 		t.Fatalf("found %d charts in metadata, expected %d", len(parsed.Configs.Charts), len(expectedCharts))