use custom 'controller' role name if available (#184)

* use custom 'controller' role name if available

* reconcile overrides formats

* handle nil

* actually parse config from the binary

* ensure node has controller override role

* intentionally break test

* update code

* undo intentional breakage

* test both that the role was used and custom labels were applied

* remove unused struct

Author: laverya

cmd/embedded-cluster/install.go

@@ -16,9 +16,11 @@ import (
 	"github.com/k0sproject/rig"
 	"github.com/k0sproject/rig/log"
 	k0sversion "github.com/k0sproject/version"
+	embeddedclusterv1beta1 "github.com/replicatedhq/embedded-cluster-operator/api/v1beta1"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli/v2"
 	"gopkg.in/yaml.v2"
+	kyaml "sigs.k8s.io/yaml"
 
 	"github.com/replicatedhq/embedded-cluster/pkg/addons"
 	"github.com/replicatedhq/embedded-cluster/pkg/config"
@@ -160,7 +162,12 @@ func updateConfig(c *cli.Context) error {
 		if err != nil {
 			return fmt.Errorf("unable to read overrides file: %w", err)
 		}
-		config.ApplyEmbeddedUnsupportedOverrides(cfg, data)
+		var overrideCfg embeddedclusterv1beta1.Config
+		if err := kyaml.Unmarshal(data, &overrideCfg); err != nil {
+			return fmt.Errorf("unable to unmarshal overrides file: %w", err)
+		}
+
+		config.ApplyEmbeddedUnsupportedOverrides(cfg, overrideCfg.Spec.UnsupportedOverrides.K0s)
 	}
 
 	opts := []addons.Option{}
@@ -257,7 +264,12 @@ func ensureK0sctlConfig(c *cli.Context, useprompt bool) error {
 		if err != nil {
 			return fmt.Errorf("unable to read overrides file: %w", err)
 		}
-		config.ApplyEmbeddedUnsupportedOverrides(cfg, data)
+		var overrideCfg embeddedclusterv1beta1.Config
+		if err := kyaml.Unmarshal(data, &overrideCfg); err != nil {
+			return fmt.Errorf("unable to unmarshal overrides file: %w", err)
+		}
+
+		config.ApplyEmbeddedUnsupportedOverrides(cfg, overrideCfg.Spec.UnsupportedOverrides.K0s)
 	}
 
 	opts := []addons.Option{}

e2e/kots-release/config.yaml

@@ -0,0 +1,24 @@
+apiVersion: embeddedcluster.replicated.com/v1beta1
+kind: Config
+metadata:
+  name: "testconfig"
+spec:
+  controller:
+    labels:
+      controller-label: controller-label-value
+    name: controller-test
+  custom:
+    - labels:
+        abc-test-label: abc-test-label-value
+        abc-test-label-two: abc-test-label-value-2
+      name: abc
+    - labels:
+        xyz-test-label: xyz-value
+      name: xyz
+  unsupportedOverrides:
+    k0s: |
+      metadata:
+        name: foo
+      spec:
+        telemetry:
+          enabled: false

e2e/scripts/single-node-install.sh

@@ -14,6 +14,7 @@ wait_for_healthy_node() {
         ready=$(kubectl get nodes | grep -v NotReady | grep -c Ready || true)
         kubectl get nodes || true
     done
+
     return 0
 }
 
@@ -45,6 +46,18 @@ wait_for_pods_running() {
     done
 }
 
+ensure_node_config() {
+    if ! kubectl describe node | grep "controller-label" ; then
+        echo "Failed to find controller-label"
+        return 1
+    fi
+
+    if ! kubectl describe node | grep "controller-test" ; then
+        echo "Failed to find controller-test"
+        return 1
+    fi
+}
+
 main() {
     if ! embedded-cluster install --no-prompt 2>&1 | tee /tmp/log ; then
         cat /etc/os-release
@@ -59,6 +72,10 @@ main() {
         echo "Failed to install embedded-cluster"
         exit 1
     fi
+    if ! ensure_node_config; then
+        echo "Cluster did not respect node config"
+        exit 1
+    fi
     if ! wait_for_pods_running 900; then
         echo "Failed to install embedded-cluster"
         exit 1

pkg/addons/embeddedclusteroperator/embeddedclusteroperator.go

@@ -13,7 +13,6 @@ import (
 	"gopkg.in/yaml.v2"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
-	kyaml "sigs.k8s.io/yaml"
 
 	"github.com/replicatedhq/embedded-cluster/pkg/addons/adminconsole"
 	"github.com/replicatedhq/embedded-cluster/pkg/customization"
@@ -76,22 +75,6 @@ func (e *EmbeddedClusterOperator) GenerateHelmConfig() ([]v1beta1.Chart, []v1bet
 	return []v1beta1.Chart{chartConfig}, nil, nil
 }
 
-// readEmbeddedClusterConfig reads and unmarshal the Config object from the embedded cluster
-// that has been embedded into this binary through a release.
-func (e *EmbeddedClusterOperator) readEmbeddedClusterConfig() (*embeddedclusterv1beta1.Config, error) {
-	rawcfg, err := customization.AdminConsole{}.EmbeddedClusterConfig()
-	if err != nil {
-		return nil, fmt.Errorf("unable to read embeddec cluster config: %w", err)
-	} else if rawcfg == nil {
-		return nil, nil
-	}
-	var cfg embeddedclusterv1beta1.Config
-	if err := kyaml.Unmarshal(rawcfg, &cfg); err != nil {
-		return nil, fmt.Errorf("unable to unmarshal embedded cluster config: %w", err)
-	}
-	return &cfg, nil
-}
-
 // Outro is executed after the cluster deployment.
 func (e *EmbeddedClusterOperator) Outro(ctx context.Context, cli client.Client) error {
 	loading := pb.Start()
@@ -101,7 +84,7 @@ func (e *EmbeddedClusterOperator) Outro(ctx context.Context, cli client.Client)
 		return err
 	}
 	loading.Closef("Embedded Cluster Operator is ready!")
-	cfg, err := e.readEmbeddedClusterConfig()
+	cfg, err := customization.AdminConsole{}.EmbeddedClusterConfig()
 	if err != nil {
 		return err
 	}

pkg/config/config.go

@@ -19,6 +19,7 @@ import (
 	"github.com/k0sproject/k0sctl/pkg/apis/k0sctl.k0sproject.io/v1beta1"
 	"github.com/k0sproject/k0sctl/pkg/apis/k0sctl.k0sproject.io/v1beta1/cluster"
 	k0sversion "github.com/k0sproject/version"
+	embeddedclusterv1beta1 "github.com/replicatedhq/embedded-cluster-operator/api/v1beta1"
 	"github.com/sirupsen/logrus"
 	yamlv2 "gopkg.in/yaml.v2"
 	"sigs.k8s.io/yaml"
@@ -51,23 +52,25 @@ func ReadConfigFile(cfgPath string) (*v1beta1.Cluster, error) {
 
 // RenderClusterConfig renders a cluster configuration interactively.
 func RenderClusterConfig(ctx context.Context, multi bool) (*v1beta1.Cluster, error) {
-	embconfig, err := customization.AdminConsole{}.EmbeddedClusterConfig()
+	clusterConfig, err := customization.AdminConsole{}.EmbeddedClusterConfig()
 	if err != nil {
 		return nil, fmt.Errorf("unable to get embedded cluster config: %w", err)
+	} else if clusterConfig == nil {
+		clusterConfig = &embeddedclusterv1beta1.Config{}
 	}
 	if multi {
 		cfg, err := renderMultiNodeConfig(ctx)
 		if err != nil {
 			return nil, fmt.Errorf("unable to render multi-node config: %w", err)
 		}
-		ApplyEmbeddedUnsupportedOverrides(cfg, embconfig)
+		ApplyEmbeddedUnsupportedOverrides(cfg, clusterConfig.Spec.UnsupportedOverrides.K0s)
 		return cfg, nil
 	}
 	cfg, err := renderSingleNodeConfig(ctx)
 	if err != nil {
 		return nil, fmt.Errorf("unable to render single-node config: %w", err)
 	}
-	ApplyEmbeddedUnsupportedOverrides(cfg, embconfig)
+	ApplyEmbeddedUnsupportedOverrides(cfg, clusterConfig.Spec.UnsupportedOverrides.K0s)
 	return renderSingleNodeConfig(ctx)
 }
 
@@ -251,16 +254,18 @@ func renderSingleNodeConfig(ctx context.Context) (*v1beta1.Cluster, error) {
 	if err != nil {
 		return nil, fmt.Errorf("unable to get preferred node IP address: %w", err)
 	}
+
+	labels := additionalControllerLabels()
+	labels["kots.io/embedded-cluster-role-0"] = getControllerRoleName()
+	labels["kots.io/embedded-cluster-role"] = "total-1"
+
 	host := &hostcfg{
 		Address: ipaddr,
 		Role:    "controller+worker",
 		Port:    22,
 		User:    usr.Username,
 		KeyPath: defaults.SSHKeyPath(),
-		Labels: map[string]string{
-			"kots.io/embedded-cluster-role-0": "controller",
-			"kots.io/embedded-cluster-role":   "total-1",
-		},
+		Labels:  labels,
 	}
 	if err := host.testConnection(); err != nil {
 		return nil, fmt.Errorf("unable to connect to %s: %w", ipaddr, err)
@@ -309,32 +314,11 @@ func UpdateHelmConfigs(cfg *v1beta1.Cluster, opts ...addons.Option) error {
 	return nil
 }
 
-// UnsupportedConfigOverrides is a auxiliary struct for parsing the unsupported overrides
-// as provided in the Kots release. XXX This should eventually become a CRD.
-type UnsupportedConfigOverrides struct {
-	Spec struct {
-		UnsupportedOverrides struct {
-			K0s *cluster.K0s `yaml:"k0s"`
-		} `yaml:"unsupportedOverrides"`
-	} `yaml:"spec"`
-}
-
 // ApplyEmbeddedUnsupportedOverrides applies the custom configuration to the cluster config.
-func ApplyEmbeddedUnsupportedOverrides(config *v1beta1.Cluster, embconfig []byte) error {
-	if embconfig == nil {
+func ApplyEmbeddedUnsupportedOverrides(config *v1beta1.Cluster, embconfig string) error {
+	if embconfig == "" {
 		return nil
 	}
-	var overrides UnsupportedConfigOverrides
-	if err := yaml.Unmarshal(embconfig, &overrides); err != nil {
-		return fmt.Errorf("unable to parse cluster config overrides: %w", err)
-	}
-	if overrides.Spec.UnsupportedOverrides.K0s == nil {
-		return nil
-	}
-	origConfigBytes, err := yaml.Marshal(overrides.Spec.UnsupportedOverrides.K0s.Config)
-	if err != nil {
-		return fmt.Errorf("unable to marshal cluster config overrides: %w", err)
-	}
 	newConfigBytes, err := yaml.Marshal(config.Spec.K0s.Config)
 	if err != nil {
 		return fmt.Errorf("unable to marshal original cluster config: %w", err)
@@ -343,7 +327,7 @@ func ApplyEmbeddedUnsupportedOverrides(config *v1beta1.Cluster, embconfig []byte
 	if err != nil {
 		return fmt.Errorf("unable to convert cluster config overrides to json: %w", err)
 	}
-	target, err := fmtconvert.YAMLToJSON(origConfigBytes)
+	target, err := fmtconvert.YAMLToJSON([]byte(embconfig))
 	if err != nil {
 		return fmt.Errorf("unable to convert original cluster config to json: %w", err)
 	}
@@ -362,3 +346,30 @@ func ApplyEmbeddedUnsupportedOverrides(config *v1beta1.Cluster, embconfig []byte
 	config.Spec.K0s.Config = newK0sConfig
 	return nil
 }
+
+func getControllerRoleName() string {
+	clusterConfig, err := customization.AdminConsole{}.EmbeddedClusterConfig()
+
+	controllerRoleName := "controller"
+	if err == nil {
+		if clusterConfig != nil {
+			if clusterConfig.Spec.Controller.Name != "" {
+				controllerRoleName = clusterConfig.Spec.Controller.Name
+			}
+		}
+	}
+	return controllerRoleName
+}
+
+func additionalControllerLabels() map[string]string {
+	clusterConfig, err := customization.AdminConsole{}.EmbeddedClusterConfig()
+
+	if err == nil {
+		if clusterConfig != nil {
+			if clusterConfig.Spec.Controller.Labels != nil {
+				return clusterConfig.Spec.Controller.Labels
+			}
+		}
+	}
+	return map[string]string{}
+}

pkg/config/config_test.go

@@ -7,8 +7,11 @@ import (
 	"testing"
 
 	"github.com/k0sproject/k0sctl/pkg/apis/k0sctl.k0sproject.io/v1beta1"
+	embeddedclusterv1beta1 "github.com/replicatedhq/embedded-cluster-operator/api/v1beta1"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 	"gopkg.in/yaml.v2"
+	kyaml "sigs.k8s.io/yaml"
 )
 
 //go:embed testdata/*yaml
@@ -39,16 +42,21 @@ func TestApplyUnsupportedOverrides(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.Name, func(t *testing.T) {
+			req := require.New(t)
+
 			var config v1beta1.Cluster
 			err := yaml.Unmarshal([]byte(tt.Config), &config)
-			assert.NoError(t, err)
-			err = ApplyEmbeddedUnsupportedOverrides(&config, []byte(tt.Override))
-			assert.NoError(t, err)
+			req.NoError(err)
+			var cfg embeddedclusterv1beta1.Config
+			err = kyaml.Unmarshal([]byte(tt.Override), &cfg)
+			req.NoError(err)
+			err = ApplyEmbeddedUnsupportedOverrides(&config, cfg.Spec.UnsupportedOverrides.K0s)
+			req.NoError(err)
 			result, err := yaml.Marshal(config)
-			assert.NoError(t, err)
+			req.NoError(err)
 			resultString := strings.TrimSpace(string(result))
 			expectedString := strings.TrimSpace(string(tt.Expected))
-			assert.Equal(t, resultString, expectedString)
+			req.Equal(expectedString, resultString)
 		})
 	}
 }

pkg/config/testdata/override-change-name.yaml

@@ -32,10 +32,9 @@ override: |-
   kind: Config
   spec:
     unsupportedOverrides:
-      k0s:
-        config:
-          metadata:
-            name: foo
+      k0s: |
+        metadata:
+          name: foo
 expected: |-
   apiVersion: k0sctl.k0sproject.io/v1beta1
   kind: Cluster

pkg/config/testdata/override-enable-telemetry.yaml

@@ -32,11 +32,10 @@ override: |-
   kind: Config
   spec:
     unsupportedOverrides:
-      k0s:
-        config:
-          spec:
-            telemetry:
-              enabled: true
+      k0s: |
+        spec:
+          telemetry:
+            enabled: true
 expected: |-
   apiVersion: k0sctl.k0sproject.io/v1beta1
   kind: Cluster

pkg/config/testdata/override-setting-ip-forward.yaml

@@ -32,14 +32,13 @@ override: |-
   kind: Config
   spec:
     unsupportedOverrides:
-      k0s:
-        config:
-          spec:
-            workerProfiles:
-            - name: ip-forward
-              values:
-                allowedUnsafeSysctls:
-                  - net.ipv4.ip_forward
+      k0s: |
+        spec:
+          workerProfiles:
+          - name: ip-forward
+            values:
+              allowedUnsafeSysctls:
+                - net.ipv4.ip_forward
 expected: |-
   apiVersion: k0sctl.k0sproject.io/v1beta1
   kind: Cluster

pkg/config/testdata/override-zero-out-sans-list.yaml

@@ -36,11 +36,10 @@ override: |-
   kind: Config
   spec:
     unsupportedOverrides:
-      k0s:
-        config:
-          spec:
-            api:
-              sans: null
+      k0s: |
+        spec:
+          api:
+            sans: null
 expected: |-
   apiVersion: k0sctl.k0sproject.io/v1beta1
   kind: Cluster