feat: apply config overrides during node join (#243)

starts to apply patches to both k0s' config `api` and `storage`
properties. patches are received from the kots api and applied
in the following order: embedded overrides -> user provided.

this commit also adds unit tests for the new functionality and
to the old pathing functions.

this is a overview of the changes present on this commit:

- added error treatment in case of failure on override apply.
- removed some empty lines.
- added two new fields to the join token response struct.
- created function to apply overrides in place (/etc/k0s/k0s.yaml).
- add unit tests for multiple override apply functions.
- create an abstraction to make reading embed data easier.

Author: ricardomaraschini

cmd/embedded-cluster/install.go

@@ -145,38 +145,36 @@ func createK0sctlConfigBackup(ctx context.Context) error {
 // updateConfig updates the k0sctl.yaml file with the latest configuration
 // options.
 func updateConfig(c *cli.Context) error {
-
 	if err := createK0sctlConfigBackup(c.Context); err != nil {
 		return fmt.Errorf("unable to create config backup: %w", err)
 	}
-
 	cfgpath := defaults.PathToConfig("k0sctl.yaml")
 	cfg, err := config.ReadConfigFile(cfgpath)
 	if err != nil {
 		return fmt.Errorf("unable to read cluster config: %w", err)
 	}
 	cfg.Spec.K0s.Version = k0sversion.MustParse(defaults.K0sVersion)
-
 	if c.String("overrides") != "" {
 		eucfg, err := parseEndUserConfig(c.String("overrides"))
 		if err != nil {
 			return fmt.Errorf("unable to process overrides file: %w", err)
 		}
-		config.ApplyEmbeddedUnsupportedOverrides(cfg, eucfg.Spec.UnsupportedOverrides.K0s)
+		if err := config.ApplyEmbeddedUnsupportedOverrides(
+			cfg, eucfg.Spec.UnsupportedOverrides.K0s,
+		); err != nil {
+			return fmt.Errorf("unable to apply overrides: %w", err)
+		}
 	}
-
 	opts := []addons.Option{}
 	if c.Bool("no-prompt") {
 		opts = append(opts, addons.WithoutPrompt())
 	}
 	for _, addon := range c.StringSlice("disable-addon") {
 		opts = append(opts, addons.WithoutAddon(addon))
 	}
-
 	if err := config.UpdateHelmConfigs(cfg, opts...); err != nil {
 		return fmt.Errorf("unable to update helm configs: %w", err)
 	}
-
 	fp, err := os.OpenFile(cfgpath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
 	if err != nil {
 		return fmt.Errorf("unable to create config file: %w", err)
@@ -247,33 +245,31 @@ func ensureK0sctlConfig(c *cli.Context, useprompt bool) error {
 	} else if !os.IsNotExist(err) {
 		return fmt.Errorf("unable to open config: %w", err)
 	}
-
 	cfg, err := config.RenderClusterConfig(c.Context, multi)
 	if err != nil {
 		return fmt.Errorf("unable to render config: %w", err)
 	}
-
 	if c.String("overrides") != "" {
 		eucfg, err := parseEndUserConfig(c.String("overrides"))
 		if err != nil {
 			return fmt.Errorf("unable to process overrides file: %w", err)
 		}
-		config.ApplyEmbeddedUnsupportedOverrides(cfg, eucfg.Spec.UnsupportedOverrides.K0s)
+		if err := config.ApplyEmbeddedUnsupportedOverrides(
+			cfg, eucfg.Spec.UnsupportedOverrides.K0s,
+		); err != nil {
+			return fmt.Errorf("unable to apply overrides: %w", err)
+		}
 	}
-
 	opts := []addons.Option{}
 	if c.Bool("no-prompt") {
 		opts = append(opts, addons.WithoutPrompt())
 	}
-
 	for _, addon := range c.StringSlice("disable-addon") {
 		opts = append(opts, addons.WithoutAddon(addon))
 	}
-
 	if err := config.UpdateHelmConfigs(cfg, opts...); err != nil {
 		return fmt.Errorf("unable to update helm configs: %w", err)
 	}
-
 	fp, err := os.OpenFile(cfgpath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600)
 	if err != nil {
 		return fmt.Errorf("unable to create config file: %w", err)

cmd/embedded-cluster/join.go

@@ -14,10 +14,14 @@ import (
 	"time"
 
 	"github.com/google/uuid"
+	"github.com/k0sproject/dig"
+	"github.com/k0sproject/k0sctl/pkg/apis/k0sctl.k0sproject.io/v1beta1/cluster"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli/v2"
+	"gopkg.in/yaml.v2"
 
 	"github.com/replicatedhq/embedded-cluster/pkg/addons"
+	"github.com/replicatedhq/embedded-cluster/pkg/config"
 	"github.com/replicatedhq/embedded-cluster/pkg/defaults"
 	"github.com/replicatedhq/embedded-cluster/pkg/goods"
 	"github.com/replicatedhq/embedded-cluster/pkg/metrics"
@@ -26,12 +30,43 @@ import (
 	"github.com/replicatedhq/embedded-cluster/pkg/prompts"
 )
 
-// JoinCommandResponse is the response from the kots api we use to fetch the k0s join
-// token. It returns the actual command we need to run and also the cluster ID.
+// JoinCommandResponse is the response from the kots api we use to fetch the k0s join token.
 type JoinCommandResponse struct {
-	K0sJoinCommand string    `json:"k0sJoinCommand"`
-	K0sToken       string    `json:"k0sToken"`
-	ClusterID      uuid.UUID `json:"clusterID"`
+	K0sJoinCommand            string    `json:"k0sJoinCommand"`
+	K0sToken                  string    `json:"k0sToken"`
+	ClusterID                 uuid.UUID `json:"clusterID"`
+	K0sUnsupportedOverrides   string    `json:"k0sUnsupportedOverrides"`
+	EndUserK0sConfigOverrides string    `json:"endUserK0sConfigOverrides"`
+}
+
+// extractK0sConfigOverridePatch parses the provided override and returns a dig.Mapping that
+// can be then applied on top a k0s configuration file to set both `api` and `storage` spec
+// fields. All other fields in the override are ignored.
+func (j JoinCommandResponse) extractK0sConfigOverridePatch(data []byte) (dig.Mapping, error) {
+	config := dig.Mapping{}
+	if err := yaml.Unmarshal(data, &config); err != nil {
+		return nil, fmt.Errorf("unable to unmarshal embedded config: %w", err)
+	}
+	result := dig.Mapping{}
+	if api := config.DigMapping("config", "spec", "api"); len(api) > 0 {
+		result.DigMapping("config", "spec")["api"] = api
+	}
+	if storage := config.DigMapping("config", "spec", "storage"); len(storage) > 0 {
+		result.DigMapping("config", "spec")["storage"] = storage
+	}
+	return result, nil
+}
+
+// EndUserOverrides returns a dig.Mapping that can be applied on top of a k0s configuration.
+// This patch is assembled based on the EndUserK0sConfigOverrides field.
+func (j JoinCommandResponse) EndUserOverrides() (dig.Mapping, error) {
+	return j.extractK0sConfigOverridePatch([]byte(j.EndUserK0sConfigOverrides))
+}
+
+// EmbeddedOverrides returns a dig.Mapping that can be applied on top of a k0s configuration.
+// This patch is assembled based on the K0sUnsupportedOverrides field.
+func (j JoinCommandResponse) EmbeddedOverrides() (dig.Mapping, error) {
+	return j.extractK0sConfigOverridePatch([]byte(j.K0sUnsupportedOverrides))
 }
 
 // getJoinToken issues a request to the kots api to get the actual join command
@@ -108,6 +143,12 @@ var joinCommand = &cli.Command{
 			metrics.ReportJoinFailed(c.Context, jcmd.ClusterID, err)
 			return err
 		}
+		loading.Infof("Applying configuration overrides")
+		if err := applyJoinConfigurationOverrides(c, jcmd); err != nil {
+			err := fmt.Errorf("unable to apply configuration overrides: %w", err)
+			metrics.ReportJoinFailed(c.Context, jcmd.ClusterID, err)
+			return err
+		}
 		loading.Infof("Creating systemd unit file")
 		if err := createSystemdUnitFile(jcmd.K0sJoinCommand); err != nil {
 			err := fmt.Errorf("unable to create systemd unit file: %w", err)
@@ -133,6 +174,75 @@ var joinCommand = &cli.Command{
 	},
 }
 
+// applyJoinConfigurationOverrides applies both config overrides received from the kots api.
+// Applies first the EmbeddedOverrides and then the EndUserOverrides.
+func applyJoinConfigurationOverrides(c *cli.Context, jcmd *JoinCommandResponse) error {
+	patch, err := jcmd.EmbeddedOverrides()
+	if err != nil {
+		return fmt.Errorf("unable to get embedded overrides: %w", err)
+	}
+	if len(patch) > 0 {
+		if data, err := yaml.Marshal(patch); err != nil {
+			return fmt.Errorf("unable to marshal embedded overrides: %w", err)
+		} else if err := patchK0sConfig("/etc/k0s/k0s.yaml", string(data)); err != nil {
+			return fmt.Errorf("unable to patch config with embedded data: %w", err)
+		}
+	}
+	if patch, err = jcmd.EndUserOverrides(); err != nil {
+		return fmt.Errorf("unable to get embedded overrides: %w", err)
+	} else if len(patch) == 0 {
+		return nil
+	}
+	if data, err := yaml.Marshal(patch); err != nil {
+		return fmt.Errorf("unable to marshal embedded overrides: %w", err)
+	} else if err := patchK0sConfig("/etc/k0s/k0s.yaml", string(data)); err != nil {
+		return fmt.Errorf("unable to patch config with embedded data: %w", err)
+	}
+	return nil
+}
+
+// patchK0sConfig patches the created k0s config with the unsupported overrides passed in.
+func patchK0sConfig(path string, patch string) error {
+	if len(patch) == 0 {
+		return nil
+	}
+	finalcfg := dig.Mapping{
+		"apiVersion": "k0s.k0sproject.io/v1beta1",
+		"kind":       "ClusterConfig",
+		"metadata":   dig.Mapping{"name": defaults.BinaryName()},
+	}
+	if _, err := os.Stat(path); err == nil {
+		data, err := os.ReadFile(path)
+		if err != nil {
+			return fmt.Errorf("unable to read node config: %w", err)
+		}
+		finalcfg = dig.Mapping{}
+		if err := yaml.Unmarshal(data, &finalcfg); err != nil {
+			return fmt.Errorf("unable to unmarshal node config: %w", err)
+		}
+	}
+	k0sconfig := cluster.K0s{Config: finalcfg.Dup()}
+	result, err := config.PatchK0sConfig(&k0sconfig, patch)
+	if err != nil {
+		return fmt.Errorf("unable to patch node config: %w", err)
+	}
+	if len(result.Config.DigMapping("spec", "api")) > 0 {
+		finalcfg.DigMapping("spec")["api"] = result.Config.DigMapping("spec", "api")
+	}
+	if len(result.Config.DigMapping("spec", "storage")) > 0 {
+		finalcfg.DigMapping("spec")["storage"] = result.Config.DigMapping("spec", "storage")
+	}
+	out, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600)
+	if err != nil {
+		return fmt.Errorf("unable to open node config file for writing: %w", err)
+	}
+	defer out.Close()
+	if err := yaml.NewEncoder(out).Encode(finalcfg); err != nil {
+		return fmt.Errorf("unable to write node config: %w", err)
+	}
+	return nil
+}
+
 // saveTokenToDisk saves the provided token in "/etc/k0s/join-token".
 func saveTokenToDisk(token string) error {
 	if err := os.MkdirAll("/etc/k0s", 0755); err != nil {

cmd/embedded-cluster/join_test.go

@@ -0,0 +1,135 @@
+package main
+
+import (
+	"embed"
+	"fmt"
+	"os"
+	"path"
+	"strings"
+	"testing"
+
+	"github.com/k0sproject/dig"
+	embeddedclusterv1beta1 "github.com/replicatedhq/embedded-cluster-operator/api/v1beta1"
+	"github.com/replicatedhq/embedded-cluster/pkg/customization"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"gopkg.in/yaml.v3"
+	k8syaml "sigs.k8s.io/yaml"
+)
+
+//go:embed testdata/*
+var testData embed.FS
+
+func Test_patchK0sConfig(t *testing.T) {
+	type test struct {
+		Name     string
+		Original string `yaml:"original"`
+		Override string `yaml:"override"`
+		Expected string `yaml:"expected"`
+	}
+	entries, err := testData.ReadDir("testdata/patch-k0s-config")
+	assert.NoError(t, err)
+	var tests []test
+	for _, entry := range entries {
+		if strings.HasPrefix(entry.Name(), "skip.") {
+			continue
+		}
+		fpath := path.Join("testdata", "patch-k0s-config", entry.Name())
+		data, err := testData.ReadFile(fpath)
+		assert.NoError(t, err)
+		var onetest test
+		err = yaml.Unmarshal(data, &onetest)
+		assert.NoError(t, err)
+		onetest.Name = fpath
+		tests = append(tests, onetest)
+	}
+	for _, tt := range tests {
+		t.Run(tt.Name, func(t *testing.T) {
+			req := require.New(t)
+
+			originalFile, err := os.CreateTemp("", "k0s-original-*.yaml")
+			req.NoError(err, "unable to create temp file")
+			defer func() {
+				originalFile.Close()
+				os.Remove(originalFile.Name())
+			}()
+			err = os.WriteFile(originalFile.Name(), []byte(tt.Original), 0644)
+			req.NoError(err, "unable to write original config")
+
+			var patch string
+			if tt.Override != "" {
+				var overrides embeddedclusterv1beta1.Config
+				err = k8syaml.Unmarshal([]byte(tt.Override), &overrides)
+				req.NoError(err, "unable to unmarshal override")
+				patch = overrides.Spec.UnsupportedOverrides.K0s
+			}
+
+			err = patchK0sConfig(originalFile.Name(), patch)
+			req.NoError(err, "unable to patch config")
+
+			var original dig.Mapping
+			err = yaml.NewDecoder(originalFile).Decode(&original)
+			req.NoError(err, "unable to decode original file")
+
+			var expected dig.Mapping
+			err = yaml.Unmarshal([]byte(tt.Expected), &expected)
+			req.NoError(err, "unable to unmarshal expected file")
+
+			assert.Equal(t, expected, original)
+			customization.DefaultProvider = &customization.AdminConsole{}
+		})
+	}
+}
+
+func TestJoinCommandResponseOverrides(t *testing.T) {
+	type test struct {
+		Name                      string
+		EmbeddedOverrides         string `yaml:"embeddedOverrides"`
+		EndUserOverrides          string `yaml:"endUserOverrides"`
+		ExpectedEmbeddedOverrides string `yaml:"expectedEmbeddedOverrides"`
+		ExpectedUserOverrides     string `yaml:"expectedUserOverrides"`
+	}
+	entries, err := testData.ReadDir("testdata/join-command-response")
+	assert.NoError(t, err)
+	var tests []test
+	for _, entry := range entries {
+		if strings.HasPrefix(entry.Name(), "skip.") {
+			continue
+		}
+		fpath := path.Join("testdata", "join-command-response", entry.Name())
+		data, err := testData.ReadFile(fpath)
+		assert.NoError(t, err)
+		var onetest test
+		err = yaml.Unmarshal(data, &onetest)
+		assert.NoError(t, err)
+		onetest.Name = fpath
+		tests = append(tests, onetest)
+	}
+	for _, tt := range tests {
+		t.Run(tt.Name, func(t *testing.T) {
+			req := require.New(t)
+			join := JoinCommandResponse{
+				K0sUnsupportedOverrides:   tt.EmbeddedOverrides,
+				EndUserK0sConfigOverrides: tt.EndUserOverrides,
+			}
+
+			embedded, err := join.EmbeddedOverrides()
+			req.NoError(err, "unable to patch config")
+			expectedEmbedded := dig.Mapping{}
+			err = yaml.Unmarshal([]byte(tt.ExpectedEmbeddedOverrides), &expectedEmbedded)
+			req.NoError(err, "unable to unmarshal expected file")
+			embeddedStr := fmt.Sprintf("%+v", embedded)
+			expectedEmbeddedStr := fmt.Sprintf("%+v", expectedEmbedded)
+			assert.Equal(t, expectedEmbeddedStr, embeddedStr)
+
+			user, err := join.EndUserOverrides()
+			req.NoError(err, "unable to patch config")
+			expectedUser := dig.Mapping{}
+			err = yaml.Unmarshal([]byte(tt.ExpectedUserOverrides), &expectedUser)
+			req.NoError(err, "unable to unmarshal expected file")
+			userStr := fmt.Sprintf("%+v", user)
+			expectedUserStr := fmt.Sprintf("%+v", expectedUser)
+			assert.Equal(t, expectedUserStr, userStr)
+		})
+	}
+}

cmd/embedded-cluster/testdata/join-command-response/empty-overrides.yaml

@@ -0,0 +1,4 @@
+endUserOverrides: ""
+embeddedOverrides: ""
+expectedUserOverrides: ""
+expectedEmbeddedOverrides: ""