feat: no longer ship kubectl, instead alias k0s, use /var/lib/embedded-cluster (#378)

laverya · web-flow · commit f5f74fba1e82 · 2024-02-16T15:23:39.000-05:00
* no longer export kubeconfig, instead alias k0s

* fix shell command

* simple

* conf dir?

* undo go changes

* reinclude kubeconfig, do not ship kubectl

* use /usr/local/bin/k0s kubectl directly in scripts

* use symlink method

* update ln

* undo reset change

* ln only required after install

* fix shell alias

* move bins to /var/lib/embedded-cluster/bin

* panic for stack traces

* move call to defaults.BinaryName out of compile time constants

* remove unused config dir, lazy init bins dir

* remove the new bins dir on reset

* link k0s during install

* link kubectl -&gt; k0s on join

* remove alias from shell command

* reinclude bins dir in test

* move logs to /var/lib/embedded-cluster too

* use the helpful helper function

* add info to sudo test

* run version test command as non-root

* non-root handling

* better non-root handling

* do not include embedded cluster ID in metadata output

* ensure embeddedClusterID is not in metadata output

* ensure embeddedClusterID is not in metadata _values_ output

* don't include embeddedBinaryName in default metadata output either
diff --git a/.github/workflows/dependencies.yaml b/.github/workflows/dependencies.yaml
@@ -23,10 +23,6 @@ jobs:
       run: |
         export VERSION=`curl https://api.github.com/repos/openebs/charts/releases/latest | jq -r .name | tr -d openebs-`
         sed -i "/^OPENEBS_CHART_VERSION/c\OPENEBS_CHART_VERSION = $VERSION" Makefile
-    - name: Kubectl
-      run: |
-        export VERSION=`curl -L -s https://dl.k8s.io/release/stable.txt`
-        sed -i "/^KUBECTL_VERSION/c\KUBECTL_VERSION = $VERSION" Makefile
     - name: K0s
       run: |
         # Remove the '-ec.X' suffix and only update if the prefix (upstream k0s release) has changed.
diff --git a/Makefile b/Makefile
@@ -13,7 +13,6 @@ EMBEDDED_OPERATOR_CHART_VERSION = 0.22.7
 OPENEBS_CHART_URL = https://openebs.github.io/charts
 OPENEBS_CHART_NAME = openebs/openebs
 OPENEBS_CHART_VERSION = 3.10.0
-KUBECTL_VERSION = v1.29.2
 K0S_VERSION = v1.29.1+k0s.1
 K0S_BINARY_SOURCE_OVERRIDE =
 TROUBLESHOOT_VERSION = v0.83.0
@@ -45,12 +44,6 @@ pkg/goods/bins/k0s: Makefile
 	chmod +x pkg/goods/bins/k0s
 	touch pkg/goods/bins/k0s
 
-pkg/goods/bins/kubectl: Makefile
-	mkdir -p pkg/goods/bins
-	curl -L -o pkg/goods/bins/kubectl "https://dl.k8s.io/release/$(KUBECTL_VERSION)/bin/linux/amd64/kubectl"
-	chmod +x pkg/goods/bins/kubectl
-	touch pkg/goods/bins/kubectl
-
 pkg/goods/bins/kubectl-support_bundle: Makefile
 	mkdir -p pkg/goods/bins
 	mkdir -p output/tmp/support-bundle
@@ -82,7 +75,6 @@ embedded-release: embedded-cluster-linux-amd64 output/tmp/release.tar.gz output/
 .PHONY: static
 static: pkg/goods/bins/k0s \
 	pkg/goods/bins/kubectl-preflight \
-	pkg/goods/bins/kubectl \
 	pkg/goods/bins/kubectl-support_bundle
 	
 .PHONY: embedded-cluster-linux-amd64
diff --git a/cmd/embedded-cluster/install.go b/cmd/embedded-cluster/install.go
@@ -257,6 +257,15 @@ func runOutro(c *cli.Context) error {
 	return addons.NewApplier(opts...).Outro(c.Context)
 }
 
+// linkK0s is a helper function that links the k0s binary to the embedded cluster bins dir as 'kubectl'.
+func linkK0s() error {
+	if err := os.Symlink(defaults.K0sBinaryPath(), defaults.PathToEmbeddedClusterBinary("kubectl")); err != nil {
+		return fmt.Errorf("unable to create symlink: %w", err)
+	}
+
+	return nil
+}
+
 // installCommands executes the "install" command. This will ensure that a k0s.yaml file exists
 // and then run `k0s install` to apply the cluster. Once this is finished then a "kubeconfig"
 // file is created. Resulting kubeconfig is stored in the configuration dir.
@@ -339,6 +348,11 @@ var installCommand = &cli.Command{
 			metrics.ReportApplyFinished(c, err)
 			return err
 		}
+		logrus.Debugf("linking k0s binary")
+		if err := linkK0s(); err != nil {
+			metrics.ReportApplyFinished(c, err)
+			return err
+		}
 		metrics.ReportApplyFinished(c, nil)
 		return nil
 	},
diff --git a/cmd/embedded-cluster/join.go b/cmd/embedded-cluster/join.go
@@ -182,6 +182,12 @@ var joinCommand = &cli.Command{
 			return err
 		}
 
+		logrus.Debugf("linking k0s binary")
+		if err := linkK0s(); err != nil {
+			metrics.ReportJoinFailed(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID, err)
+			return err
+		}
+
 		metrics.ReportJoinSucceeded(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID)
 		logrus.Infof("Join finished")
 		return nil
diff --git a/e2e/scripts/check-installation-state.sh b/e2e/scripts/check-installation-state.sh
@@ -38,5 +38,5 @@ main() {
 
 export EMBEDDED_CLUSTER_METRICS_BASEURL="https://staging.replicated.app"
 export KUBECONFIG=/var/lib/k0s/pki/admin.conf
-export PATH=$PATH:/root/.config/embedded-cluster/bin
+export PATH=$PATH:/var/lib/embedded-cluster/bin
 main
diff --git a/e2e/scripts/check-nodes-removed.sh b/e2e/scripts/check-nodes-removed.sh
@@ -32,5 +32,5 @@ main() {
 
 export EMBEDDED_CLUSTER_METRICS_BASEURL="https://staging.replicated.app"
 export KUBECONFIG=/var/lib/k0s/pki/admin.conf
-export PATH=$PATH:/root/.config/embedded-cluster/bin
+export PATH=$PATH:/var/lib/embedded-cluster/bin
 main "$@"
diff --git a/e2e/scripts/check-postupgrade-state.sh b/e2e/scripts/check-postupgrade-state.sh
@@ -78,5 +78,5 @@ main() {
 
 export EMBEDDED_CLUSTER_METRICS_BASEURL="https://staging.replicated.app"
 export KUBECONFIG=/var/lib/k0s/pki/admin.conf
-export PATH=$PATH:/root/.config/embedded-cluster/bin
+export PATH=$PATH:/var/lib/embedded-cluster/bin
 main "$@"
diff --git a/e2e/scripts/default-install.sh b/e2e/scripts/default-install.sh
@@ -85,5 +85,5 @@ main() {
 
 export EMBEDDED_CLUSTER_METRICS_BASEURL="https://staging.replicated.app"
 export KUBECONFIG=/var/lib/k0s/pki/admin.conf
-export PATH=$PATH:/root/.config/embedded-cluster/bin
+export PATH=$PATH:/var/lib/embedded-cluster/bin
 main
diff --git a/e2e/scripts/embedded-preflight.sh b/e2e/scripts/embedded-preflight.sh
@@ -189,5 +189,5 @@ main() {
 
 export EMBEDDED_CLUSTER_METRICS_BASEURL="https://staging.replicated.app"
 export KUBECONFIG=/var/lib/k0s/pki/admin.conf
-export PATH=$PATH:/root/.config/embedded-cluster/bin
+export PATH=$PATH:/var/lib/embedded-cluster/bin
 main
diff --git a/e2e/scripts/single-node-install.sh b/e2e/scripts/single-node-install.sh
@@ -164,5 +164,5 @@ main() {
 
 export EMBEDDED_CLUSTER_METRICS_BASEURL="https://staging.replicated.app"
 export KUBECONFIG=/var/lib/k0s/pki/admin.conf
-export PATH=$PATH:/root/.config/embedded-cluster/bin
+export PATH=$PATH:/var/lib/embedded-cluster/bin
 main
diff --git a/e2e/scripts/unsupported-overrides.sh b/e2e/scripts/unsupported-overrides.sh
@@ -149,5 +149,5 @@ main() {
 export EMBEDDED_CLUSTER_METRICS_BASEURL="https://staging.replicated.app"
 export KUBECONFIG=/var/lib/k0s/pki/admin.conf
 export K0SCONFIG=/etc/k0s/k0s.yaml
-export PATH=$PATH:/root/.config/embedded-cluster/bin
+export PATH=$PATH:/var/lib/embedded-cluster/bin
 main
diff --git a/e2e/scripts/wait-for-ready-nodes.sh b/e2e/scripts/wait-for-ready-nodes.sh
@@ -23,5 +23,5 @@ main() {
 
 export EMBEDDED_CLUSTER_METRICS_BASEURL="https://staging.replicated.app"
 export KUBECONFIG=/var/lib/k0s/pki/admin.conf
-export PATH=$PATH:/root/.config/embedded-cluster/bin
+export PATH=$PATH:/var/lib/embedded-cluster/bin
 main "$@"
diff --git a/e2e/sudo_test.go b/e2e/sudo_test.go
@@ -3,6 +3,7 @@ package e2e
 import (
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/replicatedhq/embedded-cluster/e2e/cluster"
 )
@@ -17,6 +18,7 @@ func TestCommandsRequireSudo(t *testing.T) {
 		EmbeddedClusterPath: "../output/bin/embedded-cluster",
 	})
 	defer tc.Destroy()
+	t.Logf(`%s: running "embedded-cluster version" as regular user`, time.Now().Format(time.RFC3339))
 	command := []string{"embedded-cluster", "version"}
 	if _, _, err := RunRegularUserCommandOnNode(t, tc, 0, command); err != nil {
 		t.Errorf("expected no error running `version` as regular user, got %v", err)
@@ -27,6 +29,7 @@ func TestCommandsRequireSudo(t *testing.T) {
 		{"embedded-cluster", "shell"},
 		{"embedded-cluster", "install", "--no-prompt"},
 	} {
+		t.Logf("%s: running %q as regular user", time.Now().Format(time.RFC3339), strings.Join(cmd, "_"))
 		stdout, stderr, err := RunRegularUserCommandOnNode(t, tc, 0, cmd)
 		if err == nil {
 			t.Fatalf("expected error running `%v` as regular user, got none", cmd)
@@ -36,4 +39,5 @@ func TestCommandsRequireSudo(t *testing.T) {
 			t.Fatalf("invalid error found running `%v` as regular user", cmd)
 		}
 	}
+	t.Logf("%s: test complete", time.Now().Format(time.RFC3339))
 }
diff --git a/e2e/version_test.go b/e2e/version_test.go
@@ -16,13 +16,14 @@ func TestVersion(t *testing.T) {
 	tc := cluster.NewTestCluster(&cluster.Input{
 		T:                   t,
 		Nodes:               1,
+		CreateRegularUser:   true,
 		Image:               "ubuntu/jammy",
 		EmbeddedClusterPath: "../output/bin/embedded-cluster",
 	})
 	defer tc.Destroy()
 	t.Logf("%s: validating embedded-cluster version in node 0", time.Now().Format(time.RFC3339))
 	line := []string{"embedded-cluster", "version"}
-	stdout, stderr, err := RunCommandOnNode(t, tc, 0, line)
+	stdout, stderr, err := RunRegularUserCommandOnNode(t, tc, 0, line)
 	if err != nil {
 		t.Fatalf("fail to install ssh on node %s: %v", tc.Nodes[0], err)
 	}
@@ -42,7 +43,7 @@ func TestVersion(t *testing.T) {
 	}
 
 	line2 := []string{"embedded-cluster", "version", "metadata"}
-	stdout, stderr, err = RunCommandOnNode(t, tc, 0, line2)
+	stdout, stderr, err = RunRegularUserCommandOnNode(t, tc, 0, line2)
 	if err != nil {
 		t.Fatalf("fail to run metadata command on node %s: %v", tc.Nodes[0], err)
 	}
@@ -56,6 +57,17 @@ func TestVersion(t *testing.T) {
 		t.Fatalf("fail to parse metadata output: %v", err)
 	}
 
+	for _, foundChart := range parsed.Configs.Charts {
+		if strings.Contains(foundChart.Values, "embeddedClusterID") {
+			t.Errorf("metadata output for chart %s contains embeddedClusterID", foundChart.Name)
+			failed = true
+		}
+		if strings.Contains(foundChart.Values, "embeddedBinaryName") {
+			t.Errorf("metadata output for chart %s contains embeddedBinaryName", foundChart.Name)
+			failed = true
+		}
+	}
+
 	expectedCharts := []string{"openebs", "embedded-cluster-operator", "admin-console", "ingress-nginx"}
 	if len(parsed.Configs.Charts) != len(expectedCharts) {
 		t.Log(output)
diff --git a/pkg/addons/adminconsole/adminconsole.go b/pkg/addons/adminconsole/adminconsole.go
@@ -53,7 +53,6 @@ var helmValues = map[string]interface{}{
 		"enabled":  true,
 		"nodePort": DEFAULT_ADMIN_CONSOLE_NODE_PORT,
 	},
-	"embeddedClusterID":      metrics.ClusterID().String(),
 	"embeddedClusterVersion": defaults.Version,
 }
 
@@ -218,6 +217,7 @@ func (a *AdminConsole) GenerateHelmConfig(onlyDefaults bool) ([]v1beta1.Chart, [
 		if err := a.addKotsApplicationToHelmValues(); err != nil {
 			return nil, nil, fmt.Errorf("unable to add kots app to helm values: %w", err)
 		}
+		helmValues["embeddedClusterID"] = metrics.ClusterID().String()
 	}
 	if err := a.addLicenseAndVersionToHelmValues(); err != nil {
 		return nil, nil, fmt.Errorf("unable to add license to helm values: %w", err)
diff --git a/pkg/addons/embeddedclusteroperator/embeddedclusteroperator.go b/pkg/addons/embeddedclusteroperator/embeddedclusteroperator.go
@@ -38,8 +38,6 @@ var helmValues = map[string]interface{}{
 	"kotsVersion":               adminconsole.Version,
 	"embeddedClusterVersion":    defaults.Version,
 	"embeddedClusterK0sVersion": defaults.K0sVersion,
-	"embeddedBinaryName":        defaults.BinaryName(),
-	"embeddedClusterID":         metrics.ClusterID().String(),
 }
 
 // EmbeddedClusterOperator manages the installation of the embedded cluster operator
@@ -82,6 +80,12 @@ func (e *EmbeddedClusterOperator) GenerateHelmConfig(onlyDefaults bool) ([]v1bet
 		TargetNS:  "embedded-cluster",
 		Order:     2,
 	}
+
+	if !onlyDefaults {
+		helmValues["embeddedBinaryName"] = defaults.BinaryName()
+		helmValues["embeddedClusterID"] = metrics.ClusterID().String()
+	}
+
 	valuesStringData, err := yaml.Marshal(helmValues)
 	if err != nil {
 		return nil, nil, fmt.Errorf("unable to marshal helm values: %w", err)
diff --git a/pkg/defaults/defaults.go b/pkg/defaults/defaults.go
@@ -42,11 +42,6 @@ func EmbeddedClusterLogsSubDir() string {
 	return def().EmbeddedClusterLogsSubDir()
 }
 
-// EmbeddedClusterConfigSubDir calls ConfigSubDir on the default provider.
-func EmbeddedClusterConfigSubDir() string {
-	return def().EmbeddedClusterConfigSubDir()
-}
-
 // K0sBinaryPath calls K0sBinaryPath on the default provider.
 func K0sBinaryPath() string {
 	return def().K0sBinaryPath()
@@ -62,11 +57,6 @@ func PathToLog(name string) string {
 	return def().PathToLog(name)
 }
 
-// PathToConfig calls PathToConfig on the default provider.
-func PathToConfig(name string) string {
-	return def().PathToConfig(name)
-}
-
 func PathToKubeConfig() string {
 	return def().PathToKubeConfig()
 }
diff --git a/pkg/defaults/provider.go b/pkg/defaults/provider.go
@@ -7,7 +7,6 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
-	"runtime"
 	"time"
 
 	"github.com/gosimple/slug"
@@ -30,43 +29,7 @@ type Provider struct {
 }
 
 // Init makes sure all the necessary directory exists on the system.
-func (d *Provider) Init() {
-	if err := os.MkdirAll(d.EmbeddedClusterConfigSubDir(), 0755); err != nil {
-		logrus.Fatalf("unable to create config dir: %s", err)
-	}
-	if err := os.MkdirAll(d.EmbeddedClusterBinsSubDir(), 0755); err != nil {
-		logrus.Fatalf("unable to create embedded-cluster bin dir: %s", err)
-	}
-	if err := os.MkdirAll(d.EmbeddedClusterLogsSubDir(), 0755); err != nil {
-		panic(fmt.Errorf("unable to create embedded-cluster logs dir: %w", err))
-	}
-}
-
-// home returns the user's home dir.
-func (d *Provider) home() string {
-	home, err := os.UserHomeDir()
-	if err != nil {
-		logrus.Fatalf("unable to get user home dir: %s", err)
-	}
-	return home
-}
-
-// config returns the user's config dir.
-func (d *Provider) config() string {
-	home, err := os.UserHomeDir()
-	if err != nil {
-		logrus.Fatalf("unable to get user home dir: %s", err)
-	}
-	// use the XDG_CONFIG_HOME environment variable if set
-	if xdgConfigHome := os.Getenv("XDG_CONFIG_HOME"); xdgConfigHome != "" {
-		return xdgConfigHome
-	}
-	// otherwise, default to $HOME/.config on linux
-	if runtime.GOOS == "linux" {
-		return filepath.Join(home, ".config")
-	}
-	return home
-}
+func (d *Provider) Init() {}
 
 // BinaryName returns the binary name, this is useful for places where we
 // need to present the name of the binary to the user (the name may vary if
@@ -84,7 +47,12 @@ func (d *Provider) BinaryName() string {
 // EmbeddedClusterLogsSubDir returns the path to the directory where embedded-cluster logs
 // are stored.
 func (d *Provider) EmbeddedClusterLogsSubDir() string {
-	return filepath.Join(d.Base, d.config(), d.BinaryName(), "logs")
+	path := filepath.Join(d.EmbeddedClusterHomeDirectory(), "logs")
+	if err := os.MkdirAll(path, 0755); err != nil {
+		logrus.Fatalf("unable to create embedded-cluster logs dir: %s", err)
+	}
+
+	return path
 }
 
 // PathToLog returns the full path to a log file. This function does not check
@@ -96,19 +64,18 @@ func (d *Provider) PathToLog(name string) string {
 // EmbeddedClusterBinsSubDir returns the path to the directory where embedded-cluster binaries
 // are stored.
 func (d *Provider) EmbeddedClusterBinsSubDir() string {
-	return filepath.Join(d.Base, d.config(), d.BinaryName(), "bin")
-}
+	path := filepath.Join(d.EmbeddedClusterHomeDirectory(), "bin")
 
-// EmbeddedClusterConfigSubDir returns the path to the directory where configuration files are
-// stored.
-func (d *Provider) EmbeddedClusterConfigSubDir() string {
-	return filepath.Join(d.Base, d.config(), d.BinaryName(), "etc")
+	if err := os.MkdirAll(path, 0755); err != nil {
+		logrus.Fatalf("unable to create embedded-cluster bin dir: %s", err)
+	}
+	return path
 }
 
 // EmbeddedClusterHomeDirectory returns the parent directory. Inside this parent directory we
 // store all the embedded-cluster related files.
 func (d *Provider) EmbeddedClusterHomeDirectory() string {
-	return filepath.Join(d.Base, d.config(), d.BinaryName())
+	return filepath.Join(d.Base, "/var/lib/embedded-cluster")
 }
 
 // K0sBinaryPath returns the path to the k0s binary when it is installed on the node. This
@@ -125,12 +92,6 @@ func (d *Provider) PathToEmbeddedClusterBinary(name string) string {
 	return filepath.Join(d.EmbeddedClusterBinsSubDir(), name)
 }
 
-// PathToConfig returns the full path to a configuration file. This function
-// does not check if the file exists.
-func (d *Provider) PathToConfig(name string) string {
-	return filepath.Join(d.EmbeddedClusterConfigSubDir(), name)
-}
-
 func (d *Provider) PathToKubeConfig() string {
 	return "/var/lib/k0s/pki/admin.conf"
 }
diff --git a/pkg/defaults/provider_test.go b/pkg/defaults/provider_test.go
@@ -13,7 +13,7 @@ func TestInit(t *testing.T) {
 	assert.NoError(t, err)
 	defer os.RemoveAll(tmpdir)
 	def := NewProvider(tmpdir)
-	assert.DirExists(t, def.EmbeddedClusterConfigSubDir(), "config dir should exist")
+	assert.DirExists(t, def.EmbeddedClusterLogsSubDir(), "logs dir should exist")
 	assert.DirExists(t, def.EmbeddedClusterBinsSubDir(), "embedded-cluster binary dir should exist")
 }
 
@@ -35,7 +35,6 @@ func TestEnsureAllDirectoriesAreInsideBase(t *testing.T) {
 	for _, fn := range []func() string{
 		def.EmbeddedClusterBinsSubDir,
 		def.EmbeddedClusterLogsSubDir,
-		def.EmbeddedClusterConfigSubDir,
 	} {
 		assert.Contains(t, fn(), tmpdir, "directory should be inside base")
 		count := strings.Count(fn(), tmpdir)
diff --git a/pkg/logging/logging.go b/pkg/logging/logging.go
