feat: introduces local artifact server and materialize subcommand (#443)

* feat: introduces local artifact server and materialise

- implement and install a localhost http server for local assets.
- implement a hidden 'materialise' command to materialise the embedded
  assets into a directory.

* chore: add e2e tests

* chore: allow binary to be overwritten

* feat: starts local artifact mirror on join

* feat: block materialise for regular users

* chore: add path to help message

* chore: standardize on 'materialized'

* chore: materialise directly to /var/lib/embedded-cluster

* chore: adding some more tests

* chore: add host collector/analyser

Author: ricardomaraschini

.github/workflows/pull-request.yaml

@@ -134,6 +134,8 @@ jobs:
           - TestResetAndReinstall
           - TestCollectSupportBundle
           - TestOldVersionUpgrade
+          - TestMaterialize
+          - TestLocalArtifactMirror
     steps:
     - name: Checkout
       uses: actions/checkout@v4

.github/workflows/release-dev.yaml

@@ -112,6 +112,8 @@ jobs:
           - TestResetAndReinstall
           - TestCollectSupportBundle
           - TestOldVersionUpgrade
+          - TestMaterialize
+          - TestLocalArtifactMirror
     steps:
       - name: Checkout
         uses: actions/checkout@v4

Makefile

@@ -69,6 +69,10 @@ pkg/goods/bins/kubectl-preflight: Makefile
 	mv output/tmp/preflight/preflight pkg/goods/bins/kubectl-preflight
 	touch pkg/goods/bins/kubectl-preflight
 
+pkg/goods/bins/local-artifact-mirror: Makefile
+	mkdir -p pkg/goods/bins
+	CGO_ENABLED=0 go build -o pkg/goods/bins/local-artifact-mirror ./cmd/local-artifact-mirror
+
 output/tmp/release.tar.gz: e2e/kots-release-install/*
 	mkdir -p output/tmp
 	tar -czf output/tmp/release.tar.gz -C e2e/kots-release-install .
@@ -89,7 +93,8 @@ go.mod: Makefile
 static: pkg/goods/bins/k0s \
 	pkg/goods/bins/kubectl-preflight \
 	pkg/goods/bins/kubectl \
-	pkg/goods/bins/kubectl-support_bundle
+	pkg/goods/bins/kubectl-support_bundle \
+	pkg/goods/bins/local-artifact-mirror
 
 .PHONY: embedded-cluster-linux-amd64
 embedded-cluster-linux-amd64: static go.mod

cmd/embedded-cluster/install.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"bytes"
-	"context"
 	"fmt"
 	"os"
 	"os/exec"
@@ -52,6 +51,30 @@ func runCommand(bin string, args ...string) (string, error) {
 	return stdout.String(), nil
 }
 
+// installAndEnableLocalArtifactMirror installs and enables the local artifact mirror. This
+// service is reponsible for serving on localhost, through http, all files that are used
+// during a cluster upgrade.
+func installAndEnableLocalArtifactMirror() error {
+	ourbin := defaults.PathToEmbeddedClusterBinary("local-artifact-mirror")
+	hstbin := defaults.LocalArtifactMirrorPath()
+	if err := os.Rename(ourbin, hstbin); err != nil {
+		return fmt.Errorf("unable to move local artifact mirror binary: %w", err)
+	}
+	if err := goods.MaterializeLocalArtifactMirrorUnitFile(); err != nil {
+		return fmt.Errorf("failed to materialize artifact mirror unit: %w", err)
+	}
+	if _, err := runCommand("systemctl", "daemon-reload"); err != nil {
+		return fmt.Errorf("unable to get reload systemctl daemon: %w", err)
+	}
+	if _, err := runCommand("systemctl", "start", "local-artifact-mirror"); err != nil {
+		return fmt.Errorf("unable to start the local artifact mirror: %w", err)
+	}
+	if _, err := runCommand("systemctl", "enable", "local-artifact-mirror"); err != nil {
+		return fmt.Errorf("unable to start the local artifact mirror: %w", err)
+	}
+	return nil
+}
+
 // runPostInstall is a helper function that run things just after the k0s install
 // command ran.
 func runPostInstall() error {
@@ -63,7 +86,7 @@ func runPostInstall() error {
 	if _, err := runCommand("systemctl", "daemon-reload"); err != nil {
 		return fmt.Errorf("unable to get reload systemctl daemon: %w", err)
 	}
-	return nil
+	return installAndEnableLocalArtifactMirror()
 }
 
 // runHostPreflights run the host preflights we found embedded in the binary
@@ -232,7 +255,7 @@ func applyUnsupportedOverrides(c *cli.Context, cfg *k0sconfig.ClusterConfig) (*k
 
 // installK0s runs the k0s install command and waits for it to finish. If no configuration
 // is found one is generated.
-func installK0s(c *cli.Context) error {
+func installK0s() error {
 	ourbin := defaults.PathToEmbeddedClusterBinary("k0s")
 	hstbin := defaults.K0sBinaryPath()
 	if err := os.Rename(ourbin, hstbin); err != nil {
@@ -249,7 +272,7 @@ func installK0s(c *cli.Context) error {
 
 // waitForK0s waits for the k0s API to be available. We wait for the k0s socket to
 // appear in the system and until the k0s status command to finish.
-func waitForK0s(ctx context.Context) error {
+func waitForK0s() error {
 	loading := spinner.Start()
 	defer loading.Close()
 	loading.Infof("Waiting for %s node to be ready", defaults.BinaryName())
@@ -361,7 +384,7 @@ var installCommand = &cli.Command{
 			return err
 		}
 		logrus.Debugf("installing k0s")
-		if err := installK0s(c); err != nil {
+		if err := installK0s(); err != nil {
 			err := fmt.Errorf("unable update cluster: %w", err)
 			metrics.ReportApplyFinished(c, err)
 			return err
@@ -373,7 +396,7 @@ var installCommand = &cli.Command{
 			return err
 		}
 		logrus.Debugf("waiting for k0s to be ready")
-		if err := waitForK0s(c.Context); err != nil {
+		if err := waitForK0s(); err != nil {
 			err := fmt.Errorf("unable to wait for node: %w", err)
 			metrics.ReportApplyFinished(c, err)
 			return err

cmd/embedded-cluster/join.go

@@ -161,15 +161,15 @@ var joinCommand = &cli.Command{
 		}
 
 		logrus.Infof("Applying configuration overrides")
-		if err := applyJoinConfigurationOverrides(c, jcmd); err != nil {
+		if err := applyJoinConfigurationOverrides(jcmd); err != nil {
 			err := fmt.Errorf("unable to apply configuration overrides: %w", err)
 			metrics.ReportJoinFailed(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID, err)
 			return err
 		}
 
-		logrus.Infof("Creating systemd unit file")
-		if err := createSystemdUnitFile(jcmd.K0sJoinCommand); err != nil {
-			err := fmt.Errorf("unable to create systemd unit file: %w", err)
+		logrus.Infof("Creating systemd unit files")
+		if err := createSystemdUnitFiles(jcmd.K0sJoinCommand); err != nil {
+			err := fmt.Errorf("unable to create systemd unit files: %w", err)
 			metrics.ReportJoinFailed(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID, err)
 			return err
 		}
@@ -189,7 +189,7 @@ var joinCommand = &cli.Command{
 
 // applyJoinConfigurationOverrides applies both config overrides received from the kots api.
 // Applies first the EmbeddedOverrides and then the EndUserOverrides.
-func applyJoinConfigurationOverrides(c *cli.Context, jcmd *JoinCommandResponse) error {
+func applyJoinConfigurationOverrides(jcmd *JoinCommandResponse) error {
 	patch, err := jcmd.EmbeddedOverrides()
 	if err != nil {
 		return fmt.Errorf("unable to get embedded overrides: %w", err)
@@ -297,8 +297,9 @@ func startK0sService() error {
 	return nil
 }
 
-// createSystemdUnitFile links the k0s systemd unit file.
-func createSystemdUnitFile(fullcmd string) error {
+// createSystemdUnitFiles links the k0s systemd unit file. this also creates a new
+// systemd unit file for the local artifact mirror service.
+func createSystemdUnitFiles(fullcmd string) error {
 	dst := fmt.Sprintf("/etc/systemd/system/%s.service", defaults.BinaryName())
 	if _, err := os.Stat(dst); err == nil {
 		if err := os.Remove(dst); err != nil {
@@ -315,7 +316,7 @@ func createSystemdUnitFile(fullcmd string) error {
 	if _, err := runCommand("systemctl", "daemon-reload"); err != nil {
 		return err
 	}
-	return nil
+	return installAndEnableLocalArtifactMirror()
 }
 
 // runK0sInstallCommand runs the k0s install command as provided by the kots

cmd/embedded-cluster/main.go

@@ -33,6 +33,7 @@ func main() {
 			versionCommand,
 			joinCommand,
 			resetCommand,
+			materializeCommand,
 		},
 	}
 	if err := app.RunContext(ctx, os.Args); err != nil {

cmd/embedded-cluster/materialize.go

@@ -0,0 +1,28 @@
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/urfave/cli/v2"
+
+	"github.com/replicatedhq/embedded-cluster/pkg/goods"
+)
+
+var materializeCommand = &cli.Command{
+	Name:   "materialize",
+	Usage:  "Materialize embedded assets on /var/lib/embedded-cluster",
+	Hidden: true,
+	Before: func(c *cli.Context) error {
+		if os.Getuid() != 0 {
+			return fmt.Errorf("materialize command must be run as root")
+		}
+		return nil
+	},
+	Action: func(c *cli.Context) error {
+		if err := goods.Materialize(); err != nil {
+			return fmt.Errorf("unable to materialize: %v", err)
+		}
+		return nil
+	},
+}

cmd/embedded-cluster/uninstall.go

@@ -385,6 +385,19 @@ var resetCommand = &cli.Command{
 			}
 		}
 
+		lamPath := "/etc/systemd/system/local-artifact-mirror.service"
+		if _, err := os.Stat(lamPath); err == nil {
+			if _, err := runCommand("systemctl", "stop", "local-artifact-mirror"); err != nil {
+				return err
+			}
+			if err := os.RemoveAll(lamPath); err != nil {
+				return err
+			}
+			if err := os.RemoveAll(defaults.LocalArtifactMirrorPath()); err != nil {
+				return err
+			}
+		}
+
 		if _, err := os.Stat(defaults.EmbeddedClusterHomeDirectory()); err == nil {
 			if err := os.RemoveAll(defaults.EmbeddedClusterHomeDirectory()); err != nil {
 				return err

cmd/local-artifact-mirror/main.go

@@ -0,0 +1,94 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"os"
+	"os/signal"
+	"path"
+	"strings"
+	"syscall"
+	"time"
+
+	"github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+
+	"github.com/replicatedhq/embedded-cluster/pkg/defaults"
+)
+
+func main() {
+	ctx, cancel := signal.NotifyContext(
+		context.Background(),
+		syscall.SIGINT,
+		syscall.SIGTERM,
+	)
+	defer cancel()
+	name := path.Base(os.Args[0])
+	var app = &cli.App{
+		Name:     name,
+		Usage:    "Runs the local artifact mirror",
+		Commands: []*cli.Command{serveCommand},
+	}
+	if err := app.RunContext(ctx, os.Args); err != nil {
+		logrus.Fatal(err)
+	}
+}
+
+// logAndFilterRequest is a middleware that logs the HTTP request details. Returns 404
+// if attempting to read the log files as those are not served by this server.
+func logAndFilterRequest(handler http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		fmt.Printf("%s %s %s\n", r.RemoteAddr, r.Method, r.URL)
+		if strings.HasPrefix(r.URL.Path, "/logs") {
+			w.WriteHeader(http.StatusNotFound)
+			return
+		}
+		handler.ServeHTTP(w, r)
+	})
+}
+
+// serveCommand starts a http server that serves files from the /var/lib/embedded-cluster
+// directory. This servers listen only on localhost and is used to serve files needed by
+// the autopilot during an upgrade.
+var serveCommand = &cli.Command{
+	Name:  "serve",
+	Usage: "Serve /var/lib/embedded-cluster files over HTTP",
+	Before: func(c *cli.Context) error {
+		if os.Getuid() != 0 {
+			return fmt.Errorf("serve command must be run as root")
+		}
+		return nil
+	},
+	Action: func(c *cli.Context) error {
+		dir := defaults.EmbeddedClusterHomeDirectory()
+
+		fileServer := http.FileServer(http.Dir(dir))
+		loggedFileServer := logAndFilterRequest(fileServer)
+		http.Handle("/", loggedFileServer)
+
+		stop := make(chan os.Signal, 1)
+		signal.Notify(stop, os.Interrupt, syscall.SIGTERM)
+
+		server := &http.Server{Addr: "127.0.0.1:50000"}
+		go func() {
+			fmt.Println("Starting server on 127.0.0.1:50000")
+			if err := server.ListenAndServe(); err != nil {
+				if err != http.ErrServerClosed {
+					panic(err)
+				}
+			}
+		}()
+
+		<-stop
+		fmt.Println("Shutting down server...")
+
+		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+		defer cancel()
+		if err := server.Shutdown(ctx); err != nil {
+			panic(err)
+		}
+		fmt.Println("Server gracefully stopped")
+		return nil
+	},
+}

e2e/local-artifact-mirror_test.go

@@ -0,0 +1,63 @@
+package e2e
+
+import (
+	"testing"
+	"time"
+
+	"github.com/replicatedhq/embedded-cluster/e2e/cluster"
+)
+
+func TestLocalArtifactMirror(t *testing.T) {
+	t.Parallel()
+	tc := cluster.NewTestCluster(&cluster.Input{
+		T:                   t,
+		Nodes:               1,
+		Image:               "ubuntu/jammy",
+		EmbeddedClusterPath: "../output/bin/embedded-cluster-original",
+	})
+	defer tc.Destroy()
+
+	t.Logf("%s: installing embedded-cluster on node 0", time.Now().Format(time.RFC3339))
+	line := []string{"default-install.sh"}
+	stdout, stderr, err := RunCommandOnNode(t, tc, 0, line)
+	if err != nil {
+		t.Log("stdout:", stdout)
+		t.Log("stderr:", stderr)
+		t.Fatalf("fail to install embedded-cluster on node %s: %v", tc.Nodes[0], err)
+	}
+
+	commands := [][]string{
+		{"apt-get", "install", "curl", "-y"},
+		{"systemctl", "status", "local-artifact-mirror"},
+		{"systemctl", "stop", "local-artifact-mirror"},
+		{"systemctl", "start", "local-artifact-mirror"},
+		{"systemctl", "status", "local-artifact-mirror"},
+		{"curl", "-o", "/tmp/kubectl-test", "127.0.0.1:50000/bin/kubectl"},
+		{"chmod", "755", "/tmp/kubectl-test"},
+		{"/tmp/kubectl-test", "version", "--client"},
+	}
+	if err := RunCommandsOnNode(t, tc, 0, commands); err != nil {
+		t.Fatalf("fail testing local artifact mirror: %v", err)
+	}
+
+	command := []string{"cp", "/etc/passwd", "/var/lib/embedded-cluster/logs/passwd"}
+	if stdout, stderr, err := RunCommandOnNode(t, tc, 0, command); err != nil {
+		t.Log("stdout:", stdout)
+		t.Log("stderr:", stderr)
+		t.Fatalf("fail to copy file: %v", err)
+	}
+
+	command = []string{"curl", "-O", "--fail", "127.0.0.1:50000/logs/passwd"}
+	t.Logf("running %v", command)
+	if _, _, err := RunCommandOnNode(t, tc, 0, command); err == nil {
+		t.Fatalf("we should not be able to fetch logs from local artifact mirror")
+	}
+
+	command = []string{"curl", "-O", "--fail", "127.0.0.1:50000/../../../etc/passwd"}
+	t.Logf("running %v", command)
+	if _, _, err := RunCommandOnNode(t, tc, 0, command); err == nil {
+		t.Fatalf("we should not be able to fetch paths with ../")
+	}
+
+	t.Logf("%s: test complete", time.Now().Format(time.RFC3339))
+}