fix(lam): wait for service after start (#1883)

* fix(lam): wait for service after start

* f

* f

* f

* f

* f

Author: emosbaugh

cmd/installer/cli/install.go

@@ -608,7 +608,7 @@ func installAndStartCluster(ctx context.Context, networkInterface string, airgap
 		return nil, fmt.Errorf("create config file: %w", err)
 	}
 	logrus.Debugf("creating systemd unit files")
-	if err := createSystemdUnitFiles(false, proxy); err != nil {
+	if err := createSystemdUnitFiles(ctx, false, proxy); err != nil {
 		return nil, fmt.Errorf("create systemd unit files: %w", err)
 	}
 
@@ -769,7 +769,7 @@ func validateAdminConsolePassword(password, passwordCheck string) bool {
 
 // createSystemdUnitFiles links the k0s systemd unit file. this also creates a new
 // systemd unit file for the local artifact mirror service.
-func createSystemdUnitFiles(isWorker bool, proxy *ecv1beta1.ProxySpec) error {
+func createSystemdUnitFiles(ctx context.Context, isWorker bool, proxy *ecv1beta1.ProxySpec) error {
 	dst := systemdUnitFileName()
 	if _, err := os.Lstat(dst); err == nil {
 		if err := os.Remove(dst); err != nil {
@@ -793,7 +793,7 @@ func createSystemdUnitFiles(isWorker bool, proxy *ecv1beta1.ProxySpec) error {
 	if _, err := helpers.RunCommand("systemctl", "daemon-reload"); err != nil {
 		return fmt.Errorf("unable to get reload systemctl daemon: %w", err)
 	}
-	if err := installAndEnableLocalArtifactMirror(); err != nil {
+	if err := installAndEnableLocalArtifactMirror(ctx); err != nil {
 		return fmt.Errorf("unable to install and enable local artifact mirror: %w", err)
 	}
 	return nil
@@ -828,7 +828,7 @@ Environment="NO_PROXY=%s"`, httpProxy, httpsProxy, noProxy)
 // installAndEnableLocalArtifactMirror installs and enables the local artifact mirror. This
 // service is responsible for serving on localhost, through http, all files that are used
 // during a cluster upgrade.
-func installAndEnableLocalArtifactMirror() error {
+func installAndEnableLocalArtifactMirror(ctx context.Context) error {
 	materializer := goods.NewMaterializer()
 	if err := materializer.LocalArtifactMirrorUnitFile(); err != nil {
 		return fmt.Errorf("failed to materialize artifact mirror unit: %w", err)
@@ -839,15 +839,50 @@ func installAndEnableLocalArtifactMirror() error {
 	if _, err := helpers.RunCommand("systemctl", "daemon-reload"); err != nil {
 		return fmt.Errorf("unable to get reload systemctl daemon: %w", err)
 	}
+	if _, err := helpers.RunCommand("systemctl", "enable", "local-artifact-mirror"); err != nil {
+		return fmt.Errorf("unable to enable the local artifact mirror service: %w", err)
+	}
+	logrus.Debugf("Starting local artifact mirror")
 	if _, err := helpers.RunCommand("systemctl", "start", "local-artifact-mirror"); err != nil {
 		return fmt.Errorf("unable to start the local artifact mirror: %w", err)
 	}
-	if _, err := helpers.RunCommand("systemctl", "enable", "local-artifact-mirror"); err != nil {
-		return fmt.Errorf("unable to start the local artifact mirror service: %w", err)
+	if err := waitForLocalArtifactMirror(ctx); err != nil {
+		return fmt.Errorf("unable to wait for the local artifact mirror: %w", err)
 	}
+	logrus.Debugf("Local artifact mirror started!")
 	return nil
 }
 
+func waitForLocalArtifactMirror(ctx context.Context) error {
+	consecutiveSuccesses := 0
+	requiredSuccesses := 3
+	maxAttempts := 30
+	checkInterval := 2 * time.Second
+
+	var lastErr error
+	for attempt := 0; attempt < maxAttempts; attempt++ {
+		_, err := helpers.RunCommand("systemctl", "status", "local-artifact-mirror")
+		if err == nil {
+			consecutiveSuccesses++
+			if consecutiveSuccesses >= requiredSuccesses {
+				return nil
+			}
+		} else {
+			consecutiveSuccesses = 0
+			lastErr = err
+		}
+
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-time.After(checkInterval):
+			continue
+		}
+	}
+
+	return lastErr
+}
+
 const (
 	localArtifactMirrorDropInFileContents = `[Service]
 Environment="LOCAL_ARTIFACT_MIRROR_PORT=%d"

cmd/installer/cli/join.go

@@ -299,7 +299,8 @@ func installAndJoinCluster(ctx context.Context, jcmd *kotsadm.JoinCommandRespons
 
 	logrus.Debugf("creating systemd unit files")
 	// both controller and worker nodes will have 'worker' in the join command
-	if err := createSystemdUnitFiles(!strings.Contains(jcmd.K0sJoinCommand, "controller"), jcmd.InstallationSpec.Proxy); err != nil {
+	isWorker := !strings.Contains(jcmd.K0sJoinCommand, "controller")
+	if err := createSystemdUnitFiles(ctx, isWorker, jcmd.InstallationSpec.Proxy); err != nil {
 		return fmt.Errorf("unable to create systemd unit files: %w", err)
 	}
 

cmd/installer/kotscli/kotscli.go

@@ -1,9 +1,7 @@
 package kotscli
 
 import (
-	"bytes"
 	"fmt"
-	"io"
 	"os"
 	"regexp"
 	"strings"
@@ -14,7 +12,6 @@ import (
 	"github.com/replicatedhq/embedded-cluster/pkg/release"
 	"github.com/replicatedhq/embedded-cluster/pkg/runtimeconfig"
 	"github.com/replicatedhq/embedded-cluster/pkg/spinner"
-	"github.com/sirupsen/logrus"
 )
 
 var (
@@ -78,21 +75,14 @@ func Install(opts InstallOptions, msg *spinner.MessageWriter) error {
 	defer msg.SetMask(nil)
 	defer msg.SetLineBreaker(nil)
 
-	stdout := bytes.NewBuffer(nil)
-	stderr := bytes.NewBuffer(nil)
-
 	runCommandOptions := helpers.RunCommandOptions{
-		Stdout: io.MultiWriter(msg, stdout),
-		Stderr: stderr,
+		Stdout:       msg,
+		LogOnSuccess: true,
 		Env: map[string]string{
 			"EMBEDDED_CLUSTER_ID": metrics.ClusterID().String(),
 		},
 	}
 	err = helpers.RunCommandWithOptions(runCommandOptions, kotsBinPath, installArgs...)
-	logrus.Debugf("kotscli stdout: %s", stdout.String())
-	if out := strings.TrimSpace(stderr.String()); out != "" {
-		logrus.Debugf("kotscli stderr: %s", out)
-	}
 	if err != nil {
 		return fmt.Errorf("unable to install the application: %w", err)
 	}