Wait for installation status on install/restore (#577)

* add installation wait spinner to kubeutils

* don't wait for installation state as that is in the installer now

* call from within admin console outro

* fix spinner

* refactor installation spinner and outro message out of admin console

* remove 'finished', do not print admin console message if no license

* renaming

* comments

* wait for installation in old version upgrade test

* fix old version upgrade test again

* test with Alma, not Rocky, Linux

* better installation failure/timeout logs

* wording

* pass the installation status to a channel

* update operator chart

* LOGS LOGS LOGS

* handle zero case

* unsupported overrides chart update

* why did this message stop showing up

* fix finishing vs finalizing

* validating license, not finalizing

* debug

* ?

* add operator

* update operator version to v0.30.2

* remove logs

* attempt to resolve race

* add mutex, because of course that's the solution

* combine kotsadm output lines

* undo mask changes

* match wording in review

Additional components are ready/waiting for additional components to be ready

* simple suggestions from code review

* refactor to remove unneeded channel architecture

* close spinner on error

* branding configmap question

Author: laverya

e2e/scripts/check-airgap-installation-state.sh

@@ -1,30 +1,6 @@
 #!/usr/bin/env bash
 set -euox pipefail
 
-wait_for_installation() {
-    ready=$(kubectl get installations --no-headers | grep -c "Installed" || true)
-    counter=0
-    while [ "$ready" -lt "1" ]; do
-        if [ "$counter" -gt 36 ]; then
-            echo "installation did not become ready"
-            kubectl get installations 2>&1 || true
-            kubectl describe installations 2>&1 || true
-            kubectl get charts -A
-            kubectl describe chart -n kube-system k0s-addon-chart-ingress-nginx
-            kubectl get secrets -A
-            kubectl describe clusterconfig -A
-            echo "operator logs:"
-            kubectl logs -n embedded-cluster -l app.kubernetes.io/name=embedded-cluster-operator --tail=100
-            return 1
-        fi
-        sleep 5
-        counter=$((counter+1))
-        echo "Waiting for installation"
-        ready=$(kubectl get installations --no-headers | grep -c "Installed" || true)
-        kubectl get installations 2>&1 || true
-    done
-}
-
 wait_for_nginx_pods() {
     ready=$(kubectl get pods -n kotsadm -o jsonpath='{.items[*].metadata.name} {.items[*].status.phase}' | grep "nginx" | grep -c Running || true)
     counter=0
@@ -86,19 +62,18 @@ ensure_app_not_upgraded() {
 
 main() {
     local version="appver-$1"
-    sleep 30 # wait for kubectl to become available
+    sleep 10 # wait for kubectl to become available
 
     echo "pods"
     kubectl get pods -A
 
     echo "ensure that installation is installed"
-    wait_for_installation
     kubectl get installations --no-headers | grep -q "Installed"
 
     echo "ensure that the admin console branding is available and has the DR label"
     if ! kubectl get cm -n kotsadm -l replicated.com/disaster-recovery=infra | grep -q kotsadm-application-metadata; then
         echo "kotsadm-application-metadata configmap not found with the DR label"
-        kubectl get cm -n kotsadm
+        kubectl get cm -n kotsadm --show-labels
         kubectl get cm -n kotsadm kotsadm-application-metadata -o yaml
         exit 1
     fi

e2e/scripts/check-installation-state.sh

@@ -68,19 +68,22 @@ ensure_app_not_upgraded() {
 
 main() {
     local version="$1"
-    sleep 30 # wait for kubectl to become available
+    sleep 10 # wait for kubectl to become available
 
     echo "pods"
     kubectl get pods -A
 
     echo "ensure that installation is installed"
-    wait_for_installation
+    if echo "$version" | grep "pre-minio-removal"; then
+        echo "waiting for installation as this is a pre-minio-removal embedded-cluster version (and so the installer doesn't wait for the installation to be ready itself)"
+        wait_for_installation
+    fi
     kubectl get installations --no-headers | grep -q "Installed"
 
     echo "ensure that the admin console branding is available and has the DR label"
     if ! kubectl get cm -n kotsadm -l replicated.com/disaster-recovery=infra | grep -q kotsadm-application-metadata; then
         echo "kotsadm-application-metadata configmap not found with the DR label"
-        kubectl get cm -n kotsadm
+        kubectl get cm -n kotsadm --show-labels
         kubectl get cm -n kotsadm kotsadm-application-metadata -o yaml
         exit 1
     fi

e2e/scripts/single-node-airgap-install.sh

@@ -124,30 +124,6 @@ check_airgap_pvc() {
     fi
 }
 
-wait_for_installation() {
-    ready=$(kubectl get installations --no-headers | grep -c "Installed" || true)
-    counter=0
-    while [ "$ready" -lt "1" ]; do
-        if [ "$counter" -gt 36 ]; then
-            echo "installation did not become ready"
-            kubectl get installations 2>&1 || true
-            kubectl describe installations 2>&1 || true
-            kubectl get charts -A
-            kubectl describe chart -n kube-system k0s-addon-chart-ingress-nginx
-            kubectl get secrets -A
-            kubectl describe clusterconfig -A
-            echo "operator logs:"
-            kubectl logs -n embedded-cluster -l app.kubernetes.io/name=embedded-cluster-operator --tail=100
-            return 1
-        fi
-        sleep 5
-        counter=$((counter+1))
-        echo "Waiting for installation"
-        ready=$(kubectl get installations --no-headers | grep -c "Installed" || true)
-        kubectl get installations 2>&1 || true
-    done
-}
-
 main() {
     if ! embedded-cluster install --no-prompt --license /tmp/license.yaml --airgap-bundle /tmp/release.airgap 2>&1 | tee /tmp/log ; then
         echo "Failed to install embedded-cluster"
@@ -192,7 +168,6 @@ main() {
     fi
 
     echo "ensure that installation is installed"
-    wait_for_installation
     kubectl get installations --no-headers | grep -q "Installed"
 }
 

e2e/scripts/unsupported-overrides.sh

@@ -46,7 +46,7 @@ spec:
                 name: embedded-cluster-operator
                 namespace: embedded-cluster
                 order: 2
-                version: 0.13.0
+                version: 0.30.1
               - chartname: oci://registry.replicated.com/library/admin-console
                 name: admin-console
                 namespace: kotsadm

pkg/addons/adminconsole/adminconsole.go

@@ -249,49 +249,29 @@ func (a *AdminConsole) Outro(ctx context.Context, cli client.Client) error {
 		return fmt.Errorf("error waiting for admin console: %v", lasterr)
 	}
 
-	loading.Closef("Admin Console is ready!")
-	if a.licenseFile == "" {
-		return nil
-	}
+	if a.licenseFile != "" {
+		license, err := helpers.ParseLicense(a.licenseFile)
+		if err != nil {
+			loading.CloseWithError()
+			return fmt.Errorf("unable to parse license: %w", err)
+		}
 
-	license, err := helpers.ParseLicense(a.licenseFile)
-	if err != nil {
-		loading.CloseWithError()
-		return fmt.Errorf("unable to parse license: %w", err)
+		if err := kotscli.Install(kotscli.InstallOptions{
+			AppSlug:      license.Spec.AppSlug,
+			LicenseFile:  a.licenseFile,
+			Namespace:    a.namespace,
+			AirgapBundle: a.airgapBundle,
+		}, loading); err != nil {
+			loading.CloseWithError()
+			return err
+		}
 	}
 
-	if err := kotscli.Install(kotscli.InstallOptions{
-		AppSlug:      license.Spec.AppSlug,
-		LicenseFile:  a.licenseFile,
-		Namespace:    a.namespace,
-		AirgapBundle: a.airgapBundle,
-	}); err != nil {
-		return err
-	}
+	loading.Closef("Admin Console is ready!")
 
-	a.printSuccessMessage(license.Spec.AppSlug)
 	return nil
 }
 
-// printSuccessMessage prints the success message when the admin console is online.
-func (a *AdminConsole) printSuccessMessage(appSlug string) {
-	successColor := "\033[32m"
-	colorReset := "\033[0m"
-	ipaddr := defaults.TryDiscoverPublicIP()
-	if ipaddr == "" {
-		var err error
-		ipaddr, err = defaults.PreferredNodeIPAddress()
-		if err != nil {
-			logrus.Errorf("unable to determine node IP address: %v", err)
-			ipaddr = "NODE-IP-ADDRESS"
-		}
-	}
-	successMessage := fmt.Sprintf("Visit the admin console to configure and install %s: %shttp://%s:%v%s",
-		appSlug, successColor, ipaddr, DEFAULT_ADMIN_CONSOLE_NODE_PORT, colorReset,
-	)
-	logrus.Info(successMessage)
-}
-
 // New creates a new AdminConsole object.
 func New(ns string, useprompt bool, config v1beta1.ClusterConfig, licenseFile string, airgapBundle string) (*AdminConsole, error) {
 	return &AdminConsole{

pkg/addons/applier.go

@@ -12,7 +12,9 @@ import (
 	k0sconfig "github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1"
 	embeddedclusterv1beta1 "github.com/replicatedhq/embedded-cluster-kinds/apis/v1beta1"
 	"github.com/replicatedhq/embedded-cluster-kinds/types"
+	kotsv1beta1 "github.com/replicatedhq/kotskinds/apis/kots/v1beta1"
 	"github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
+	"github.com/sirupsen/logrus"
 	corev1 "k8s.io/api/core/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
@@ -65,6 +67,16 @@ func (a *Applier) Outro(ctx context.Context) error {
 			return err
 		}
 	}
+
+	err = spinForInstallation(ctx, kcli)
+	if err != nil {
+		return err
+	}
+
+	err = printKotsadmLinkMessage(a.licenseFile)
+	if err != nil {
+		return fmt.Errorf("unable to print success message: %w", err)
+	}
 	return nil
 }
 
@@ -282,6 +294,57 @@ func (a *Applier) waitForKubernetes(ctx context.Context) error {
 	}
 }
 
+func spinForInstallation(ctx context.Context, cli client.Client) error {
+	installSpin := spinner.Start()
+	installSpin.Infof("Waiting for additional components to be ready")
+
+	err := kubeutils.WaitForInstallation(ctx, cli, installSpin)
+	if err != nil {
+		installSpin.CloseWithError()
+		return fmt.Errorf("unable to wait for installation to be ready: %w", err)
+	}
+	installSpin.Closef("Additional components are ready!")
+	return nil
+}
+
+// printKotsadmLinkMessage prints the success message when the admin console is online.
+func printKotsadmLinkMessage(licenseFile string) error {
+	var err error
+	license := &kotsv1beta1.License{}
+	if licenseFile != "" {
+		license, err = helpers.ParseLicense(licenseFile)
+		if err != nil {
+			return fmt.Errorf("unable to parse license: %w", err)
+		}
+	}
+
+	successColor := "\033[32m"
+	colorReset := "\033[0m"
+	ipaddr := defaults.TryDiscoverPublicIP()
+	if ipaddr == "" {
+		var err error
+		ipaddr, err = defaults.PreferredNodeIPAddress()
+		if err != nil {
+			logrus.Errorf("unable to determine node IP address: %v", err)
+			ipaddr = "NODE-IP-ADDRESS"
+		}
+	}
+	var successMessage string
+	if license != nil {
+		successMessage = fmt.Sprintf("Visit the admin console to configure and install %s: %shttp://%s:%v%s",
+			license.Spec.AppSlug, successColor, ipaddr, adminconsole.DEFAULT_ADMIN_CONSOLE_NODE_PORT, colorReset,
+		)
+	} else {
+		successMessage = fmt.Sprintf("Visit the admin console to configure and install your application: %shttp://%s:%v%s",
+			successColor, ipaddr, adminconsole.DEFAULT_ADMIN_CONSOLE_NODE_PORT, colorReset,
+		)
+
+	}
+	logrus.Info(successMessage)
+
+	return nil
+}
+
 // NewApplier creates a new Applier instance with all addons registered.
 func NewApplier(opts ...Option) *Applier {
 	applier := &Applier{

pkg/kotscli/kotscli.go

@@ -24,7 +24,7 @@ type InstallOptions struct {
 	AirgapBundle string
 }
 
-func Install(opts InstallOptions) error {
+func Install(opts InstallOptions, msg *spinner.MessageWriter) error {
 	kotsBinPath, err := goods.MaterializeInternalBinary("kubectl-kots")
 	if err != nil {
 		return fmt.Errorf("unable to materialize kubectl-kots binary: %w", err)
@@ -64,19 +64,21 @@ func Install(opts InstallOptions) error {
 		lbreakfn = KotsOutputLineBreaker()
 	}
 
-	loading := spinner.Start(spinner.WithMask(maskfn), spinner.WithLineBreaker(lbreakfn))
+	msg.SetLineBreaker(lbreakfn)
+	msg.SetMask(maskfn)
+	defer msg.SetMask(nil)
+	defer msg.SetLineBreaker(nil)
+
 	runCommandOptions := helpers.RunCommandOptions{
-		Writer: loading,
+		Writer: msg,
 		Env: map[string]string{
 			"EMBEDDED_CLUSTER_ID": metrics.ClusterID().String(),
 		},
 	}
 	if err := helpers.RunCommandWithOptions(runCommandOptions, kotsBinPath, installArgs...); err != nil {
-		loading.CloseWithError()
 		return fmt.Errorf("unable to install the application: %w", err)
 	}
 
-	loading.Closef("Finished!")
 	return nil
 }