test installation without anything embedded in the binary (#363)

* test installation without anything embedded in the binary

* noop

* do not test if the install with no customization respects customization

Author: laverya

.github/workflows/pull-request.yaml

@@ -81,6 +81,7 @@ jobs:
         echo "# channel release object" > e2e/kots-release-install/release.yaml
         echo "versionLabel: \"${SHORT_SHA}\"" >> e2e/kots-release-install/release.yaml
         cat e2e/kots-release-install/release.yaml
+        cp output/bin/embedded-cluster output/bin/embedded-cluster-original
         make embedded-release # this is done after the metadata.json is generated so as to not include additional charts
     - name: Cache files for integration test
       if: github.actor != 'dependabot[bot]'
@@ -101,6 +102,7 @@ jobs:
         name: embedded-release
         path: |
           output/bin/embedded-cluster
+          output/bin/embedded-cluster-original
           output/bin/embedded-cluster-release-builder
 
   e2e:
@@ -121,6 +123,7 @@ jobs:
           - TestMultiNodeInstallation
           - TestMultiNodeReset
           - TestCommandsRequireSudo
+          - TestInstallWithoutEmbed
     steps:
     - name: Checkout
       uses: actions/checkout@v4

.github/workflows/release-dev.yaml

@@ -62,6 +62,7 @@ jobs:
           echo "# channel release object" > e2e/kots-release-install/release.yaml
           echo "versionLabel: \"${SHORT_SHA}\"" >> e2e/kots-release-install/release.yaml
           cat e2e/kots-release-install/release.yaml
+          cp output/bin/embedded-cluster output/bin/embedded-cluster-original
           make embedded-release # this is done after the metadata.json is generated so as to not include additional charts
       - name: Cache files for integration test
         if: github.actor != 'dependabot[bot]'
@@ -82,6 +83,7 @@ jobs:
           name: embedded-release
           path: |
             output/bin/embedded-cluster
+            output/bin/embedded-cluster-original
             output/bin/embedded-cluster-release-builder
   e2e:
     runs-on: ubuntu-latest
@@ -101,6 +103,7 @@ jobs:
           - TestMultiNodeInstallation
           - TestMultiNodeReset
           - TestCommandsRequireSudo
+          - TestInstallWithoutEmbed
     steps:
       - name: Checkout
         uses: actions/checkout@v4

e2e/install_test.go

@@ -336,6 +336,26 @@ func TestMultiNodeInstallation(t *testing.T) {
 	t.Logf("%s: test complete", time.Now().Format(time.RFC3339))
 }
 
+func TestInstallWithoutEmbed(t *testing.T) {
+	t.Parallel()
+	tc := cluster.NewTestCluster(&cluster.Input{
+		T:                   t,
+		Nodes:               1,
+		Image:               "rockylinux/8",
+		EmbeddedClusterPath: "../output/bin/embedded-cluster-original",
+	})
+	defer tc.Destroy()
+	t.Logf("%s: installing embedded-cluster on node 0", time.Now().Format(time.RFC3339))
+	line := []string{"default-install.sh"}
+	if stdout, stderr, err := RunCommandOnNode(t, tc, 0, line); err != nil {
+		t.Log("stdout:", stdout)
+		t.Log("stderr:", stderr)
+		t.Fatalf("fail to install embedded-cluster on node %s: %v", tc.Nodes[0], err)
+	}
+
+	t.Logf("%s: test complete", time.Now().Format(time.RFC3339))
+}
+
 func runPuppeteerAppStatusCheck(t *testing.T, node int, tc *cluster.Output) {
 	t.Logf("%s: installing puppeteer on node %d", time.Now().Format(time.RFC3339), node)
 	line := []string{"install-puppeteer.sh"}

e2e/scripts/default-install.sh

@@ -0,0 +1,89 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+wait_for_healthy_node() {
+    ready=$(kubectl get nodes | grep -v NotReady | grep -c Ready || true)
+    counter=0
+    while [ "$ready" -lt "1" ]; do
+        if [ "$counter" -gt 36 ]; then
+            return 1
+        fi
+        sleep 5
+        counter=$((counter+1))
+        echo "Waiting for node to be ready"
+        ready=$(kubectl get nodes | grep -v NotReady | grep -c Ready || true)
+        kubectl get nodes || true
+    done
+
+    return 0
+}
+
+wait_for_pods_running() {
+    local timeout="$1"
+    local start_time
+    local current_time
+    local elapsed_time
+    start_time=$(date +%s)
+    while true; do
+        current_time=$(date +%s)
+        elapsed_time=$((current_time - start_time))
+        if [ "$elapsed_time" -ge "$timeout" ]; then
+            kubectl get pods -A -o yaml || true
+            kubectl describe nodes || true
+            echo "Timed out waiting for all pods to be running."
+            return 1
+        fi
+        local non_running_pods
+        non_running_pods=$(kubectl get pods --all-namespaces --no-headers 2>/dev/null | awk '$4 != "Running" && $4 != "Completed" { print $0 }' | wc -l || echo 1)
+        if [ "$non_running_pods" -ne 0 ]; then
+            echo "Not all pods are running. Waiting."
+            kubectl get pods,nodes -A || true
+            sleep 5
+            continue
+        fi
+        echo "All pods are running."
+        return 0
+    done
+}
+
+check_openebs_storage_class() {
+    scs=$(kubectl get sc --no-headers | wc -l)
+    if [ "$scs" -ne "1" ]; then
+        echo "Expected 1 storage class, found $scs"
+        kubectl get sc
+        return 1
+    fi
+}
+
+main() {
+    if ! embedded-cluster install --no-prompt 2>&1 | tee /tmp/log ; then
+        cat /etc/os-release
+        echo "Failed to install embedded-cluster"
+        exit 1
+    fi
+    if ! grep -q "Admin Console is ready!" /tmp/log; then
+        echo "Failed to install embedded-cluster"
+        exit 1
+    fi
+    if ! wait_for_healthy_node; then
+        echo "Failed to install embedded-cluster"
+        exit 1
+    fi
+    if ! wait_for_pods_running 900; then
+        echo "Failed to install embedded-cluster"
+        exit 1
+    fi
+    if ! check_openebs_storage_class; then
+        echo "Failed to validate if only openebs storage class is present"
+        exit 1
+    fi
+    if ! systemctl restart embedded-cluster; then
+        echo "Failed to restart embedded-cluster service"
+        exit 1
+    fi
+}
+
+export EMBEDDED_CLUSTER_METRICS_BASEURL="https://staging.replicated.app"
+export KUBECONFIG=/root/.config/embedded-cluster/etc/kubeconfig
+export PATH=$PATH:/root/.config/embedded-cluster/bin
+main