Set SSL Cert ConfigMap in KOTS and Improve Detection of Unhealthy Pods in Tests (#2239)

* re-add privateCAs value so that KOTS can use it to configure the SDK

* don't use old value and instead just set SSL_CERT_CONFIGMAP env var

* add additional handling to pod validation to not fail on job pods in an error state

* fix job completion validation

* add retry and timeout for pod validation

* allow pending pods in validation for TestMultiNodeHAInstallation test

* fix unbound variable

* bypass pending pods for multinode airgap ha

* fix extraEnv and check for unready pods

* fix dry run test

Author: diamonwiggins

e2e/install_test.go

@@ -1325,6 +1325,9 @@ func TestMultiNodeHAInstallation(t *testing.T) {
 
 	checkPostUpgradeStateWithOptions(t, tc, postUpgradeStateOptions{
 		node: 2,
+		withEnv: map[string]string{
+			"ALLOW_PENDING_PODS": "true",
+		},
 	})
 
 	t.Logf("%s: test complete", time.Now().Format(time.RFC3339))
@@ -1450,6 +1453,9 @@ func TestMultiNodeAirgapHAInstallation(t *testing.T) {
 
 	checkPostUpgradeStateWithOptions(t, tc, postUpgradeStateOptions{
 		node: 2,
+		withEnv: map[string]string{
+			"ALLOW_PENDING_PODS": "true",
+		},
 	})
 
 	t.Logf("%s: test complete", time.Now().Format(time.RFC3339))

e2e/scripts/check-airgap-installation-state.sh

@@ -43,7 +43,7 @@ main() {
         exit 1
     fi
 
-    validate_no_pods_in_crashloop
+    validate_all_pods_healthy
 }
 
 main "$@"

e2e/scripts/check-airgap-post-ha-state.sh

@@ -81,7 +81,7 @@ main() {
     # scale the second deployment back down so that they aren't restored in the DR test
     kubectl scale -n "$APP_NAMESPACE" deployment/second --replicas=0
 
-    validate_no_pods_in_crashloop
+    validate_all_pods_healthy
 }
 
 main "$@"

e2e/scripts/check-installation-state.sh

@@ -49,7 +49,7 @@ main() {
         validate_data_dirs
     fi
 
-    validate_no_pods_in_crashloop
+    validate_all_pods_healthy
 }
 
 main "$@"

e2e/scripts/check-post-ha-state.sh

@@ -61,7 +61,7 @@ main() {
         exit 1
     fi
 
-    validate_no_pods_in_crashloop
+    validate_all_pods_healthy
 }
 
 main "$@"

e2e/scripts/check-postupgrade-state.sh

@@ -111,7 +111,7 @@ main() {
 
     validate_data_dirs
 
-    validate_no_pods_in_crashloop
+    validate_all_pods_healthy
 }
 
 main "$@"

e2e/scripts/common.sh

@@ -455,12 +455,116 @@ validate_data_dirs() {
     fi
 }
 
-validate_no_pods_in_crashloop() {
-    if kubectl get pods -A | grep CrashLoopBackOff -q ; then
-        echo "found pods in CrashLoopBackOff state"
-        kubectl get pods -A | grep CrashLoopBackOff
-        exit 1
+validate_non_job_pods_healthy() {
+    local unhealthy_pods
+    local unready_pods
+    
+    # Check for environment variable override (used by specific tests)
+    if [ "${ALLOW_PENDING_PODS:-}" = "true" ]; then
+        # Allow Running, Completed, Succeeded, Pending
+        unhealthy_pods=$(kubectl get pods -A --no-headers -o custom-columns="NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase,OWNER:.metadata.ownerReferences[0].kind" | \
+            awk '$4 != "Job" && ($3 != "Running" && $3 != "Completed" && $3 != "Succeeded" && $3 != "Pending") { print $1 "/" $2 " (" $3 ")" }')
+        echo "All non-Job pods are healthy (allowing Pending pods)"
+    else
+        # Default: only allow Running, Completed, Succeeded  
+        unhealthy_pods=$(kubectl get pods -A --no-headers -o custom-columns="NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase,OWNER:.metadata.ownerReferences[0].kind" | \
+            awk '$4 != "Job" && ($3 != "Running" && $3 != "Completed" && $3 != "Succeeded") { print $1 "/" $2 " (" $3 ")" }')
+        echo "All non-Job pods are healthy"
+    fi
+    
+    # Check container readiness for Running pods (skip Completed/Succeeded pods as they don't need to be ready)
+    unready_pods=$(kubectl get pods -A --no-headers -o custom-columns="NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase,READY:.status.containerStatuses[*].ready,OWNER:.metadata.ownerReferences[0].kind" | \
+        awk '$5 != "Job" && $3 == "Running" && ($4 == "" || $4 !~ /^(true[[:space:]]*)*$/) { print $1 "/" $2 " (not ready)" }')
+    
+    local has_issues=0
+    
+    if [ -n "$unhealthy_pods" ]; then
+        echo "found non-Job pods in unhealthy state:"
+        echo "$unhealthy_pods"
+        has_issues=1
+    fi
+    
+    if [ -n "$unready_pods" ]; then
+        echo "found non-Job pods that are Running but not ready:"
+        echo "$unready_pods"
+        has_issues=1
+    fi
+    
+    if [ $has_issues -eq 1 ]; then
+        return 1
+    fi
+    
+    return 0
+}
+
+validate_jobs_completed() {
+    local incomplete_jobs
+    # Check that all Jobs have succeeded (status.succeeded should equal spec.completions)
+    # Flag any job that hasn't fully succeeded
+    incomplete_jobs=$(kubectl get jobs -A --no-headers -o custom-columns="NAMESPACE:.metadata.namespace,NAME:.metadata.name,COMPLETIONS:.spec.completions,SUCCESSFUL:.status.succeeded" | \
+        awk '$4 != $3 { print $1 "/" $2 " (succeeded: " $4 "/" $3 ")" }')
+    
+    if [ -n "$incomplete_jobs" ]; then
+        echo "found Jobs that have not completed successfully:"
+        echo "$incomplete_jobs"
+        echo ""
+        echo "Job details:"
+        kubectl get jobs -A
+        return 1
+    fi
+    echo "All Jobs have completed successfully"
+    return 0
+}
+
+validate_all_pods_healthy() {
+    local timeout=300  # 5 minutes
+    local start_time
+    local current_time
+    local elapsed_time
+    start_time=$(date +%s)
+    
+    # Show what mode we're in
+    if [ "${ALLOW_PENDING_PODS:-}" = "true" ]; then
+        echo "Validating pod and job health (allowing Pending pods)..."
+    else
+        echo "Validating pod and job health (default: Running, Completed, Succeeded)..."
     fi
+    
+    while true; do
+        current_time=$(date +%s)
+        elapsed_time=$((current_time - start_time))
+        
+        if [ "$elapsed_time" -ge "$timeout" ]; then
+            echo "Timed out waiting for pods and jobs to be healthy after 5 minutes"
+            
+            # Show detailed failure info
+            validate_non_job_pods_healthy || true
+            echo ""
+            validate_jobs_completed || true
+            
+            return 1
+        fi
+        
+        # Check if both validations pass
+        local pods_healthy=0
+        local jobs_healthy=0
+        
+        if validate_non_job_pods_healthy >/dev/null 2>&1; then
+            pods_healthy=1
+        fi
+        
+        if validate_jobs_completed >/dev/null 2>&1; then
+            jobs_healthy=1
+        fi
+        
+        if [ $pods_healthy -eq 1 ] && [ $jobs_healthy -eq 1 ]; then
+            echo "All pods and jobs are healthy"
+            return 0
+        fi
+        
+        echo "Waiting for pods and jobs to be healthy... (${elapsed_time}s elapsed)"
+        sleep 10
+    done
 }
 
 validate_worker_profile() {

pkg/addons/adminconsole/static/values.tpl.yaml

@@ -19,3 +19,6 @@ passwordSecretRef:
     name: kotsadm-password
 service:
     enabled: false
+extraEnv:
+  - name: SSL_CERT_CONFIGMAP
+    value: "kotsadm-private-cas"

pkg/addons/adminconsole/values.go

@@ -57,6 +57,10 @@ func (a *AdminConsole) GenerateHelmValues(ctx context.Context, kcli client.Clien
 			"name":  "ENABLE_IMPROVED_DR",
 			"value": "true",
 		},
+		{
+			"name":  "SSL_CERT_CONFIGMAP",
+			"value": "kotsadm-private-cas",
+		},
 	}
 
 	if a.Proxy != nil {

tests/dryrun/install_test.go

@@ -722,6 +722,10 @@ func TestHTTPProxyWithCABundleConfiguration(t *testing.T) {
 				"name":  "ENABLE_IMPROVED_DR",
 				"value": "true",
 			},
+			{
+				"name":  "SSL_CERT_CONFIGMAP",
+				"value": "kotsadm-private-cas",
+			},
 			{
 				"name":  "HTTP_PROXY",
 				"value": "http://localhost:3128",