Merge remote-tracking branch 'origin/main' into k0s-1-29

sgalsaleh · sgalsaleh · commit e28dcda3465d · 2025-06-04T13:57:28.000-07:00
diff --git a/.gitignore b/.gitignore
@@ -30,4 +30,11 @@ hack/release
 *.test
 
 # helm charts dependencies
-*.tgz
+*.tgz
+
+# test coverage files
+cover.out
+
+# e2e test runs locally
+e2e/playwright*.tar.gz
+e2e/support-bundle*.tar.gz
diff --git a/cmd/installer/goods/support/host-support-bundle.tmpl.yaml b/cmd/installer/goods/support/host-support-bundle.tmpl.yaml
@@ -299,6 +299,16 @@ spec:
       collectorName: "ps-detect-antivirus-and-security-tools"
       command: "sh"
       args: [-c, "ps -ef | grep -E 'clamav|sophos|esets_daemon|fsav|symantec|mfend|ds_agent|kav|bdagent|s1agent|falcon|illumio|xagt|wdavdaemon|mdatp' | grep -v grep"]
+  - systemPackages:
+      collectorName: security-tools-packages
+      ubuntu:
+        - sdcss-kmod
+        - sdcss
+        - sdcss-scripts
+      rhel:
+        - sdcss-kmod
+        - sdcss
+        - sdcss-scripts
   - filesystemPerformance:
         collectorName: filesystem-write-latency-etcd
         timeout: 5m
@@ -620,7 +630,7 @@ spec:
   - textAnalyze:
       checkName: "Detect Threat Management and Network Security Tools"
       fileName: host-collectors/run-host/ps-detect-antivirus-and-security-tools.txt
-      regex: '\b(clamav|sophos|esets_daemon|fsav|symantec|mfend|ds_agent|kav|bdagent|s1agent|falcon|illumio|xagt|wdavdaemon)\b'
+      regex: '\b(clamav|sophos|esets_daemon|fsav|symantec|mfend|ds_agent|kav|bdagent|s1agent|falcon|illumio|xagt|wdavdaemon|mdatp)\b'
       ignoreIfNoFiles: true
       outcomes:
         - fail:
@@ -629,6 +639,15 @@ spec:
         - pass:
             when: "false"
             message: "No antivirus or network security tools detected."
+  - systemPackages:
+      checkName: "Detected Security Packages"
+      collectorName: security-tools-packages
+      outcomes:
+        - fail:
+            when: '{{ "{{" }} .IsInstalled {{ "}}" }}'
+            message: Package {{ "{{" }} .Name {{ "}}" }} is installed. This tool can interfere with kubernetes operation. Ensure the tool is either disabled or configured to not interfere with kubernetes operation.
+        - pass:
+            message: Package {{ "{{" }} .Name {{ "}}" }} is not installed
   - filesystemPerformance:
       checkName: Filesystem Write Latency
       collectorName: filesystem-write-latency-etcd
diff --git a/e2e/install_test.go b/e2e/install_test.go
@@ -998,7 +998,6 @@ func TestMultiNodeAirgapUpgradeSameK0s(t *testing.T) {
 		Nodes:        2,
 		Distribution: "ubuntu",
 		Version:      "22.04",
-		InstanceType: "r1.medium",
 	})
 	defer tc.Cleanup()
 
@@ -1074,7 +1073,6 @@ func TestMultiNodeAirgapUpgrade(t *testing.T) {
 		Nodes:        2,
 		Distribution: "ubuntu",
 		Version:      "22.04",
-		InstanceType: "r1.medium",
 	})
 	defer tc.Cleanup()
 
@@ -1157,7 +1155,6 @@ func TestMultiNodeAirgapUpgradePreviousStable(t *testing.T) {
 		Nodes:        2,
 		Distribution: "ubuntu",
 		Version:      "22.04",
-		InstanceType: "r1.medium",
 	})
 	defer tc.Cleanup(withEnv)
 
@@ -1328,6 +1325,9 @@ func TestMultiNodeHAInstallation(t *testing.T) {
 
 	checkPostUpgradeStateWithOptions(t, tc, postUpgradeStateOptions{
 		node: 2,
+		withEnv: map[string]string{
+			"ALLOW_PENDING_PODS": "true",
+		},
 	})
 
 	t.Logf("%s: test complete", time.Now().Format(time.RFC3339))
@@ -1346,7 +1346,6 @@ func TestMultiNodeAirgapHAInstallation(t *testing.T) {
 		Nodes:                  4,
 		Distribution:           "ubuntu",
 		Version:                "22.04",
-		InstanceType:           "r1.medium",
 		SupportBundleNodeIndex: 2,
 	})
 	defer tc.Cleanup()
@@ -1454,6 +1453,9 @@ func TestMultiNodeAirgapHAInstallation(t *testing.T) {
 
 	checkPostUpgradeStateWithOptions(t, tc, postUpgradeStateOptions{
 		node: 2,
+		withEnv: map[string]string{
+			"ALLOW_PENDING_PODS": "true",
+		},
 	})
 
 	t.Logf("%s: test complete", time.Now().Format(time.RFC3339))
@@ -1609,7 +1611,6 @@ func TestFiveNodesAirgapUpgrade(t *testing.T) {
 		Nodes:        5,
 		Distribution: "ubuntu",
 		Version:      "22.04",
-		InstanceType: "r1.medium",
 	})
 	defer tc.Cleanup()
 
diff --git a/e2e/playwright/tests/deploy-upgrade/test.spec.ts b/e2e/playwright/tests/deploy-upgrade/test.spec.ts
@@ -35,7 +35,7 @@ async function fillConfigForm(iframe: FrameLocator) {
 
 async function handlePreflightChecks(iframe: FrameLocator) {
   await expect(iframe.getByText('Preflight checks', { exact: true })).toBeVisible({ timeout: 10 * 1000 });
-  await expect(iframe.getByRole('button', { name: 'Rerun' })).toBeVisible({ timeout: 10 * 1000 });
+  await expect(iframe.getByRole('button', { name: 'Rerun' })).toBeVisible({ timeout: 30 * 1000 });
   await expect(iframe.locator('#app')).toContainText('The Volume Snapshots CRD exists');
   await expect(iframe.getByRole('button', { name: 'Back: Config' })).toBeVisible();
   await iframe.getByRole('button', { name: 'Next: Confirm and deploy' }).click();
diff --git a/e2e/restore_test.go b/e2e/restore_test.go
@@ -359,7 +359,6 @@ func TestSingleNodeAirgapDisasterRecovery(t *testing.T) {
 		Nodes:        1,
 		Distribution: "ubuntu",
 		Version:      "22.04",
-		InstanceType: "r1.medium",
 	})
 	defer tc.Cleanup()
 
@@ -642,7 +641,6 @@ func TestMultiNodeAirgapHADisasterRecovery(t *testing.T) {
 		Nodes:        3,
 		Distribution: "ubuntu",
 		Version:      "22.04",
-		InstanceType: "r1.medium",
 	})
 	defer tc.Cleanup(withEnv)
 
diff --git a/e2e/scripts/check-airgap-installation-state.sh b/e2e/scripts/check-airgap-installation-state.sh
@@ -43,7 +43,7 @@ main() {
         exit 1
     fi
 
-    validate_no_pods_in_crashloop
+    validate_all_pods_healthy
 }
 
 main "$@"
diff --git a/e2e/scripts/check-airgap-post-ha-state.sh b/e2e/scripts/check-airgap-post-ha-state.sh
@@ -81,7 +81,7 @@ main() {
     # scale the second deployment back down so that they aren't restored in the DR test
     kubectl scale -n "$APP_NAMESPACE" deployment/second --replicas=0
 
-    validate_no_pods_in_crashloop
+    validate_all_pods_healthy
 }
 
 main "$@"
diff --git a/e2e/scripts/check-installation-state.sh b/e2e/scripts/check-installation-state.sh
@@ -49,7 +49,7 @@ main() {
         validate_data_dirs
     fi
 
-    validate_no_pods_in_crashloop
+    validate_all_pods_healthy
 }
 
 main "$@"
diff --git a/e2e/scripts/check-post-ha-state.sh b/e2e/scripts/check-post-ha-state.sh
@@ -61,7 +61,7 @@ main() {
         exit 1
     fi
 
-    validate_no_pods_in_crashloop
+    validate_all_pods_healthy
 }
 
 main "$@"
diff --git a/e2e/scripts/check-postupgrade-state.sh b/e2e/scripts/check-postupgrade-state.sh
@@ -111,7 +111,7 @@ main() {
 
     validate_data_dirs
 
-    validate_no_pods_in_crashloop
+    validate_all_pods_healthy
 }
 
 main "$@"
diff --git a/e2e/scripts/common.sh b/e2e/scripts/common.sh
@@ -455,12 +455,116 @@ validate_data_dirs() {
     fi
 }
 
-validate_no_pods_in_crashloop() {
-    if kubectl get pods -A | grep CrashLoopBackOff -q ; then
-        echo "found pods in CrashLoopBackOff state"
-        kubectl get pods -A | grep CrashLoopBackOff
-        exit 1
+validate_non_job_pods_healthy() {
+    local unhealthy_pods
+    local unready_pods
+    
+    # Check for environment variable override (used by specific tests)
+    if [ "${ALLOW_PENDING_PODS:-}" = "true" ]; then
+        # Allow Running, Completed, Succeeded, Pending
+        unhealthy_pods=$(kubectl get pods -A --no-headers -o custom-columns="NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase,OWNER:.metadata.ownerReferences[0].kind" | \
+            awk '$4 != "Job" && ($3 != "Running" && $3 != "Completed" && $3 != "Succeeded" && $3 != "Pending") { print $1 "/" $2 " (" $3 ")" }')
+        echo "All non-Job pods are healthy (allowing Pending pods)"
+    else
+        # Default: only allow Running, Completed, Succeeded  
+        unhealthy_pods=$(kubectl get pods -A --no-headers -o custom-columns="NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase,OWNER:.metadata.ownerReferences[0].kind" | \
+            awk '$4 != "Job" && ($3 != "Running" && $3 != "Completed" && $3 != "Succeeded") { print $1 "/" $2 " (" $3 ")" }')
+        echo "All non-Job pods are healthy"
+    fi
+    
+    # Check container readiness for Running pods (skip Completed/Succeeded pods as they don't need to be ready)
+    unready_pods=$(kubectl get pods -A --no-headers -o custom-columns="NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase,READY:.status.containerStatuses[*].ready,OWNER:.metadata.ownerReferences[0].kind" | \
+        awk '$5 != "Job" && $3 == "Running" && ($4 == "" || $4 !~ /^(true[[:space:]]*)*$/) { print $1 "/" $2 " (not ready)" }')
+    
+    local has_issues=0
+    
+    if [ -n "$unhealthy_pods" ]; then
+        echo "found non-Job pods in unhealthy state:"
+        echo "$unhealthy_pods"
+        has_issues=1
+    fi
+    
+    if [ -n "$unready_pods" ]; then
+        echo "found non-Job pods that are Running but not ready:"
+        echo "$unready_pods"
+        has_issues=1
+    fi
+    
+    if [ $has_issues -eq 1 ]; then
+        return 1
+    fi
+    
+    return 0
+}
+
+validate_jobs_completed() {
+    local incomplete_jobs
+    # Check that all Jobs have succeeded (status.succeeded should equal spec.completions)
+    # Flag any job that hasn't fully succeeded
+    incomplete_jobs=$(kubectl get jobs -A --no-headers -o custom-columns="NAMESPACE:.metadata.namespace,NAME:.metadata.name,COMPLETIONS:.spec.completions,SUCCESSFUL:.status.succeeded" | \
+        awk '$4 != $3 { print $1 "/" $2 " (succeeded: " $4 "/" $3 ")" }')
+    
+    if [ -n "$incomplete_jobs" ]; then
+        echo "found Jobs that have not completed successfully:"
+        echo "$incomplete_jobs"
+        echo ""
+        echo "Job details:"
+        kubectl get jobs -A
+        return 1
+    fi
+    echo "All Jobs have completed successfully"
+    return 0
+}
+
+validate_all_pods_healthy() {
+    local timeout=300  # 5 minutes
+    local start_time
+    local current_time
+    local elapsed_time
+    start_time=$(date +%s)
+    
+    # Show what mode we're in
+    if [ "${ALLOW_PENDING_PODS:-}" = "true" ]; then
+        echo "Validating pod and job health (allowing Pending pods)..."
+    else
+        echo "Validating pod and job health (default: Running, Completed, Succeeded)..."
     fi
+    
+    while true; do
+        current_time=$(date +%s)
+        elapsed_time=$((current_time - start_time))
+        
+        if [ "$elapsed_time" -ge "$timeout" ]; then
+            echo "Timed out waiting for pods and jobs to be healthy after 5 minutes"
+            
+            # Show detailed failure info
+            validate_non_job_pods_healthy || true
+            echo ""
+            validate_jobs_completed || true
+            
+            return 1
+        fi
+        
+        # Check if both validations pass
+        local pods_healthy=0
+        local jobs_healthy=0
+        
+        if validate_non_job_pods_healthy >/dev/null 2>&1; then
+            pods_healthy=1
+        fi
+        
+        if validate_jobs_completed >/dev/null 2>&1; then
+            jobs_healthy=1
+        fi
+        
+        if [ $pods_healthy -eq 1 ] && [ $jobs_healthy -eq 1 ]; then
+            echo "All pods and jobs are healthy"
+            return 0
+        fi
+        
+        echo "Waiting for pods and jobs to be healthy... (${elapsed_time}s elapsed)"
+        sleep 10
+    done
 }
 
 validate_worker_profile() {
diff --git a/operator/Makefile b/operator/Makefile
@@ -126,7 +126,7 @@ $(CONTROLLER_GEN): $(LOCALBIN)
 
 .PHONY: schemas
 schemas: fmt controller-gen
-	go build ${LDFLAGS} -o bin/schemagen ./schemagen
+	go build -o bin/schemagen ./schemagen
 	./bin/schemagen --output-dir ./schemas
 
 .PHONY: envtest
diff --git a/pkg/addons/adminconsole/static/metadata.yaml b/pkg/addons/adminconsole/static/metadata.yaml
@@ -5,26 +5,26 @@
 # $ make buildtools
 # $ output/bin/buildtools update addon <addon name>
 #
-version: 1.124.16-ec.5
+version: 1.124.17
 location: oci://proxy.replicated.com/anonymous/registry.replicated.com/library/admin-console
 images:
     kotsadm:
         repo: proxy.replicated.com/anonymous/kotsadm/kotsadm
         tag:
-            amd64: v1.124.16-ec.5-amd64@sha256:f4519dd15b4b978157e699ea16616fba646d7b599e8c579b0e83774c1e2e02bd
-            arm64: v1.124.16-ec.5-arm64@sha256:53310cc5e666a8d1acafa4309a88aceedc134a86616f58306f1f665a89ddf9ed
+            amd64: v1.124.17-amd64@sha256:60da5bcc432cf2046ff3241ac034fd9bd602f592b66d457583ce9c2d19966383
+            arm64: v1.124.17-arm64@sha256:21413e5ea2859f96abd9b5b49407afde94aa4b95579d17d153515e4ae6ebcb35
     kotsadm-migrations:
         repo: proxy.replicated.com/anonymous/kotsadm/kotsadm-migrations
         tag:
-            amd64: v1.124.16-ec.5-amd64@sha256:1c3daf5c301fc6a9d64dc7f8776d40db9be5a0e82766ba21d2fd4daa35a525eb
-            arm64: v1.124.16-ec.5-arm64@sha256:8139f3566f96f772bf52244c0f9ddaad94193a39e64b76ee700e02adad8b441a
+            amd64: v1.124.17-amd64@sha256:65208a0c17a3334d6199c39f54db7d4b62d97f0c3efb944aeb917cd95a72b856
+            arm64: v1.124.17-arm64@sha256:782c6c3b8621b99fed0a26e864f68008e57a46840f30f2dfc5a404df6685a100
     kurl-proxy:
         repo: proxy.replicated.com/anonymous/kotsadm/kurl-proxy
         tag:
-            amd64: v1.124.16-ec.5-amd64@sha256:1a367bae52522e45eb6a30794d7a5796b6010703926524aca31e151eca382d8a
-            arm64: v1.124.16-ec.5-arm64@sha256:d0629818c1bb38a2a876becdce3c94c4b96dfd9f9d21e4f3d7225703994518d0
+            amd64: v1.124.17-amd64@sha256:723c222ec08e6bdbc907c41e957e54cfd2b357b7b9f7d5a84e8a0083028642f8
+            arm64: v1.124.17-arm64@sha256:a972e9a8fcb428263f94c8512d06675c19a48e878b54933888d5df5a47fa2f8f
     rqlite:
         repo: proxy.replicated.com/anonymous/kotsadm/rqlite
         tag:
-            amd64: 8.37.0-r0-amd64@sha256:e84bc7bc4c26b81c5082b6da9c2f7f37099d9dbcbcb819fbdc27cfb56d1dbc1e
-            arm64: 8.37.0-r0-arm64@sha256:c5b28b24a7633fb09d7fa9a63ee59b37ce4211a62cc0049544c027d1c3021e0e
+            amd64: 8.37.2-r0-amd64@sha256:72e527882ddbb28c055f85257001969f1e6bea30816747a9d42b677e8ad802b2
+            arm64: 8.37.2-r0-arm64@sha256:3d78afc4011c7d24c603bd6312a66be9e0e663057e397ebcd0600be101464a4b
diff --git a/pkg/addons/adminconsole/static/values.tpl.yaml b/pkg/addons/adminconsole/static/values.tpl.yaml
@@ -19,3 +19,6 @@ passwordSecretRef:
     name: kotsadm-password
 service:
     enabled: false
+extraEnv:
+  - name: SSL_CERT_CONFIGMAP
+    value: "kotsadm-private-cas"
diff --git a/pkg/addons/adminconsole/values.go b/pkg/addons/adminconsole/values.go
@@ -57,6 +57,10 @@ func (a *AdminConsole) GenerateHelmValues(ctx context.Context, kcli client.Clien
 			"name":  "ENABLE_IMPROVED_DR",
 			"value": "true",
 		},
+		{
+			"name":  "SSL_CERT_CONFIGMAP",
+			"value": "kotsadm-private-cas",
+		},
 	}
 
 	if a.Proxy != nil {
diff --git a/pkg/addons/openebs/static/metadata.yaml b/pkg/addons/openebs/static/metadata.yaml
@@ -5,21 +5,21 @@
 # $ make buildtools
 # $ output/bin/buildtools update addon <addon name>
 #
-version: 4.2.0
+version: 4.3.0-develop
 location: oci://proxy.replicated.com/anonymous/registry.replicated.com/ec-charts/openebs
 images:
     kubectl:
         repo: proxy.replicated.com/anonymous/replicated/ec-kubectl
         tag:
-            amd64: 1.33.1-r1-amd64@sha256:5c2c30d0c7c7487e563a99f12eb4e486cc7d5c873b3e02e43f161c1e13d2a427
-            arm64: 1.33.1-r1-arm64@sha256:4c532d484cfe82030d34df937d11539f1191635aaee299b9a614b06586210d02
+            amd64: 1.33.1-r2-amd64@sha256:3fdcb728e6b4c950db0498fe2d84836dc8a00568daf7d08f024b2408c6eae4f9
+            arm64: 1.33.1-r2-arm64@sha256:d738a903196054e3523aa6a782115d875dd5353ae98b93607f6e8380144f89b5
     openebs-linux-utils:
         repo: proxy.replicated.com/anonymous/replicated/ec-openebs-linux-utils
         tag:
-            amd64: 4.2.0-amd64@sha256:695a78180f7b07f80379199b851791042e11db039f655479aa0d1fd736cdc51a
-            arm64: 4.2.0-arm64@sha256:abdccdf5ca021223d64c1045fbf9401c6e7cca4a6b10fe1838dd1b995fd658b3
+            amd64: 4.3.0-develop-amd64@sha256:0e9dfd0ada5b9d218b10e26bc1a02a7c769df6a794a5f97a6a2b94fcbe7eea7f
+            arm64: 4.3.0-develop-arm64@sha256:ec4e819e41b74981eb750993690b5db48fe46099d91394b5578a0764f7e34e4c
     openebs-provisioner-localpv:
         repo: proxy.replicated.com/anonymous/replicated/ec-openebs-provisioner-localpv
         tag:
-            amd64: 4.2.0-r5-amd64@sha256:d58b1f4745e802cd90ed9e1d0aae4f7b6e07d8441898bd3aa1dceffa24e1ddd2
-            arm64: 4.2.0-r5-arm64@sha256:e154823edf377db6be8a1969913c3d03618a275053b81c0ad3a56e249df79170
+            amd64: 4.3.0-r0-amd64@sha256:dc68b841610e53bd91bea01ccd5593aa4db863fbcc999e46b2c14fda58dad679
+            arm64: 4.3.0-r0-arm64@sha256:18be54d2322a61ace6cc185fba88ff79dab4c76e049ca96da82e38ee5712f112
diff --git a/pkg/addons/seaweedfs/static/metadata.yaml b/pkg/addons/seaweedfs/static/metadata.yaml
diff --git a/tests/dryrun/install_test.go b/tests/dryrun/install_test.go
diff --git a/web/static.go b/web/static.go
diff --git a/web/static_test.go b/web/static_test.go

Original file line number	Diff line number	Diff line change
`@@ -43,7 +43,7 @@ main() {`
`43`	`43`	`exit 1`
`44`	`44`	`fi`
`45`	`45`
`46`		`- validate_no_pods_in_crashloop`
	`46`	`+ validate_all_pods_healthy`
`47`	`47`	`}`
`48`	`48`
`49`	`49`	`main "$@"`
Original file line number	Diff line number	Diff line change
`@@ -81,7 +81,7 @@ main() {`
`81`	`81`	`# scale the second deployment back down so that they aren't restored in the DR test`
`82`	`82`	`kubectl scale -n "$APP_NAMESPACE" deployment/second --replicas=0`
`83`	`83`
`84`		`- validate_no_pods_in_crashloop`
	`84`	`+ validate_all_pods_healthy`
`85`	`85`	`}`
`86`	`86`
`87`	`87`	`main "$@"`
Original file line number	Diff line number	Diff line change
`@@ -49,7 +49,7 @@ main() {`
`49`	`49`	`validate_data_dirs`
`50`	`50`	`fi`
`51`	`51`
`52`		`- validate_no_pods_in_crashloop`
	`52`	`+ validate_all_pods_healthy`
`53`	`53`	`}`
`54`	`54`
`55`	`55`	`main "$@"`
Original file line number	Diff line number	Diff line change
`@@ -61,7 +61,7 @@ main() {`
`61`	`61`	`exit 1`
`62`	`62`	`fi`
`63`	`63`
`64`		`- validate_no_pods_in_crashloop`
	`64`	`+ validate_all_pods_healthy`
`65`	`65`	`}`
`66`	`66`
`67`	`67`	`main "$@"`
Original file line number	Diff line number	Diff line change
`@@ -111,7 +111,7 @@ main() {`
`111`	`111`
`112`	`112`	`validate_data_dirs`
`113`	`113`
`114`		`- validate_no_pods_in_crashloop`
	`114`	`+ validate_all_pods_healthy`
`115`	`115`	`}`
`116`	`116`
`117`	`117`	`main "$@"`