Merge pull request #89 from wunderio/feature/sts-dp-events

statefulset and deployment events for failed releases

Author: ArtisKrumins

cmd/ciReleaseDebugFailed.go

@@ -213,7 +213,7 @@ var ciReleaseDeployCmd = &cobra.Command{
 			SILTA_ENVIRONMENT_NAME='%s'
 			BRANCHNAME='%s'
 			NGINX_IMAGE_URL='%s'
-			CLUSTER_DOMAIN='%s'	
+			CLUSTER_DOMAIN='%s'
 			EXTRA_NOAUTHIPS='%s'
 			EXTRA_VPCNATIVE='%s'
 			EXTRA_CLUSTERTYPE='%s'
@@ -222,37 +222,16 @@ var ciReleaseDeployCmd = &cobra.Command{
 			EXTRA_HELM_FLAGS='%s'
 			DEPLOYMENT_TIMEOUT='%s'
 
-			# Detect pods in FAILED state
-			function show_failing_pods() {
-				echo ""
-				failed_pods=$(kubectl get pod -l "release=$RELEASE_NAME,cronjob!=true" -n "$NAMESPACE" -o custom-columns="POD:metadata.name,STATE:status.containerStatuses[*].ready" --no-headers | grep -E "<none>|false" | grep -Eo '^[^ ]+')
-				if [[ ! -z "$failed_pods" ]] ; then
-					echo "Failing pods:"
-					while IFS= read -r pod; do
-						echo "---- ${NAMESPACE} / ${pod} ----"
-						echo "* Events"
-						kubectl get events --field-selector involvedObject.name=${pod},type!=Normal --show-kind=true --ignore-not-found=true --namespace ${NAMESPACE}
-						echo ""
-						echo "* Logs"
-						containers=$(kubectl get pods "${pod}" --namespace "${NAMESPACE}" -o json | jq -r 'try .status | .containerStatuses[] | select(.ready == false).name')
-						if [[ ! -z "$containers" ]] ; then
-							for container in ${containers}; do
-								kubectl logs "${pod}" --prefix=true --since="${DEPLOYMENT_TIMEOUT}" --namespace "${NAMESPACE}" -c "${container}" 
-							done
-						else
-							echo "no logs found"
-						fi
-
-						echo "----"
-					done <<< "$failed_pods"
+			function deployment_failed() {
+				debug_failed_deployment
+				rm -f helm-output.log || true
+			}
 
-					false
-				else
-					true
-				fi
+			function debug_failed_deployment() {
+				silta ci release debug-failed --release-name="${RELEASE_NAME}" --namespace="${NAMESPACE}"
 			}
 
-			trap show_failing_pods ERR
+			trap deployment_failed ERR
 
 			helm upgrade --install "${RELEASE_NAME}" "${CHART_NAME}" \
 				--repo "${CHART_REPOSITORY}" \
@@ -298,7 +277,7 @@ var ciReleaseDeployCmd = &cobra.Command{
 			GIT_REPOSITORY_URL='%s'
 			GITAUTH_USERNAME='%s'
 			GITAUTH_PASSWORD='%s'
-			CLUSTER_DOMAIN='%s'	
+			CLUSTER_DOMAIN='%s'
 			NAMESPACE='%s'
 			EXTRA_NOAUTHIPS='%s'
 			EXTRA_VPCNATIVE='%s'
@@ -310,38 +289,16 @@ var ciReleaseDeployCmd = &cobra.Command{
 			DEPLOYMENT_TIMEOUT='%s'
 			DEPLOYMENT_TIMEOUT_SECONDS='%d'
 
-			# Detect pods in FAILED state
-			function show_failing_pods() {
-				echo ""
-				failed_pods=$(kubectl get pod -l "release=$RELEASE_NAME,cronjob!=true" -n "$NAMESPACE" -o custom-columns="POD:metadata.name,STATE:status.containerStatuses[*].ready" --no-headers | grep -E "<none>|false" | grep -Eo '^[^ ]+')
-				if [[ ! -z "$failed_pods" ]] ; then
-					echo "Failing pods:"
-					while IFS= read -r pod; do
-						echo "---- ${NAMESPACE} / ${pod} ----"
-						echo "* Events"
-						kubectl get events --field-selector involvedObject.name=${pod},type!=Normal --show-kind=true --ignore-not-found=true --namespace ${NAMESPACE}
-						echo ""
-						echo "* Logs"
-						containers=$(kubectl get pods "${pod}" --namespace "${NAMESPACE}" -o json | jq -r 'try .status | .containerStatuses[] | select(.ready == false).name')
-						if [[ ! -z "$containers" ]] ; then
-							for container in ${containers}; do
-								kubectl logs "${pod}" --prefix=true --since="${DEPLOYMENT_TIMEOUT}" --namespace "${NAMESPACE}" -c "${container}" 
-							done
-						else
-							echo "no logs found"
-						fi
-
-						echo "----"
-					done <<< "$failed_pods"
+			function deployment_failed() {
+				debug_failed_deployment
+				rm -f helm-output.log || true
+			}
 
-					false
-				else
-					true
-				fi
-				rm -f helm-output.log
+			function debug_failed_deployment() {
+				silta ci release debug-failed --release-name="${RELEASE_NAME}" --namespace="${NAMESPACE}"
 			}
 
-			trap show_failing_pods ERR
+			trap deployment_failed ERR
 
 			helm upgrade --install "${RELEASE_NAME}" "${CHART_NAME}" \
 				--repo "${CHART_REPOSITORY}" \
@@ -381,17 +338,17 @@ var ciReleaseDeployCmd = &cobra.Command{
 					# Helm command is complete.
 					if ! ps -p "$pid" > /dev/null; then
 						echo "Helm output:"
-						cat helm-output.log
+						cat helm-output.log || true
 						wait $pid
 						if grep -q "UPGRADE FAILED" helm-output.log ; then
-							show_failing_pods
+							debug_failed_deployment
 						fi
 						break
 					fi
 
 					if [ $TIME_WAITING -gt ${DEPLOYMENT_TIMEOUT_SECONDS} ]; then
 						echo "Timeout waiting for resources."
-						show_failing_pods
+						debug_failed_deployment
 						exit 1
 					fi
 
@@ -407,7 +364,7 @@ var ciReleaseDeployCmd = &cobra.Command{
 					echo "$statefulsets" | xargs -n 1 kubectl rollout status statefulset -n "$NAMESPACE" --timeout 5m
 				fi
 				kubectl get deployment -n "$NAMESPACE" -l "release=${RELEASE_NAME}" -o name | xargs -n 1 kubectl rollout status -n "$NAMESPACE" --timeout 5m
-				rm -f helm-output.log
+				rm -f helm-output.log || true
 				`,
 				releaseName, chartName, chartRepository, chartVersionOverride,
 				siltaEnvironmentName, branchname,
@@ -513,7 +470,7 @@ var ciReleaseDeployCmd = &cobra.Command{
 			GIT_REPOSITORY_URL='%s'
 			GITAUTH_USERNAME='%s'
 			GITAUTH_PASSWORD='%s'
-			CLUSTER_DOMAIN='%s'	
+			CLUSTER_DOMAIN='%s'
 			EXTRA_NOAUTHIPS='%s'
 			EXTRA_VPCNATIVE='%s'
 			EXTRA_CLUSTERTYPE='%s'
@@ -526,38 +483,16 @@ var ciReleaseDeployCmd = &cobra.Command{
 			DEPLOYMENT_TIMEOUT='%s'
 			DEPLOYMENT_TIMEOUT_SECONDS='%d'
 
-			# Detect pods in FAILED state
-			function show_failing_pods() {
-				echo ""
-				failed_pods=$(kubectl get pod -l "release=$RELEASE_NAME,cronjob!=true" -n "$NAMESPACE" -o custom-columns="POD:metadata.name,STATE:status.containerStatuses[*].ready" --no-headers | grep -E "<none>|false" | grep -Eo '^[^ ]+')
-				if [[ ! -z "$failed_pods" ]] ; then
-					echo "Failing pods:"
-					while IFS= read -r pod; do
-						echo "---- ${NAMESPACE} / ${pod} ----"
-						echo "* Events"
-						kubectl get events --field-selector involvedObject.name=${pod},type!=Normal --show-kind=true --ignore-not-found=true --namespace ${NAMESPACE}
-						echo ""
-						echo "* Logs"
-						containers=$(kubectl get pods "${pod}" --namespace "${NAMESPACE}" -o json | jq -r 'try .status | .containerStatuses[] | select(.ready == false).name')
-						if [[ ! -z "$containers" ]] ; then
-							for container in ${containers}; do
-								kubectl logs "${pod}" --prefix=true --since="${DEPLOYMENT_TIMEOUT}" --namespace "${NAMESPACE}" -c "${container}" 
-							done
-						else
-							echo "no logs found"
-						fi
-
-						echo "----"
-					done <<< "$failed_pods"
+			function deployment_failed() {
+				debug_failed_deployment
+				rm -f helm-output.log || true
+			}
 
-					false
-				else
-					true
-				fi
-				rm -f helm-output.log
+			function debug_failed_deployment() {
+				silta ci release debug-failed --release-name="${RELEASE_NAME}" --namespace="${NAMESPACE}"
 			}
 
-			trap show_failing_pods ERR
+			trap deployment_failed ERR
 
 			helm upgrade --install "${RELEASE_NAME}" "${CHART_NAME}" \
 				--repo "${CHART_REPOSITORY}" \
@@ -601,17 +536,17 @@ var ciReleaseDeployCmd = &cobra.Command{
 					# Helm command is complete.
 					if ! ps -p "$pid" > /dev/null; then
 						echo "Helm output:"
-						cat helm-output.log
+						cat helm-output.log || true
 						wait $pid
 						if grep -q "UPGRADE FAILED" helm-output.log ; then
-							show_failing_pods
+							debug_failed_deployment
 						fi	
 						break
 					fi
 
 					if [ $TIME_WAITING -gt ${DEPLOYMENT_TIMEOUT_SECONDS} ]; then
 						echo "Timeout waiting for resources."
-						show_failing_pods
+						debug_failed_deployment
 						exit 1
 					fi
 
@@ -627,7 +562,7 @@ var ciReleaseDeployCmd = &cobra.Command{
 					echo "$statefulsets" | xargs -n 1 kubectl rollout status statefulset -n "$NAMESPACE" --timeout 5m
 				fi
 				kubectl get deployment -n "$NAMESPACE" -l "release=${RELEASE_NAME}" -o name | xargs -n 1 kubectl rollout status -n "$NAMESPACE" --timeout 5m
-				rm -f helm-output.log
+				rm -f helm-output.log || true
 				`,
 				releaseName, chartName, chartRepository, chartVersionOverride,
 				siltaEnvironmentName, branchname,

cmd/ciReleaseDeploy.go

@@ -23,8 +23,9 @@ silta ci release [flags]
 
 * [silta ci](silta_ci.md)	 - Silta CI Commands
 * [silta ci release clean-failed](silta_ci_release_clean-failed.md)	 - Clean failed releases
+* [silta ci release debug-failed](silta_ci_release_debug-failed.md)	 - Debug failed deployment resources
 * [silta ci release delete](silta_ci_release_delete.md)	 - Delete a release
-* [silta ci release delete-resources](silta_ci_release_delete-resources.md)	 - Delete orphhaned release resources
+* [silta ci release delete-resources](silta_ci_release_delete-resources.md)	 - Delete orphaned release resources
 * [silta ci release deploy](silta_ci_release_deploy.md)	 - Deploy release
 * [silta ci release diff](silta_ci_release_diff.md)	 - Diff release resources
 * [silta ci release environmentname](silta_ci_release_environmentname.md)	 - Return environment name

docs/silta_ci_release.md

@@ -0,0 +1,37 @@
+## silta ci release debug-failed
+
+Debug failed deployment resources
+
+### Synopsis
+
+Debug failed deployment by checking:
+- OOMKilled containers
+- Failed pods with their events and logs
+- Not-ready statefulsets with their events
+- Not-ready deployments with their events
+
+This command is typically called when a deployment fails to help diagnose the issue.
+
+```
+silta ci release debug-failed [flags]
+```
+
+### Options
+
+```
+  -h, --help                  help for debug-failed
+      --namespace string      Namespace
+      --release-name string   Release name
+```
+
+### Options inherited from parent commands
+
+```
+      --debug     Print variables, do not execute external commands, rather print them
+      --use-env   Use environment variables for value assignment (default true)
+```
+
+### SEE ALSO
+
+* [silta ci release](silta_ci_release.md)	 - CI release related commands
+

docs/silta_ci_release_debug-failed.md

@@ -1,6 +1,12 @@
 ## silta ci release delete-resources
 
-Delete orphhaned release resources
+Delete orphaned release resources
+
+### Synopsis
+
+Deletes release resources based on labels ("release", "app.kubernetes.io/instance" and "app=<release-name>-es" (for Elasticsearch storage))
+		This command can be used to clean up resources when helm release configmaps are absent.
+	
 
 ```
 silta ci release delete-resources [flags]

docs/silta_ci_release_delete-resources.md

@@ -98,6 +98,7 @@ require (
 	github.com/pelletier/go-toml/v2 v2.2.4 // indirect
 	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
 	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/prometheus/client_model v0.6.2 // indirect
 	github.com/prometheus/common v0.65.0 // indirect
 	github.com/prometheus/procfs v0.16.1 // indirect

go.mod

@@ -18,24 +18,9 @@ import (
 // UninstallRelease removes a Helm release and related resources
 func UninstallHelmRelease(kubernetesClient *kubernetes.Clientset, helmClient *helmAction.Configuration, namespace string, releaseName string, deletePVCs bool) error {
 
-	// Do not bail when release removal fails, remove related resources anyway.
 	log.Printf("Removing release: %s", releaseName)
-	uninstall := helmAction.NewUninstall(helmClient)
-	uninstall.KeepHistory = false // Remove release secrets as well
-	uninstall.DisableHooks = false
-	uninstall.Timeout = 300 // seconds, adjust as needed
-	uninstall.Wait = true   // Wait for resources to be deleted
 
-	resp, err := uninstall.Run(releaseName)
-	if err != nil {
-		log.Printf("Failed to remove helm release: %s", err)
-	} else {
-		if resp != nil && resp.Info != "" {
-			log.Printf("Helm uninstall info: %s", resp.Info)
-		}
-	}
-
-	// Delete related jobs
+	// Delete related jobs (mainly post-release jobs)
 	selectorLabels := []string{
 		"release",
 		"app.kubernetes.io/instance",
@@ -53,6 +38,22 @@ func UninstallHelmRelease(kubernetesClient *kubernetes.Clientset, helmClient *he
 		}
 	}
 
+	// Delete helm release
+	uninstall := helmAction.NewUninstall(helmClient)
+	uninstall.KeepHistory = false // Remove release secrets as well
+	uninstall.DisableHooks = false
+	uninstall.Timeout = 300 * time.Second // seconds, adjust as needed
+	uninstall.Wait = true                 // Wait for resources to be deleted
+
+	resp, err := uninstall.Run(releaseName)
+	if err != nil {
+		log.Printf("Failed to remove helm release: %s", err)
+	} else {
+		if resp != nil && resp.Info != "" {
+			log.Printf("Helm uninstall info: %s", resp.Info)
+		}
+	}
+
 	if deletePVCs {
 
 		// Find and remove related PVC's by release name label

internal/common/ciReleaseFunctions.go

@@ -6,6 +6,8 @@ import (
 	"os/exec"
 	"strings"
 	"testing"
+
+	"github.com/pmezard/go-difflib/difflib"
 )
 
 var cliBinaryName = "./silta"
@@ -38,13 +40,31 @@ func CliExecTest(t *testing.T, command string, environment []string, testString
 		} else {
 			t.Logf("Error: %s", err.String())
 			t.Errorf("Expected :\n '%s' \n Received: \n '%s'\n'%s'", testString, out.String(), err.String())
+			d := diffOutput(t, testString, out.String())
+			t.Errorf("Diff:\n %s", d)
 		}
 
 	} else {
 		if strings.Contains(out.String(), testString) || strings.Contains(err.String(), testString) {
 		} else {
 			t.Logf("Error: %s", err.String())
 			t.Errorf("Expected :\n '%s' \n Received: \n '%s'\n'%s'", testString, out.String(), err.String())
+			d := diffOutput(t, testString, out.String())
+			t.Errorf("Diff:\n %s", d)
 		}
 	}
 }
+
+func diffOutput(t *testing.T, expected string, received string) string {
+	diff := difflib.UnifiedDiff{
+		A:        difflib.SplitLines(expected),
+		B:        difflib.SplitLines(received),
+		FromFile: "Expected",
+		FromDate: "",
+		ToFile:   "Received",
+		ToDate:   "",
+		Context:  1,
+	}
+	text, _ := difflib.GetUnifiedDiffString(diff)
+	return text
+}