cluster-cleaner: handle more edge-cases and force deletions on resources that are stuck (#4164)

# Summary

- brute-force if we are stuck: removing finalizers if they are stuck
after 10 seconds trying to delete the resource
- adding bunch more logging
- handle edge-cases e.g. some resources were deleted at a different
place

## Proof of Work

- running manually in openshift static cluster freed the cluster

## Checklist
- [ ] Have you linked a jira ticket and/or is the ticket in the title?
- [ ] Have you checked whether your jira ticket required DOCSP changes?
- [ ] Have you checked for release_note changes?

## Reminder (Please remove this when merging)
- Please try to Approve or Reject Changes the PR, keep PRs in review as
short as possible
- Our Short Guide for PRs:
[Link](REDACTED)
- Remember the following Communication Standards - use comment prefixes
for clarity:
  * **blocking**: Must be addressed before approval.
  * **follow-up**: Can be addressed in a later PR or ticket.
  * **q**: Clarifying question.
  * **nit**: Non-blocking suggestions.
  * **note**: Side-note, non-actionable. Example: Praise
  * --> no prefix is considered a question

Author: nammn

docker/cluster-cleaner/Chart.yaml

@@ -1,3 +1,3 @@
 name: cluster-cleaner
 description: The background cleaner
-version: 0.13
+version: 0.14

docker/cluster-cleaner/Makefile

@@ -1,4 +1,4 @@
-IMAGE_VERSION=0.13
+IMAGE_VERSION=0.14
 
 .PHONY: all
 all: build push install

docker/cluster-cleaner/scripts/clean-failed-namespaces.sh

@@ -1,5 +1,23 @@
 #!/usr/bin/env sh
 
+delete_resources_safely() {
+    resource_type="$1"
+    namespace="$2"
+
+    echo "Attempting normal deletion of $resource_type in $namespace..."
+    kubectl delete "${resource_type}" --all -n "${namespace}" --wait=true --timeout=10s || true
+
+    # Check if any resources are still stuck
+    resources=$(kubectl get "$resource_type" -n "${namespace}" --no-headers -o custom-columns=":metadata.name")
+
+    for resource in ${resources}; do
+        echo "${resource_type}/${resource} is still present, force deleting..."
+
+        kubectl patch "${resource_type}" "${resource}" -n "${namespace}" -p '{"metadata":{"finalizers":null}}' --type=merge || true
+        kubectl delete "${resource_type}" "${resource}" -n "${namespace}" --force --grace-period=0 || true
+    done
+}
+
 if [ -z ${DELETE_OLDER_THAN_AMOUNT+x} ] || [ -z ${DELETE_OLDER_THAN_UNIT+x} ]; then
     echo "Need to set both 'DELETE_OLDER_THAN_AMOUNT' and 'DELETE_OLDER_THAN_UNIT' environment variables."
     exit 1
@@ -11,20 +29,49 @@ if [ -z ${LABELS+x} ]; then
 fi
 
 echo "Deleting namespaces for evg tasks that are older than ${DELETE_OLDER_THAN_AMOUNT} ${DELETE_OLDER_THAN_UNIT} with label ${LABELS}"
+echo "Which are:"
+kubectl get namespace -l "${LABELS}" -o name
 for namespace in $(kubectl get namespace -l "${LABELS}" -o name); do
-    creation_time=$(kubectl get "${namespace}" -o jsonpath='{.metadata.creationTimestamp}')
+    creation_time=$(kubectl get "${namespace}" -o jsonpath='{.metadata.creationTimestamp}' 2>/dev/null || echo "")
 
-    if ! ./is_older_than.py "${creation_time}" "${DELETE_OLDER_THAN_AMOUNT}" "${DELETE_OLDER_THAN_UNIT}"; then
+    if [ -z "$creation_time" ]; then
+        echo "Namespace ${namespace} does not exist or has no creation timestamp, skipping."
         continue
     fi
 
     namespace_name=$(echo "${namespace}" | cut -d '/' -f 2)
 
-    csrs_in_namespace=$(kubectl get csr -o name | grep "${namespace_name}")
-    kubectl delete "${csrs_in_namespace}"
+    if ! ./is_older_than.py "${creation_time}" "${DELETE_OLDER_THAN_AMOUNT}" "${DELETE_OLDER_THAN_UNIT}"; then
+        echo "Skipping ${namespace_name}, not old enough."
+        continue
+    fi
+
+    echo "Deleting ${namespace_name}"
+
+    csrs_in_namespace=$(kubectl get csr -o name | grep "${namespace_name}" || true)
+    if [ -n "${csrs_in_namespace}" ]; then
+        kubectl delete "${csrs_in_namespace}"
+    fi
+
+    delete_resources_safely "mdb" "${namespace_name}"
+    delete_resources_safely "mdbu" "${namespace_name}"
+    delete_resources_safely "om" "${namespace_name}"
+
+    echo "Attempting to delete namespace: ${namespace_name}"
 
-    kubectl delete mdb --all -n "${namespace_name=}"
-    kubectl delete mdbu --all -n "${namespace_name=}"
-    kubectl delete om --all -n "{namespace_name=}"
-    kubectl delete "${namespace}"
+    if kubectl get namespace "${namespace_name}" >/dev/null 2>&1; then
+        kubectl delete namespace "${namespace_name}" --wait=true --timeout=10s || true
+    else
+        echo "Namespace ${namespace_name} not found, skipping deletion."
+    fi
+
+    if kubectl get namespace "${namespace_name}" >/dev/null 2>&1; then
+        echo "Namespace ${namespace_name} is still stuck, removing finalizers..."
+        kubectl patch namespace "${namespace_name}" -p '{"metadata":{"finalizers":null}}' --type=merge
+
+        echo "Force deleting namespace: ${namespace_name}"
+        kubectl delete namespace "${namespace_name}" --wait=true --timeout=30s
+    else
+        echo "Namespace ${namespace_name} deleted successfully."
+    fi
 done