Fix: CNF uninstallation resource discovery

Refs: #2310
- Original implementation of wait_for_deployment_uninstallation utilized
  label discovery to find descendant resources of some core resource,
  this introduced errors as labels werent a sufficient identifier. To
  resolve this issue we now search for descendants through UIDs and
  ownerRef.
- Utility functions to retrieve UIDs and recursively find descendants
  were introduced in the KubectlClient.Get module
- Minor refactors to existing code.
- constant WORKLOAD_RESOURCES was extended to include job and cronjob
  since they are also workload resources.
- spec test to verify that failing uninstallation gets caught was tagged
  incorrectly, resulting in it being skipped in actions (this discovery
  prompted some changes to it)

Signed-off-by: svteb <slavo.valko@tietoevry.com>

Author: svteb

spec/setup_spec.cr

@@ -221,11 +221,11 @@ describe "Installation" do
     end
   end
 
-  it "cnf_uninstall should fail for a stuck manifest deployment" do
+  it "cnf_uninstall should fail for a stuck manifest deployment", tags: ["cnf_installation"] do
     result = ShellCmd.cnf_install("cnf-path=sample-cnfs/sample_stuck_finalizer/cnf-testsuite.yml")
     result[:status].success?.should be_true
 
-    result = ShellCmd.cnf_uninstall(timeout: 60, expect_failure: true)
+    result = ShellCmd.cnf_uninstall(timeout: 30, expect_failure: true)
     (/Some resources of "stuck_finalizer" did not finish deleting within the timeout/ =~ result[:output]).should_not be_nil
   ensure
     # Patch out finalizer
@@ -241,20 +241,21 @@ describe "Installation" do
     (/CNF uninstallation skipped/ =~ result[:output]).should_not be_nil
   end
 
-  it "cnf_uninstall should fail for a stuck helm deployment" do
+  it "cnf_uninstall should fail for a stuck helm deployment", tags: ["cnf_installation"] do
     result = ShellCmd.cnf_install("cnf-path=sample-cnfs/sample_stuck_helm_deployment/")
     result[:status].success?.should be_true
 
     # Inject finalizer into Deployment pod template
     KubectlClient::Utils.patch(
-      "deployment",
-      "stuck-nginx",
+      "pod",
+      nil,
       "cnf-default",
-      "json",
-      "[{\"op\":\"add\",\"path\":\"/spec/template/metadata/finalizers\",\"value\":[\"example.com/stuck\"]}]"
+      "merge",
+      %({"metadata":{"finalizers":["example.com/stuck"]}}),
+      { "app.kubernetes.io/instance" => "stuck-nginx" }
     )
 
-    result = ShellCmd.cnf_uninstall(timeout: 60, expect_failure: true)
+    result = ShellCmd.cnf_uninstall(timeout: 30, expect_failure: true)
     (/Some resources of "stuck-nginx" did not finish deleting within the timeout/ =~ result[:output]).should_not be_nil
   ensure
     # Patch out finalizer

src/modules/kubectl_client/kubectl_client.cr

@@ -11,12 +11,17 @@ module KubectlClient
 
   alias CMDResult = NamedTuple(status: Process::Status, output: String, error: String)
   alias BackgroundCMDResult = NamedTuple(process: Process, output: String, error: String)
-
-  WORKLOAD_RESOURCES = {deployment:      "Deployment",
-                        pod:             "Pod",
-                        replicaset:      "ReplicaSet",
-                        statefulset:     "StatefulSet",
-                        daemonset:       "DaemonSet"}
+  alias ResourceDescendant = NamedTuple(kind: String, name: String, namespace: String?, uid: String )
+
+  WORKLOAD_RESOURCES = {
+    deployment:  "Deployment",
+    pod:         "Pod",
+    replicaset:  "ReplicaSet",
+    statefulset: "StatefulSet",
+    daemonset:   "DaemonSet",
+    job:         "Job",
+    cronjob:     "CronJob"
+  }
 
   module ShellCMD
     # logger should have method name (any other scopes, if necessary) that is calling attached using .for() method.

src/modules/kubectl_client/modules/get.cr

@@ -546,5 +546,71 @@ module KubectlClient
 
       runtimes.uniq
     end
+
+    def self.resource_uid(kind : String, resource_name : String, namespace : String? = nil) : String?
+      logger = @@logger.for("resource_uid")
+      logger.debug { "Get uid of #{kind}/#{resource_name} in #{namespace || "default"}" }
+
+      begin
+        resource_json = resource(kind, resource_name, namespace)
+        resource_json.dig?("metadata", "uid").try &.as_s?
+      rescue KubectlClient::ShellCMD::NotFoundError
+        nil
+      end
+    end
+
+    def self.resources_by_owner_uid(kind : String, owner_uid : String, namespace : String? = nil) : Array(JSON::Any)
+      logger = @@logger.for("resources_by_owner_uid")
+      logger.debug { "Looking for #{kind} owned by #{owner_uid} in #{namespace || "all namespaces"}" }
+
+      resources_json = resource(kind, namespace: namespace, all_namespaces: namespace.nil?)
+      items = (resources_json.dig?("items").try &.as_a?) || [] of JSON::Any
+
+      matched = items.select do |item|
+        refs = (item.dig?("metadata", "ownerReferences").try &.as_a?) || [] of JSON::Any
+        refs.any? { |ref| ref["uid"].as_s == owner_uid }
+      end
+
+      matched
+    end
+
+    # find all descendant uids (BFS)
+    def self.descendants(
+      root_kind      : String,
+      root_name      : String,
+      root_namespace : String? = nil
+    ) : Array(ResourceDescendant)
+      seen_uids = Set(String).new
+      queue     = [] of ResourceDescendant
+      results   = [] of ResourceDescendant
+
+      queue << { kind: root_kind, name: root_name, namespace: root_namespace, uid: "" }
+
+      while current = queue.shift?
+        kind      = current[:kind]
+        name      = current[:name]
+        namespace = current[:namespace]
+
+        uid = resource_uid(kind, name, namespace)
+        next unless uid
+
+        if seen_uids.includes?(uid)
+          next
+        end
+
+        seen_uids.add(uid)
+        results << { kind: kind, name: name, namespace: namespace, uid: uid }
+
+        # enqueue its direct children
+        WORKLOAD_RESOURCES.values.each do |child_kind|
+          resources_by_owner_uid(child_kind, uid, namespace).each do |child|
+            child_name = child.dig("metadata", "name").as_s
+            queue << { kind: child_kind, name: child_name, namespace: namespace, uid: "" }
+          end
+        end
+      end
+
+      results
+    end
   end
 end

src/modules/kubectl_client/modules/wait.cr

@@ -143,67 +143,80 @@ module KubectlClient
     def self.resource_wait_for_uninstall(
       kind          : String,
       resource_name : String,
-      labels        : Hash(String, String) = {} of String => String,
-      namespace     : String? = "default",
-      wait_count    : Int32   = GENERIC_OPERATION_TIMEOUT
+      namespace     : String?                   = "default",
+      wait_count    : Int32                     = GENERIC_OPERATION_TIMEOUT,
+      descendants   : Array(ResourceDescendant) = [] of ResourceDescendant
     ) : Bool
-      logger = @@logger.for("resource_wait_for_uninstall")
-      logger.info { "Waiting for resource #{kind}/#{resource_name} to uninstall" }
-    
-      seconds_count = 0
-      deleted_once = false
-    
-      # Initial fetch (marks deleted_once if missing up front)
-      resource =
-        begin
-          KubectlClient::Get.resource(kind, resource_name, namespace)
-        rescue KubectlClient::ShellCMD::NotFoundError
-          deleted_once = true
-          EMPTY_JSON
-        end
-    
-      until seconds_count > wait_count
-        if seconds_count % RESOURCE_WAIT_LOG_INTERVAL == 0
-          logger.info { "seconds elapsed while waiting: #{seconds_count}" }
+      logger     = @@logger.for("resource_wait_for_uninstall")
+      logger.info { "Waiting up to #{wait_count}s for #{kind}/#{resource_name} to uninstall" }
+
+      # Make a mutable copy of the descendant list
+      pending    = descendants.dup
+      seconds    = 0
+      root_alive = true
+
+      while seconds <= wait_count
+        if (seconds % RESOURCE_WAIT_LOG_INTERVAL).zero?
+          logger.info { "#{seconds}s elapsed, pending descendants=#{pending.size}" }
         end
-    
-        # Only call Get.resource until we see it NotFound
-        present = if !deleted_once
+
+        # Check whether the root resource still exists
+        if root_alive
           begin
             KubectlClient::Get.resource(kind, resource_name, namespace)
-            true
           rescue KubectlClient::ShellCMD::NotFoundError
-            deleted_once = true
-            false
+            root_alive = false
+            logger.info { "Root resource #{kind}/#{resource_name} is gone" }
           end
-        else
-          false
         end
-    
-        # Once parent is gone, look for any leftover pods
-        pods = if !present
-          all_pods = KubectlClient::Get.resource("pods", nil, namespace).dig("items").as_a
-          KubectlClient::Get.pods_by_labels(all_pods, labels)
-        elsif resource != EMPTY_JSON
-          begin
-            KubectlClient::Get.pods_by_resource_labels(resource, namespace)
-          rescue KubectlClient::ShellCMD::NotFoundError
-            [] of JSON::Any
+
+        # Once the root is gone, prune descendants by UID
+        unless root_alive
+          pending.clone.each do |descendant|
+            begin
+              data = KubectlClient::Get.resource(
+                descendant[:kind],
+                nil,
+                descendant[:namespace],
+                descendant[:namespace].nil?,
+              )
+
+              items = data["items"].as_a? || [] of JSON::Any
+              exists = items.any? do |obj|
+                obj["metadata"]["uid"].as_s == descendant[:uid]
+              end
+            rescue KubectlClient::ShellCMD::NotFoundError
+              exists = false
+            end
+
+            unless exists
+              pending.delete(descendant)
+              logger.info { "Descendant #{descendant[:kind]}/#{descendant[:name]} (uid=#{descendant[:uid]}) deleted, #{pending.size} remaining" }
+            end
+          end
+
+          if pending.empty?
+            logger.info { "#{kind}/#{resource_name} and all descendants are deleted" }
+            return true
           end
-        else
-          [] of JSON::Any
-        end
-    
-        if !present && pods.empty?
-          logger.info { "#{kind}/#{resource_name} fully deleted" }
-          return true
         end
-    
-        sleep 1.second
-        seconds_count += 1
+
+        sleep 1
+        seconds += 1
       end
-    
-      logger.warn { "#{kind}/#{resource_name} still present after #{wait_count}s" }
+
+      # If we reach here, we timed out
+      leftovers = [] of String
+      leftovers << "root=#{kind}/#{resource_name}" if root_alive
+
+      unless pending.empty?
+        details = pending.map do |desc|
+          "#{desc[:kind]}/#{desc[:name]}(namespace=#{desc[:namespace]})"
+        end.join(", ")
+        leftovers << "pending=#{details}"
+      end
+
+      logger.warn { "Timeout after #{wait_count}s; still present: #{leftovers.join(" & ")}" }
       false
     end