Fix CI issues

* Fix destroy-cluster failing because of an unset var
* Improve debugging of cleanup-expired-clusters.sh

Author: Roming22

ci/images/ci-runner/hack/bin/cleanup-expired-clusters.sh

@@ -24,8 +24,10 @@ is_cluster_expired() {
 }
 
 destroy_expired_clusters() {
+    expired_clusters=()
     mapfile -t clusters < <(rosa list clusters --region "$REGION" | grep "ready" | awk '{print $2}')
 
+    echo "[$(date +"%Y/%m/%d %H:%M:%S")] Cluster count: ${#clusters[@]}"
     for cluster in "${clusters[@]}"; do
         if [[ "${EXCLUDE_CLUSTER[*]}" =~ $cluster ]]; then
             continue
@@ -34,13 +36,21 @@ destroy_expired_clusters() {
         is_expired=$(is_cluster_expired "$cluster")
 
         if [[ "$is_expired" == "true" ]]; then
-            export CLUSTER_NAME="$cluster" 
-            "$SCRIPT_DIR"/destroy-cluster.sh 
+            expired_clusters+=( "$cluster" )
         fi
     done
+
+    echo "[$(date +"%Y/%m/%d %H:%M:%S")] Expired cluster count: ${#expired_clusters[@]}"
+    count=0
+    for cluster in "${expired_clusters[@]}"; do
+        count=$(( count + 1 ))
+        echo "Destroying $cluster [$count/${#expired_clusters[@]}]"
+        export CLUSTER_NAME="$cluster"
+        "$SCRIPT_DIR"/destroy-cluster.sh
+    done
 }
 
 setx_off
 rosa login --token="$ROSA_TOKEN"
 setx_on
-destroy_expired_clusters
+destroy_expired_clusters

ci/images/ci-runner/hack/bin/destroy-cluster.sh

@@ -17,7 +17,7 @@ if [ -e "$failure_file" ]; then
     echo "Delete '$failure_file' within 15 minutes to keep the cluster alive for investigation."
     echo
     echo "KUBECONFIG:"
-    cat "$KUBECONFIG_DIR/$KUBECONFIG"
+    cat "$KUBECONFIG_DIR/kubeconfig"
     echo
     echo "Connect to the ci-runner with: kubectl exec -n default --stdin --tty ci-runner -- bash"
     echo
@@ -35,7 +35,7 @@ fi
 source "$SCRIPT_DIR/utils.sh"
 
 if [[ -n "$CLUSTER_NAME"  ]]; then
-    echo "Started to destroy cluster [$CLUSTER_NAME]..."
+    echo "[$(date +"%Y/%m/%d %H:%M:%S")] Started to destroy cluster [$CLUSTER_NAME]..."
 
     printf "Log in to your Red Hat account...\n" | indent 2
     setx_off
@@ -44,7 +44,8 @@ if [[ -n "$CLUSTER_NAME"  ]]; then
 
     rosa delete cluster --region "$REGION" --cluster="$CLUSTER_NAME" -y
     rosa logs uninstall --region "$REGION" --cluster="$CLUSTER_NAME" --watch
-
 else
     echo "No OCP cluster need to be destroyed."
 fi
+
+echo "[$(date +"%Y/%m/%d %H:%M:%S")] Done"