restore genericDelete in cleanup.sh

Author: Lily He

src/integration-tests/bash/cleanup.sh

@@ -27,10 +27,14 @@
 #
 # The test runs in 4 phases:
 #
-#   Phase 1:  Delete domain resources with label 'weblogic.domainUID'.
+#   Phase 1:  Delete test kubernetes artifacts with labels.
 #
-#   Phase 2:  Delete wls operator with lable 'weblogic.operatorName' and
-#             delete voyager controller.
+#   Phase 2: Wait 15 seconds to see if stage 1 succeeded, and
+#             if not, repeatedly search for all test related kubectl
+#             artifacts and try delete them directly for up to 60 more
+#             seconds.  This phase has no dependency on the
+#             previous test run's yaml files.  It makes no
+#             attempt to delete artifacts in a particular order.
 #
 #   Phase 3:  Use a kubernetes job to delete the PV directories
 #             on the kubernetes cluster.
@@ -57,6 +61,7 @@ function fail {
 
 #!/bin/bash
 #
+# Usage:
 # getResWithLabel outfilename
 #
 function getResWithLabel {
@@ -75,15 +80,17 @@ function getResWithLabel {
 }
 
 #
+# Usage:
 # deleteResWithLabel outputfile
 #
-function deleteResWithLabel {
+function deleteWithOneLabel {
   echo @@ Delete resources with label $LABEL_SELECTOR.
   # clean the output file first
   if [ -e $1 ]; then
     rm $1
   fi
 
+  echo @@ Deleting resources with label $LABEL_SELECTOR.
   getResWithLabel $1
   # delete namespaced types
   cat $1 | awk '{ print $4 }' | grep -v "^$" | sort -u | while read line; do
@@ -98,19 +105,23 @@ function deleteResWithLabel {
 
   echo "@@ Waiting for pods to stop running."
   local total=0
-  for count in {1..100}; do
-    pods=($(kubectl get pods --all-namespaces -l weblogic.domainUID -o jsonpath='{range .items[*]}{.metadata.name} {end}'))
+  local mstart=`date +%s`
+  local mnow=mstart
+  local maxwaitsecs=60
+  while [ $((mnow - mstart)) -lt $maxwaitsecs ]; do
+    pods=($(kubectl get pods --all-namespaces -l $LABEL_SELECTOR -o jsonpath='{range .items[*]}{.metadata.name} {end}'))
     total=${#pods[*]}
     if [ $total -eq 0 ] ; then
         break
     else
-      echo "@@ There are still $total running pods with label $LABEL_SELECTOR."
+      echo "@@ There are $total running pods with label $LABEL_SELECTOR."
     fi
     sleep 3
+    mnow=`date +%s`
   done
 
   if [ $total -gt 0 ]; then
-    echo "Warning: after waiting 300 seconds, there are still $total running pods with label $LABEL_SELECTOR."
+    echo "Warning: after waiting $maxwaitsecs seconds, there are still $total running pods with label $LABEL_SELECTOR."
   fi
 }
 
@@ -122,6 +133,7 @@ function deleteVoyagerController {
 }
 
 #
+# Usage:
 # deleteNamespaces outputfile
 #
 function deleteNamespaces {
@@ -132,46 +144,179 @@ function deleteNamespaces {
   done
 
 }
-echo @@ Starting cleanup.
-script="${BASH_SOURCE[0]}"
-scriptDir="$( cd "$(dirname "${script}")" > /dev/null 2>&1 ; pwd -P)"
 
-echo "@@ RESULT_ROOT=$RESULT_ROOT TMP_DIR=$TMP_DIR RESULT_DIR=$RESULT_DIR PROJECT_ROOT=$PROJECT_ROOT"
+function deleteWithLabels {
+  NAMESPACED_TYPES="pod,job,deploy,rs,service,pvc,ingress,cm,serviceaccount,role,rolebinding,secret"
 
-mkdir -p $TMP_DIR || fail No permision to create directory $TMP_DIR
+  HANDLE_VOYAGER="false"
+  VOYAGER_ING_NAME="ingresses.voyager.appscode.com"
+  if [ `kubectl get crd $VOYAGER_ING_NAME --ignore-not-found | grep $VOYAGER_ING_NAME | wc -l` = 1 ]; then
+    NAMESPACED_TYPES="$VOYAGER_ING_NAME,$NAMESPACED_TYPES"
+    HANDLE_VOYAGER="true"
+  fi
 
-NAMESPACED_TYPES="pod,job,deploy,rs,service,pvc,ingress,cm,serviceaccount,role,rolebinding,secret"
+  DOMAIN_CRD="domains.weblogic.oracle"
+  if [ `kubectl get crd $DOMAIN_CRD --ignore-not-found | grep $DOMAIN_CRD | wc -l` = 1 ]; then
+    NAMESPACED_TYPES="$DOMAIN_CRD,$NAMESPACED_TYPES"
+  fi
 
-HANDLE_VOYAGER="false"
-VOYAGER_ING_NAME="ingresses.voyager.appscode.com"
-if [ `kubectl get crd $VOYAGER_ING_NAME --ignore-not-found | grep $VOYAGER_ING_NAME | wc -l` = 1 ]; then
-  NAMESPACED_TYPES="$VOYAGER_ING_NAME,$NAMESPACED_TYPES"
-  HANDLE_VOYAGER="true"
-fi
+  NOT_NAMESPACED_TYPES="pv,crd,clusterroles,clusterrolebindings"
 
-DOMAIN_CRD="domains.weblogic.oracle"
-if [ `kubectl get crd $DOMAIN_CRD --ignore-not-found | grep $DOMAIN_CRD | wc -l` = 1 ]; then
-  NAMESPACED_TYPES="$DOMAIN_CRD,$NAMESPACED_TYPES"
-fi
+  tempfile="/tmp/$(basename $0).tmp.$$"  # == /tmp/[script-file-name].tmp.[pid]
 
-NOT_NAMESPACED_TYPES="pv,crd,clusterroles,clusterrolebindings"
+  echo @@ Deleting domain resources.
+  LABEL_SELECTOR="weblogic.domainUID"
+  deleteWithOneLabel "$tempfile-0"
 
-tempfile="/tmp/$(basename $0).tmp.$$"  # == /tmp/[script-file-name].tmp.[pid]
+  echo @@ Deleting wls operator resources.
+  LABEL_SELECTOR="weblogic.operatorName"
+  deleteWithOneLabel "$tempfile-1"
 
-echo @@ Deleting domain resources.
-LABEL_SELECTOR="weblogic.domainUID"
-deleteResWithLabel "$tempfile-0"
-deleteNamespaces "$tempfile-0"
+  deleteNamespaces "$tempfile-0"
+  deleteNamespaces "$tempfile-1"
 
-echo @@ Deleting wls operator resources.
-LABEL_SELECTOR="weblogic.operatorName"
-deleteResWithLabel "$tempfile-1"
-deleteNamespaces "$tempfile-1"
+  echo @@ Deleting voyager controller.
+  if [ "$HANDLE_VOYAGER" = "true" ]; then
+    deleteVoyagerController
+  fi
+}
 
-echo @@ Deleting voyager controller.
-if [ "$HANDLE_VOYAGER" = "true" ]; then
-  deleteVoyagerController
-fi
+# function genericDelete
+#
+#   This function is a 'generic kubernetes delete' that takes three arguments:
+#
+#     arg1:  Comma separated list of types of kubernetes namespaced types to search/delete.
+#            example: "all,cm,pvc,ns,roles,rolebindings,secrets"
+#
+#     arg2:  Comma separated list of types of kubernetes non-namespaced types to search/delete.
+#            example: "crd,pv,clusterroles,clusterrolebindings"
+#
+#     arg3:  '|' (pipe) separated list of keywords.
+#            Artifacts with a label or name that contains one
+#            or more of the keywords are delete candidates.
+#            example:  "logstash|kibana|elastisearch|weblogic|elk|domain"
+#
+#   It runs in two stages:
+#     In the first, wait to see if artifacts delete on their own.
+#     In the second, try to delete any leftovers.
+#
+function genericDelete {
+
+  for iteration in first second; do
+    # In the first iteration, we wait to see if artifacts delete.
+    # in the second iteration, we try to delete any leftovers.
+
+    if [ "$iteration" = "first" ]; then
+      local maxwaitsecs=15
+    else
+      local maxwaitsecs=60
+    fi
+
+    echo "@@ Waiting up to $maxwaitsecs seconds for ${1:?} and ${2:?} artifacts that contain string ${3:?} to delete."
+
+    local artcount_no
+    local artcount_yes
+    local artcount_total
+    local resfile_no
+    local resfile_yes
+
+    local mstart=`date +%s`
+
+    while : ; do
+      resfile_no="$TMP_DIR/kinv_filtered_nonamespace.out.tmp"
+      resfile_yes="$TMP_DIR/kinv_filtered_yesnamespace.out.tmp"
+
+      # leftover namespaced artifacts
+      kubectl get $1 \
+          -o=jsonpath='{range .items[*]}{.metadata.namespace}{" "}{.kind}{"/"}{.metadata.name}{"\n"}{end}' \
+          --all-namespaces=true 2>&1 \
+          | egrep -e "($3)" | sort > $resfile_yes 2>&1
+      artcount_yes="`cat $resfile_yes | wc -l`"
+
+      # leftover non-namespaced artifacts
+      kubectl get $2 \
+          -o=jsonpath='{range .items[*]}{.kind}{"/"}{.metadata.name}{"\n"}{end}' \
+          --all-namespaces=true 2>&1 \
+          | egrep -e "($3)" | sort > $resfile_no 2>&1
+      artcount_no="`cat $resfile_no | wc -l`"
+
+      artcount_total=$((artcount_yes + artcount_no))
+
+      mnow=`date +%s`
+
+      if [ $((artcount_total)) -eq 0 ]; then
+        echo "@@ No artifacts found."
+        return 0
+      fi
+
+      if [ "$iteration" = "first" ]; then
+        # in the first iteration we just wait to see if artifacts go away on there own
+
+        echo "@@ Waiting for $artcount_total artifacts to delete.  Wait time $((mnow - mstart)) seconds (max=$maxwaitsecs).  Waiting for:"
+
+        cat $resfile_yes | awk '{ print "n=" $1 " " $2 }'
+        cat $resfile_no | awk '{ print $1 }'
+
+      else
+        # in the second thirty seconds we try to delete remaining artifacts
+
+        echo "@@ Trying to delete ${artcount_total} leftover artifacts, including ${artcount_yes} namespaced artifacts and ${artcount_no} non-namespaced artifacts, wait time $((mnow - mstart)) seconds (max=$maxwaitsecs)."
+
+        if [ ${artcount_yes} -gt 0 ]; then
+          cat "$resfile_yes" | while read line; do
+            local args="`echo \"$line\" | awk '{ print "-n " $1 " delete " $2 " --ignore-not-found" }'`"
+            echo "kubectl $args"
+            kubectl $args
+          done
+        fi
+
+        if [ ${artcount_no} -gt 0 ]; then
+          cat "$resfile_no" | while read line; do
+            echo "kubectl delete $line --ignore-not-found"
+            kubectl delete $line --ignore-not-found
+          done
+        fi
+
+      fi
+
+      if [ $((mnow - mstart)) -gt $((maxwaitsecs)) ]; then
+        if [ "$iteration" = "first" ]; then
+          echo "@@ Warning:  ${maxwaitsecs} seconds reached.   Will try deleting unexpected resources via kubectl delete."
+        else
+          echo "@@ Error:  ${maxwaitsecs} seconds reached and possibly ${artcount_total} artifacts remaining.  Giving up."
+        fi
+        break
+      fi
+
+      sleep 5
+    done
+  done
+  return 1
+}
+
+echo @@ Starting cleanup.
+script="${BASH_SOURCE[0]}"
+scriptDir="$( cd "$(dirname "${script}")" > /dev/null 2>&1 ; pwd -P)"
+
+echo "@@ RESULT_ROOT=$RESULT_ROOT TMP_DIR=$TMP_DIR RESULT_DIR=$RESULT_DIR PROJECT_ROOT=$PROJECT_ROOT"
+
+mkdir -p $TMP_DIR || fail No permision to create directory $TMP_DIR
+
+# first, try to delete with labels since the conversion is that all created resources need to
+# have the proper label(s)
+echo @@ Starting deleteWithLabels
+deleteWithLabels
+
+# second, try a generic delete in case there are some leftover resources, this runs in two phases:
+#   phase 1:  wait to see if artifacts dissappear naturally due to the above orderlyDelete
+#   phase 2:  kubectl delete left over artifacts
+# arguments
+#   arg1 - namespaced kubernetes artifacts
+#   arg2 - non-namespaced artifacts
+#   arg3 - keywords in deletable artificats
+echo @@ Starting genericDelete
+genericDelete "all,cm,pvc,roles,rolebindings,serviceaccount,secrets" "crd,pv,ns,clusterroles,clusterrolebindings" "logstash|kibana|elastisearch|weblogic|elk|domain|traefik|voyager"
+SUCCESS="$?"
 
 # Delete pv directories using a job (/scratch maps to PV_ROOT on the k8s cluster machines).