Merge pull request #164 from oracle/int-test-leasing-2

Int test leasing 2

Author: rjeberhard

src/integration-tests/bash/cleanup.sh

@@ -16,6 +16,9 @@
 #   PV_ROOT      The root directory on the kubernetes cluster
 #                used for persistent volumes.
 #
+#   LEASE_ID     Set this if you want cleanup to release the 
+#                given lease on a failure.
+#
 # See 'run.sh' for a detailed description of RESULT_ROOT and PV_ROOT.
 #
 # --------------------
@@ -40,6 +43,9 @@
 #
 #   Phase 4:  Delete the local test output directory.
 #
+#   Phase 5:  If we own a lease, then release it on a failure
+#             see LEASE_ID above.
+#
 
 DOMAINS=(domain1 domain2 domain3 domain4 domain5)
 DOMAIN_NAMESPACES=(default default test1 test2 default)
@@ -103,11 +109,17 @@ function genericDelete {
       resfile_yes="$TMP_DIR/kinv_filtered_yesnamespace.out.tmp"
 
       # leftover namespaced artifacts
-      kubectl get $1 --show-labels=true --all-namespaces=true 2>&1 | egrep -e "($3)" | awk '{ print $1 " " $2 }' | sort > $resfile_yes 2>&1
+      kubectl get $1 \
+          -o=jsonpath='{range .items[*]}{.metadata.namespace}{" "}{.kind}{"/"}{.metadata.name}{"\n"}{end}' \
+          --all-namespaces=true 2>&1 \
+          | egrep -e "($3)" | sort > $resfile_yes 2>&1
       artcount_yes="`cat $resfile_yes | wc -l`"
 
       # leftover non-namespaced artifacts
-      kubectl get $2 --show-labels=true --all-namespaces=true 2>&1 | egrep -e "($3)" | awk '{ print $1 }' | sort > $resfile_no 2>&1
+      kubectl get $2 \
+          -o=jsonpath='{range .items[*]}{.kind}{"/"}{.metadata.name}{"\n"}{end}' \
+          --all-namespaces=true 2>&1 \
+          | egrep -e "($3)" | sort > $resfile_no 2>&1
       artcount_no="`cat $resfile_no | wc -l`"
 
       artcount_total=$((artcount_yes + artcount_no))
@@ -334,6 +346,18 @@ rm -f /tmp/test_suite.*
 
 # Bye
 
+if [ ! "$LEASE_ID" = "" ] && [ ! "$SUCCESS" = "0" ]; then
+  # release the lease if we own it
+  ${SCRIPTPATH}/lease.sh -d "$LEASE_ID" > /tmp/release_lease.out 2>&1
+  if [ "$?" = "0" ]; then
+    echo @@ Lease released.
+  else
+    echo @@ Lease could not be released:
+    cat /tmp/release_lease.out
+  fi
+  rm -f /tmp/release_lease.out
+fi
+
 echo @@ Exiting with status $SUCCESS
 exit $SUCCESS
 

src/integration-tests/bash/lease.sh

@@ -285,6 +285,48 @@ EOF
   fi
 }
 
+function makeLocalLeaseAndReplaceRemote {
+  # Replace the remote lease with a new lease that we own
+  # It's assumed that it's already determined it's safe to try and get the lease
+  # (either the lease is unowned, expired, or owned by us).
+  #
+  # TBD: There's a small race where this call temporarily deletes the lease before
+  # it replaces it with a new one,
+  # which means someone else could come in and snipe it even if we already
+  # own an older version of the lease and the older version hasn't expired.
+  # If this happens, this call will fail when it tries to 'checkLease'
+  # and the caller therefore is forced to give up their lease.   In theory,
+  # this race could be resolved by using a 'replace -f' pattern - but this
+  # failed with unexpected errors on some kubectl setups but not others.
+  #
+
+  makeLocalLease
+  if [ $? -ne 0 ]; then
+    traceError "failed - could not generate a new local lease"
+    return 1
+  fi
+
+  deleteRemoteLeaseUnsafe
+  if [ $? -ne 0 ]; then
+    traceError "failed - could not delete remote lease" 
+    return 1
+  fi
+
+  kubectl create configmap ${CONFIGMAP_NAME} --from-file ${LOCAL_ROOT}/${LOCAL_FILE} -n default
+  if [ $? -ne 0 ]; then
+    traceError "failed - could not replace" 
+    return 1
+  fi
+
+  # finally, check if we now actually own the lease (someone could have been replacing at the same time)
+  checkLease 
+  if [ $? -ne 0 ]; then
+    traceError "failed - replaced remote lease, but we somehow lost a race or can no longer communicate with kubernetes"
+    return 1
+  fi
+  return 0
+}
+
 function getRemoteLease {
   #
   #  first, if the remote lease configmap doesn't exist
@@ -395,25 +437,11 @@ function obtainLease {
       # so assume it can be replaced and we can try takeover the lease 
 
       # first make a local candidate lease
-      makeLocalLease
-      if [ $? -ne 0 ]; then
-        traceError "failed - could not generate a new local lease"
-        return 1
-      fi
-
-      # next, try replace remote lease with the candidate lease
-      kubectl create configmap ${CONFIGMAP_NAME} --from-file ${LOCAL_ROOT}/${LOCAL_FILE} -o yaml -n default --dry-run | kubectl replace -f -
-      if [ $? -ne 0 ]; then
-        traceError "failed - could not replace remote lease"
-        return 1
-      fi
-
-      # finally, check if we now actually own the lease (someone could have been replacing at the same time)
-      checkLease 
+      makeLocalLeaseAndReplaceRemote
       if [ $? -eq 0 ]; then
         return 0
       else
-        traceError "failed - replaced remote lease, but kubernetes is not responding or we lost a race and another potential owner replaced it too, will keep retrying up to the timeout"
+        traceError "failed to replace remote lease, will keep retrying up to the timeout"
       fi
     fi
     local mnow=`date +%s`
@@ -447,26 +475,12 @@ function renewLease {
   fi
 
   # now make a new local candidate lease
-  makeLocalLease
-  if [ $? -ne 0 ]; then
-    traceError "failed - could not generate a new local lease"
-    return 1
-  fi
-
-  # next, try replace remote lease with the candidate lease
-  kubectl create configmap ${CONFIGMAP_NAME} --from-file ${LOCAL_ROOT}/${LOCAL_FILE} -o yaml -n default --dry-run | kubectl replace -f -
+  makeLocalLeaseAndReplaceRemote
   if [ $? -ne 0 ]; then
-    traceError "failed - could not get replace remote lease"
+    traceError "failed to replace remote lease"
     return 1
-  fi
-
-  # finally, check if we now actually own the lease (someone could have been replacing at the same time)
-  checkLease 
-  if [ $? -eq 0 ]; then
-    return 0
   else
-    traceError "failed - replaced remote lease, but we somehow lost a race or can no longer communicate with kubernetes"
-    return 1
+    return 0
   fi
 }