Merge branch 'shutdown-processing' into 'main'

Allow for faster shut down and don't block waiting for a shutting down instance to be ready

See merge request weblogic-cloud/weblogic-kubernetes-operator!4725

Author: rjeberhard

operator/src/main/java/oracle/kubernetes/operator/steps/ManagedServerUpIteratorStep.java

@@ -149,7 +149,7 @@ static class ManagedPodReadyStep extends Step {
               (WlsDomainConfig) packet.get(ProcessingConstants.DOMAIN_TOPOLOGY);
       V1Pod managedPod = info.getServerPod(serverName);
 
-      if (managedPod == null || !isPodReady(managedPod)) {
+      if (managedPod == null || (!isPodReady(managedPod) && !isPodMarkedForShutdown(managedPod))) {
         // requeue to wait for managed pod to be ready
         return doRequeue(packet);
       }
@@ -158,7 +158,11 @@ static class ManagedPodReadyStep extends Step {
     }
 
     protected boolean isPodReady(V1Pod result) {
-      return result != null && !PodHelper.isDeleting(result) && PodHelper.isReady(result);
+      return PodHelper.isReady(result);
+    }
+
+    protected boolean isPodMarkedForShutdown(V1Pod result) {
+      return PodHelper.isDeleting(result) || PodHelper.isPodAlreadyLabeledForShutdown(result);
     }
   }
 

operator/src/main/resources/scripts/stopServer.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright (c) 2017, 2022, Oracle and/or its affiliates.
+# Copyright (c) 2017, 2024, Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
 
 #
@@ -46,17 +46,32 @@ export SHUTDOWN_TIMEOUT_ARG=${SHUTDOWN_TIMEOUT:-30}
 # Calculate the wait timeout to issue "kill -9" before the pod is destroyed. 
 # Allow 3 seconds for the NFS v3 manager to detect the process destruction and release file locks.
 export SIGKILL_WAIT_TIMEOUT=$(expr $SHUTDOWN_TIMEOUT_ARG - 3)
-wait_and_kill_after_timeout(){
+
+# Block until the given file appears or the given timeout is reached.
+wait_file() {
+  local file="$1"; shift
+  local wait_seconds="${1:-10}"; shift
+  test $wait_seconds -lt 1 && echo 'At least 1 second is required' && return 1
+
+  until test $((wait_seconds--)) -eq 0 -o -e "$file" ; do sleep 1; done
+
+  test $wait_seconds -ge 0
+}
+
+wait_and_kill_after_timeout() {
   trace "Wait for ${SIGKILL_WAIT_TIMEOUT} seconds for ${SERVER_NAME} to shut down." >> ${STOP_OUT_FILE}
   sleep ${SIGKILL_WAIT_TIMEOUT}
-  trace "The server ${SERVER_NAME} didn't shut down in ${SIGKILL_WAIT_TIMEOUT} seconds, " \
-        "killing the server processes." >> ${STOP_OUT_FILE}
-  # Adjust PATH if necessary before calling jps
-  adjustPath
-
-  #Specifically killing the NM first as it can auto-restart a killed WL server.
-  kill -9 `jps -v | grep " NodeManager " | awk '{ print $1 }'`
-  kill -9 `jps -v | grep -v Jps | awk '{ print $1 }'`
+  wait_file "${SHUTDOWN_MARKER_FILE}" ${SIGKILL_WAIT_TIMEOUT} || {
+    trace "The server ${SERVER_NAME} didn't shut down in ${SIGKILL_WAIT_TIMEOUT} seconds, " \
+          "killing the server processes." >> ${STOP_OUT_FILE}
+    # Adjust PATH if necessary before calling jps
+    adjustPath
+
+    #Specifically killing the NM first as it can auto-restart a killed WL server.
+    kill -9 `jps -v | grep " NodeManager " | awk '{ print $1 }'`
+    kill -9 `jps -v | grep -v Jps | awk '{ print $1 }'`
+  }
+
   touch ${SHUTDOWN_MARKER_FILE}
 }
 
@@ -106,8 +121,11 @@ check_for_shutdown() {
 
 
 # Check if the server is already shutdown
-check_for_shutdown
-[ $? -eq 0 ] && trace "Server is already shutting down, is shutdown or failed" &>>  ${STOP_OUT_FILE} && exit 0
+if check_for_shutdown ; then
+  trace "Server is already shutting down, is shutdown or failed" &>>  ${STOP_OUT_FILE}
+  touch ${SHUTDOWN_MARKER_FILE}
+  exit 0
+fi
 
 # Otherwise, connect to the node manager and stop the server instance
 [ ! -f "${SCRIPTPATH}/wlst.sh" ] && trace SEVERE "Missing file '${SCRIPTPATH}/wlst.sh'." && exit 1