Make managed server shutdown more resilient

Author: rjeberhard

common/src/main/resources/Operator.properties

@@ -132,8 +132,8 @@ WLSKO-0195=The operator failed after retrying for {0,number,#} minutes. \
 WLSKO-0198={0} Fiber {1}
 WLSKO-0199=WL pod shutdown: Initiating shutdown of WebLogic server {0} via REST interface.
 WLSKO-0200=WL pod shutdown: Successfully shutdown WebLogic server {0} via REST interface.
-WLSKO-0201=WL pod shutdown: Failed to shutdown WebLogic server {0} via REST interface with response: {1}.
-WLSKO-0203=WL pod shutdown: Failed to shutdown WebLogic server {0} via REST interface due to exception: {1}.
+WLSKO-0201=WL pod shutdown: Failed to shut down WebLogic server {0} via REST interface with response: {1}.
+WLSKO-0203=WL pod shutdown: Failed to shut down WebLogic server {0} via REST interface due to exception: {1}.
 WLSKO-0204=WL pod shutdown: Retry shutdown of WebLogic server {0} via REST interface.
 WLSKO-0213=InputFile: {0} does not exist or could not be read.
 WLSKO-0214=The output dir {0} specified by ''-d'' does not exist or could not be read.

operator/src/main/java/oracle/kubernetes/operator/DomainProcessorImpl.java

@@ -621,8 +621,8 @@ private void processServerPodWatch(V1Pod pod, String watchType) {
           }
         }
         boolean isReady = PodHelper.isReady(pod);
-        boolean isLabedlForShutdown = PodHelper.isPodAlreadyLabeledForShutdown(pod);
-        if ((isEvicted || isReady != isLabedlForShutdown || PodHelper.isFailed(pod)) && !PodHelper.isDeleting(pod)) {
+        boolean isLabeledForShutdown = PodHelper.isPodAlreadyAnnotatedForShutdown(pod);
+        if ((isEvicted || isReady != isLabeledForShutdown || PodHelper.isFailed(pod)) && !PodHelper.isDeleting(pod)) {
           createMakeRightOperation(info).interrupt().withExplicitRecheck().execute();
         }
         boolean isUnschedulable = PodHelper.hasUnSchedulableCondition(pod);

operator/src/main/java/oracle/kubernetes/operator/DomainStatusUpdater.java

@@ -1297,11 +1297,11 @@ private boolean isServerComplete(@Nonnull String serverName) {
             && isNotMarkedForRoll(serverName);
       }
 
-      // returns true if the server pod does not have a label indicating that it needs to be rolled
+      // returns true if the server pod does not have an annotation indicating that it needs to be rolled
       private boolean isNotMarkedForRoll(String serverName) {
         return Optional.ofNullable(getInfo().getServerPod(serverName))
             .map(V1Pod::getMetadata)
-            .map(V1ObjectMeta::getLabels)
+            .map(V1ObjectMeta::getAnnotations)
             .map(Map::keySet).orElse(Collections.emptySet()).stream()
             .noneMatch(k -> k.equals(TO_BE_ROLLED_LABEL));
       }

operator/src/main/java/oracle/kubernetes/operator/MakeRightDomainOperation.java

@@ -81,7 +81,7 @@ private static boolean wasInspectionRun(Packet packet) {
     if (lastTransitionTime != null) {
       String time = packet.getValue(INTROSPECTION_TIME);
       if (time != null) {
-        OffsetDateTime lastIntrospectionTime = OffsetDateTime.parse(time.replace('_', ':'));
+        OffsetDateTime lastIntrospectionTime = OffsetDateTime.parse(time);
         if (lastIntrospectionTime.isAfter(lastTransitionTime)) {
           return true;
         }

operator/src/main/java/oracle/kubernetes/operator/helpers/ConfigMapHelper.java

@@ -220,7 +220,8 @@ abstract static class ConfigMapContext extends StepContextBase {
     private final String name;
     private final String namespace;
     private V1ConfigMap model;
-    private final Map<String, String> labels = new HashMap<>();
+    private Map<String, String> annotations;
+    private Map<String, String> labels;
     protected final SemanticVersion productVersion;
 
     ConfigMapContext(Step conflictStep, String name, String namespace, Map<String, String> contents,
@@ -275,6 +276,7 @@ private V1ObjectMeta createMetadata() {
           new V1ObjectMeta()
           .name(name)
           .namespace(namespace)
+          .annotations(annotations)
           .labels(labels));
 
       if (productVersion != null) {
@@ -284,8 +286,20 @@ private V1ObjectMeta createMetadata() {
       return metadata;
     }
 
+    @SuppressWarnings("SameParameterValue")
+    void addAnnotation(String name, String value) {
+      if (annotations == null) {
+        annotations = new HashMap<>();
+      }
+      annotations.put(name, value);
+      model = null;
+    }
+
     @SuppressWarnings("SameParameterValue")
     void addLabel(String name, String value) {
+      if (labels == null) {
+        labels = new HashMap<>();
+      }
       labels.put(name, value);
       model = null;
     }
@@ -332,7 +346,7 @@ public Result onSuccess(Packet packet, KubernetesApiResponse<V1ConfigMap> callRe
         Optional.ofNullable(domain).map(DomainResource::getMetadata).map(V1ObjectMeta::getGeneration)
             .ifPresent(value -> addLabel(INTROSPECTION_DOMAIN_SPEC_GENERATION, value.toString()));
         Optional.ofNullable((String) packet.get(INTROSPECTION_TIME))
-                .ifPresent(value -> addLabel(INTROSPECTION_TIME, value));
+                .ifPresent(value -> addAnnotation(INTROSPECTION_TIME, value));
         V1ConfigMap existingMap = withoutTransientData(callResponse.getObject());
         if (existingMap == null) {
           return doNext(createConfigMap(getNext()), packet);
@@ -369,8 +383,12 @@ private Step replaceConfigMap(Step next) {
         return RequestBuilder.CM.update(model, createReplaceResponseStep(next));
       }
 
+      private Map<String,String> getAnnotations() {
+        return Optional.ofNullable(annotations).map(Collections::unmodifiableMap).orElse(Collections.emptyMap());
+      }
+
       private Map<String,String> getLabels() {
-        return Collections.unmodifiableMap(labels);
+        return Optional.ofNullable(labels).map(Collections::unmodifiableMap).orElse(Collections.emptyMap());
       }
 
       private boolean mustPatchCurrentMap(V1ConfigMap currentMap) {
@@ -921,7 +939,12 @@ private void recordIntrospectVersionAndGeneration(V1ConfigMap result, Packet pac
                       () -> packet.remove(INTROSPECTION_STATE_LABEL));
       Optional.ofNullable(labels).map(l -> l.get(INTROSPECTION_DOMAIN_SPEC_GENERATION))
               .ifPresent(generation -> packet.put(INTROSPECTION_DOMAIN_SPEC_GENERATION, generation));
-      Optional.ofNullable(labels).map(l -> l.get(INTROSPECTION_TIME))
+
+      Map<String, String> annotations = Optional.ofNullable(result)
+              .map(V1ConfigMap::getMetadata)
+              .map(V1ObjectMeta::getAnnotations).orElse(null);
+
+      Optional.ofNullable(annotations).map(l -> l.get(INTROSPECTION_TIME))
               .ifPresent(value -> packet.put(INTROSPECTION_TIME, value));
     }
 

operator/src/main/java/oracle/kubernetes/operator/helpers/DomainPresenceInfo.java

@@ -373,7 +373,7 @@ public Stream<V1Pod> getServerPods() {
    */
   public Stream<V1Pod> getServerPodsNotBeingDeleted() {
     return getActiveServers().values().stream().filter(
-        sko -> !PodHelper.isPodAlreadyLabeledForShutdown(sko.getPod().get())).map(this::getPod)
+        sko -> !PodHelper.isPodAlreadyAnnotatedForShutdown(sko.getPod().get())).map(this::getPod)
         .filter(Objects::nonNull);
   }
 
@@ -383,7 +383,7 @@ private V1Pod getPod(ServerKubernetesObjects sko) {
 
   public boolean isServerPodBeingDeleted(String serverName) {
     return Optional.ofNullable(getSko(serverName)).map(ServerKubernetesObjects::getPod).map(AtomicReference::get)
-        .map(PodHelper::isPodAlreadyLabeledForShutdown).orElse(false);
+        .map(PodHelper::isPodAlreadyAnnotatedForShutdown).orElse(false);
   }
 
   public boolean isServerPodDeleted(String serverName) {

operator/src/main/java/oracle/kubernetes/operator/helpers/JobHelper.java

@@ -505,7 +505,7 @@ private class ReadPodLogStep extends Step {
 
         Optional.ofNullable(jobPod).map(V1Pod::getMetadata)
             .map(V1ObjectMeta::getCreationTimestamp).map(OffsetDateTime::toString)
-                .ifPresent(creationTime -> packet.put(INTROSPECTION_TIME, creationTime.replace(':', '_')));
+                .ifPresent(creationTime -> packet.put(INTROSPECTION_TIME, creationTime));
         return doNext(readDomainIntrospectorPodLog(jobPodName, containerName, getNext()), packet);
       }
 

operator/src/main/java/oracle/kubernetes/operator/helpers/PodHelper.java

@@ -7,6 +7,7 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Optional;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
@@ -374,7 +375,7 @@ private static boolean hasLabel(V1Pod pod, String label) {
         .map(l -> l.containsKey(label)).orElse(false);
   }
 
-  private static Step patchPod(V1Pod pod, String label, Step next) {
+  private static Step patchPodLabel(V1Pod pod, String label, Step next) {
     if (!hasLabel(pod, label)) {
       JsonPatchBuilder patchBuilder = Json.createPatchBuilder();
       patchBuilder.add("/metadata/labels/" + label, "true");
@@ -386,6 +387,41 @@ private static Step patchPod(V1Pod pod, String label, Step next) {
     return next;
   }
 
+  /**
+   * get pod's annotation value for a annotation name.
+   * @param pod pod
+   * @param annotationName annotation name
+   * @return annotation value
+   */
+  public static String getPodAnnotation(V1Pod pod, String annotationName) {
+    return Optional.ofNullable(pod)
+            .map(V1Pod::getMetadata)
+            .map(V1ObjectMeta::getAnnotations)
+            .map(m -> m.get(annotationName))
+            .orElse(null);
+  }
+
+  private static boolean hasAnnotation(V1Pod pod, String annotation) {
+    return Optional.ofNullable(pod).map(V1Pod::getMetadata).map(V1ObjectMeta::getAnnotations)
+            .map(l -> l.containsKey(annotation)).orElse(false);
+  }
+
+  private static Step patchPodAnnotation(V1Pod pod, String annotation, Step next) {
+    return patchPodAnnotation(pod, annotation, "true", next);
+  }
+
+  private static Step patchPodAnnotation(V1Pod pod, String annotation, String value, Step next) {
+    if (!hasAnnotation(pod, annotation)) {
+      JsonPatchBuilder patchBuilder = Json.createPatchBuilder();
+      patchBuilder.add("/metadata/annotations/" + annotation, value);
+      V1ObjectMeta meta = pod.getMetadata();
+      return RequestBuilder.POD.patch(meta.getNamespace(), meta.getName(),
+              V1Patch.PATCH_FORMAT_JSON_PATCH,
+              new V1Patch(patchBuilder.build().toString()), patchResponse(next));
+    }
+    return next;
+  }
+
   private static ResponseStep<V1Pod> patchResponse(Step next) {
     return new PatchPodResponseStep(next);
   }
@@ -413,22 +449,26 @@ public Result onFailure(Packet packet, KubernetesApiResponse<V1Pod> callResponse
   }
 
   /**
-   * Label pod as needing to shut down.
+   * Annotate pod as needing to shut down.
    * @param pod Pod
    * @param next Next step
-   * @return Step that will check for existing label and add if it is missing
+   * @return Step that will check for existing annotation and add if it is missing
    */
-  public static Step labelPodAsNeedingToShutdown(V1Pod pod, Step next) {
-    return patchPod(pod, LabelConstants.TO_BE_SHUTDOWN_LABEL, next);
+  public static Step annotatePodAsNeedingToShutdown(V1Pod pod, String value, Step next) {
+    return patchPodAnnotation(pod, LabelConstants.TO_BE_SHUTDOWN_LABEL, value, next);
   }
 
   /**
-   * Check if the pod is already labeld for shut down.
+   * Check if the pod is already annotated for shut down.
    * @param pod Pod
-   * @return true, if the pod is already labeled.
+   * @return true, if the pod is already annotated.
    */
-  public static boolean isPodAlreadyLabeledForShutdown(V1Pod pod) {
-    return hasLabel(pod, LabelConstants.TO_BE_SHUTDOWN_LABEL);
+  public static boolean isPodAlreadyAnnotatedForShutdown(V1Pod pod) {
+    return !Objects.isNull(getPodShutdownAnnotation(pod));
+  }
+
+  public static String getPodShutdownAnnotation(V1Pod pod) {
+    return getPodAnnotation(pod, LabelConstants.TO_BE_SHUTDOWN_LABEL);
   }
 
   /**
@@ -711,7 +751,7 @@ String getServerName() {
     @Override
     // let the pod rolling step update the pod
     Step replaceCurrentPod(V1Pod pod, Step next) {
-      return labelPodAsNeedingToRoll(pod, new DeferProcessing(pod, next));
+      return annotatePodAsNeedingToRoll(pod, new DeferProcessing(pod, next));
     }
 
     private class DeferProcessing extends Step {
@@ -767,8 +807,8 @@ private void deferProcessing(Step deferredStep) {
     }
 
     // Patch the pod to indicate a pending roll.
-    private Step labelPodAsNeedingToRoll(V1Pod pod, Step next) {
-      return patchPod(pod, LabelConstants.TO_BE_ROLLED_LABEL, next);
+    private Step annotatePodAsNeedingToRoll(V1Pod pod, Step next) {
+      return patchPodAnnotation(pod, LabelConstants.TO_BE_ROLLED_LABEL, next);
     }
 
     private Fiber.StepAndPacket createRollRequest(Step deferredStep) {

operator/src/main/java/oracle/kubernetes/operator/steps/HttpRequestProcessing.java

@@ -6,6 +6,7 @@
 import java.net.URI;
 import java.net.http.HttpRequest;
 import java.net.http.HttpResponse;
+import java.time.Duration;
 import java.time.OffsetDateTime;
 import java.util.Collections;
 import java.util.HashMap;
@@ -30,7 +31,8 @@
 
 abstract class HttpRequestProcessing {
 
-  private static final Long HTTP_TIMEOUT_SECONDS = 60L;
+  public static final Long HTTP_TIMEOUT_SECONDS = 60L;
+
   private static final Map<String, CookieList> COOKIES = new ConcurrentHashMap<>();
 
   private final Packet packet;
@@ -84,10 +86,11 @@ AuthorizationSource getAuthorizationSource() {
     return SecretHelper.getAuthorizationSource(getPacket());
   }
 
-  final HttpRequest.Builder createRequestBuilder(String url) {
+  final HttpRequest.Builder createRequestBuilder(String url, long timeout) {
     final URI uri = URI.create(url);
     HttpRequest.Builder builder = HttpRequest.newBuilder()
           .uri(uri)
+          .timeout(Duration.ofSeconds(timeout))
           .header("Accept", "application/json")
           .header("Content-Type", "application/json")
           .header("X-Requested-By", "WebLogic Operator");

operator/src/main/java/oracle/kubernetes/operator/steps/ManagedServerUpIteratorStep.java

@@ -162,7 +162,7 @@ protected boolean isPodReady(V1Pod result) {
     }
 
     protected boolean isPodMarkedForShutdown(V1Pod result) {
-      return PodHelper.isDeleting(result) || PodHelper.isPodAlreadyLabeledForShutdown(result);
+      return PodHelper.isDeleting(result) || PodHelper.isPodAlreadyAnnotatedForShutdown(result);
     }
   }