Owls 105463 - Generate domain DELETED event when domain has deletion timestamp or not found during make-right (#3846)

* Fix to generate DOMAIN_DELETED event when domain has deletion timestamp or not found during make-right.

Co-authored-by: Ryan Eberhard <[email protected]>

Author: ankedia

operator/src/main/java/oracle/kubernetes/operator/PodWatcher.java

@@ -39,9 +39,9 @@
 
 import static oracle.kubernetes.common.logging.MessageKeys.EXECUTE_MAKE_RIGHT_DOMAIN;
 import static oracle.kubernetes.common.logging.MessageKeys.LOG_WAITING_COUNT;
-import static oracle.kubernetes.operator.KubernetesConstants.HTTP_NOT_FOUND;
 import static oracle.kubernetes.operator.ProcessingConstants.SERVER_NAME;
 import static oracle.kubernetes.operator.WebLogicConstants.SHUTDOWN_STATE;
+import static oracle.kubernetes.operator.WebLogicConstants.UNKNOWN_STATE;
 
 /**
  * Watches for changes to pods.
@@ -428,7 +428,7 @@ protected ResponseStep resumeIfReady(WaitForReadyStep.Callback callback) {
       return new WaitForServerShutdownResponseStep(callback, serverName);
     }
 
-    private class WaitForServerShutdownResponseStep extends ResponseStep<DomainResource> {
+    private class WaitForServerShutdownResponseStep extends DefaultResponseStep<DomainResource> {
 
       private final WaitForReadyStep<DomainResource>.Callback callback;
       private final String serverName;
@@ -442,7 +442,7 @@ private class WaitForServerShutdownResponseStep extends ResponseStep<DomainResou
       @Override
       public NextAction onSuccess(Packet packet, CallResponse<DomainResource> callResponse) {
         DomainPresenceInfo info = packet.getSpi(DomainPresenceInfo.class);
-        if (isReady(callResponse.getResult()) || callback.didResumeFiber()) {
+        if (isServerShutdown(info) || isReady(callResponse.getResult()) || callback.didResumeFiber()) {
           Optional.ofNullable(info).ifPresent(i -> i.updateLastKnownServerStatus(serverName, SHUTDOWN_STATE));
           callback.proceedFromWait(callResponse.getResult());
           return doEnd(packet);
@@ -452,11 +452,9 @@ public NextAction onSuccess(Packet packet, CallResponse<DomainResource> callResp
         }
       }
 
-      @Override
-      public NextAction onFailure(Packet packet, CallResponse<DomainResource> callResponse) {
-        return callResponse.getStatusCode() == HTTP_NOT_FOUND
-            ? doNext(packet)
-            : super.onFailure(packet, callResponse);
+      private boolean isServerShutdown(DomainPresenceInfo info) {
+        return Optional.ofNullable(info).map(i -> i.getLastKnownServerStatus(serverName))
+            .map(s -> SHUTDOWN_STATE.equals(s.getStatus()) || UNKNOWN_STATE.equals(s.getStatus())).orElse(false);
       }
     }
   }

operator/src/main/java/oracle/kubernetes/operator/makeright/MakeRightDomainOperationImpl.java

@@ -1,4 +1,4 @@
-// Copyright (c) 2022, Oracle and/or its affiliates.
+// Copyright (c) 2022, 2023, Oracle and/or its affiliates.
 // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
 
 package oracle.kubernetes.operator.makeright;
@@ -56,12 +56,14 @@
 import static oracle.kubernetes.operator.KubernetesConstants.HTTP_NOT_FOUND;
 import static oracle.kubernetes.operator.LabelConstants.INTROSPECTION_STATE_LABEL;
 import static oracle.kubernetes.operator.ProcessingConstants.DOMAIN_INTROSPECT_REQUESTED;
+import static oracle.kubernetes.operator.helpers.EventHelper.EventItem.DOMAIN_DELETED;
 
 /**
  * A factory which creates and executes steps to align the cached domain status with the value read from Kubernetes.
  */
 public class MakeRightDomainOperationImpl extends MakeRightOperationImpl<DomainPresenceInfo>
     implements MakeRightDomainOperation {
+
   private boolean explicitRecheck;
   private boolean deleting;
   private boolean inspectionRun;
@@ -258,10 +260,14 @@ private boolean isForDomain(ClusterResource clusterResource, DomainPresenceInfo
   }
 
   private Step createDomainDownPlan() {
-    return Step.chain(new DownHeadStep(), new DeleteDomainStep(), new UnregisterStep());
+    Step eventStep = null;
+    if (Optional.ofNullable(eventData).map(e -> e.getItem() != DOMAIN_DELETED).orElse(true)) {
+      eventStep = EventHelper.createEventStep(new EventHelper.EventData(DOMAIN_DELETED));
+    }
+    return Step.chain(eventStep, new DeleteDomainStep(), new UnregisterStatusUpdaterStep(), new UnregisterDPIStep());
   }
 
-  private class DownHeadStep extends Step {
+  private class UnregisterStatusUpdaterStep extends Step {
 
     @Override
     public NextAction apply(Packet packet) {
@@ -364,19 +370,21 @@ public NextAction onSuccess(Packet packet, CallResponse<DomainResource> callResp
     }
 
     private void updateCache(DomainPresenceInfo info, DomainResource domain) {
-      info.setDeleting(false);
+      info.setDeleting(domain.getMetadata().getDeletionTimestamp() != null);
       info.setDomain(domain);
     }
 
     @Override
     public NextAction onFailure(Packet packet, CallResponse<DomainResource> callResponse) {
-      return callResponse.getStatusCode() == HTTP_NOT_FOUND
-          ? doNext(createDomainDownPlan(), packet)
-          : super.onFailure(packet, callResponse);
+      if (callResponse.getStatusCode() == HTTP_NOT_FOUND) {
+        DomainPresenceInfo.fromPacket(packet).ifPresent(i -> i.setDeleting(true));
+        return doNext(createDomainDownPlan(), packet);
+      }
+      return super.onFailure(packet, callResponse);
     }
   }
 
-  private class UnregisterStep extends Step {
+  private class UnregisterDPIStep extends Step {
 
     @Override
     public NextAction apply(Packet packet) {

operator/src/main/java/oracle/kubernetes/operator/steps/ShutdownManagedServerStep.java

@@ -27,7 +27,6 @@
 import oracle.kubernetes.operator.helpers.CallBuilder;
 import oracle.kubernetes.operator.helpers.DomainPresenceInfo;
 import oracle.kubernetes.operator.helpers.PodHelper;
-import oracle.kubernetes.operator.helpers.ResponseStep;
 import oracle.kubernetes.operator.helpers.SecretHelper;
 import oracle.kubernetes.operator.http.client.HttpAsyncRequestStep;
 import oracle.kubernetes.operator.http.client.HttpResponseStep;
@@ -46,7 +45,6 @@
 import oracle.kubernetes.weblogic.domain.model.DomainResource;
 import oracle.kubernetes.weblogic.domain.model.Shutdown;
 
-import static oracle.kubernetes.operator.KubernetesConstants.HTTP_NOT_FOUND;
 import static oracle.kubernetes.operator.KubernetesConstants.WLS_CONTAINER_NAME;
 import static oracle.kubernetes.operator.LabelConstants.CLUSTERNAME_LABEL;
 import static oracle.kubernetes.operator.WebLogicConstants.ADMIN_STATE;
@@ -398,18 +396,11 @@ void setHttpAsyncRequestStep(HttpAsyncRequestStep requestStep) {
     }
   }
 
-  static class DomainUpdateStep extends ResponseStep<DomainResource> {
+  static class DomainUpdateStep extends DefaultResponseStep<DomainResource> {
     @Override
     public NextAction onSuccess(Packet packet, CallResponse<DomainResource> callResponse) {
       packet.getSpi(DomainPresenceInfo.class).setDomain(callResponse.getResult());
       return doNext(packet);
     }
-
-    @Override
-    public NextAction onFailure(Packet packet, CallResponse<DomainResource> callResponse) {
-      return callResponse.getStatusCode() == HTTP_NOT_FOUND
-          ? doNext(packet)
-          : super.onFailure(packet, callResponse);
-    }
   }
 }

operator/src/test/java/oracle/kubernetes/operator/PodWatcherTest.java

@@ -479,14 +479,17 @@ void whenDomainNotFound_waitForServerShutdownDoesNotRecordKubernetesFailure() {
     final DomainResource domain = DomainProcessorTestSetup.createTestDomain();
     final AtomicBoolean stopping = new AtomicBoolean(false);
     final PodWatcher watcher = createWatcher(stopping);
-    testSupport.addDomainPresenceInfo(new DomainPresenceInfo(domain));
+    DomainPresenceInfo info = new DomainPresenceInfo(domain);
+    testSupport.addDomainPresenceInfo(info);
 
     try {
       testSupport.failOnResource(KubernetesTestSupport.DOMAIN, NAME, NS, HTTP_NOT_FOUND);
       testSupport.runSteps(watcher.waitForServerShutdown(NAME, domain, terminalStep));
     } finally {
       stopping.set(true);
     }
+    info.updateLastKnownServerStatus(NAME, SHUTDOWN_STATE);
+    testSupport.setTime(10, TimeUnit.SECONDS);
 
     assertThat(terminalStep.wasRun(), is(true));
     assertThat(domain, not(hasCondition(FAILED).withReason(KUBERNETES)));