Backport PR#2707 to resolve issue with rolling completed (#3417)

* Backport PR#2707 to resolve issue with rolling completed

Author: doxiao

integration-tests/src/test/java/oracle/weblogic/kubernetes/ItIntrospectVersion.java

@@ -18,6 +18,7 @@
 import java.util.concurrent.Callable;
 
 import io.kubernetes.client.custom.V1Patch;
+import io.kubernetes.client.openapi.models.CoreV1Event;
 import io.kubernetes.client.openapi.models.V1Container;
 import io.kubernetes.client.openapi.models.V1EnvVar;
 import io.kubernetes.client.openapi.models.V1LocalObjectReference;
@@ -27,6 +28,7 @@
 import io.kubernetes.client.openapi.models.V1SecretReference;
 import io.kubernetes.client.openapi.models.V1Volume;
 import io.kubernetes.client.openapi.models.V1VolumeMount;
+import io.kubernetes.client.util.Yaml;
 import oracle.weblogic.domain.AdminServer;
 import oracle.weblogic.domain.AdminService;
 import oracle.weblogic.domain.Channel;
@@ -92,15 +94,18 @@
 import static oracle.weblogic.kubernetes.utils.CommonTestUtils.testUntil;
 import static oracle.weblogic.kubernetes.utils.CommonTestUtils.verifyCredentials;
 import static oracle.weblogic.kubernetes.utils.CommonTestUtils.verifyServerCommunication;
+import static oracle.weblogic.kubernetes.utils.CommonTestUtils.withStandardRetryPolicy;
 import static oracle.weblogic.kubernetes.utils.ConfigMapUtils.createConfigMapForDomainCreation;
 import static oracle.weblogic.kubernetes.utils.DeployUtil.deployUsingRest;
 import static oracle.weblogic.kubernetes.utils.DomainUtils.createDomainAndVerify;
 import static oracle.weblogic.kubernetes.utils.ImageUtils.dockerLoginAndPushImageToRegistry;
 import static oracle.weblogic.kubernetes.utils.JobUtils.createDomainJob;
 import static oracle.weblogic.kubernetes.utils.JobUtils.getIntrospectJobName;
+import static oracle.weblogic.kubernetes.utils.K8sEvents.DOMAIN_ROLL_COMPLETED;
 import static oracle.weblogic.kubernetes.utils.K8sEvents.DOMAIN_ROLL_STARTING;
 import static oracle.weblogic.kubernetes.utils.K8sEvents.POD_CYCLE_STARTING;
 import static oracle.weblogic.kubernetes.utils.K8sEvents.checkEvent;
+import static oracle.weblogic.kubernetes.utils.K8sEvents.getOpGeneratedEvent;
 import static oracle.weblogic.kubernetes.utils.OKDUtils.createRouteForOKD;
 import static oracle.weblogic.kubernetes.utils.OperatorUtils.installAndVerifyOperator;
 import static oracle.weblogic.kubernetes.utils.PatchDomainUtils.patchDomainResource;
@@ -421,6 +426,19 @@ void testDomainIntrospectVersionRolling() {
         "Normal", timestamp, withStandardRetryPolicy);
     */
 
+    CoreV1Event event = getOpGeneratedEvent(introDomainNamespace, DOMAIN_ROLL_STARTING, "Normal", timestamp);
+    logger.info(Yaml.dump(event));
+    logger.info("verify the event message contains the domain resource changed message");
+    assertTrue(event.getMessage().contains("resource changed"));
+
+    event = getOpGeneratedEvent(introDomainNamespace, POD_CYCLE_STARTING, "Normal", timestamp);
+    logger.info(Yaml.dump(event));
+    logger.info("verify the event message contains the property changed in domain resource");
+    assertTrue(event.getMessage().contains("ADMIN_PORT"));
+
+    checkEvent(opNamespace, introDomainNamespace, domainUid, DOMAIN_ROLL_COMPLETED,
+        "Normal", timestamp, withStandardRetryPolicy);
+
     // verify the admin port is changed to newAdminPort
     assertEquals(newAdminPort, assertDoesNotThrow(()
         -> getServicePort(introDomainNamespace, getExternalServicePodName(adminServerPodName), "default"),

operator/src/main/java/oracle/kubernetes/operator/helpers/RollingHelper.java

@@ -1,4 +1,4 @@
-// Copyright (c) 2018, 2021, Oracle and/or its affiliates.
+// Copyright (c) 2018, 2022, Oracle and/or its affiliates.
 // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
 
 package oracle.kubernetes.operator.helpers;
@@ -14,6 +14,7 @@
 import java.util.concurrent.TimeUnit;
 
 import io.kubernetes.client.openapi.models.V1Pod;
+import oracle.kubernetes.operator.DomainStatusUpdater;
 import oracle.kubernetes.operator.ProcessingConstants;
 import oracle.kubernetes.operator.logging.LoggingFacade;
 import oracle.kubernetes.operator.logging.LoggingFactory;
@@ -144,25 +145,38 @@ public NextAction apply(Packet packet) {
       }
 
       if (!work.isEmpty()) {
-        return doForkJoin(createAfterRollStep(getNext()), packet, work);
+        return doForkJoin(createAfterRollStep(getNext(), true), packet, work);
       }
-
       return doNext(createAfterRollStep(getNext()), packet);
     }
 
     private Step createAfterRollStep(Step next) {
-      return new AfterRollStep(next);
+      return createAfterRollStep(next, false);
+    }
+
+    private Step createAfterRollStep(Step next, boolean rolled) {
+      return new AfterRollStep(next, rolled);
     }
   }
 
-  private static class AfterRollStep extends Step {
-    public AfterRollStep(Step next) {
+  static class AfterRollStep extends Step {
+    boolean rolled;
+
+    public AfterRollStep(Step next, boolean rolled) {
       super(next);
+      this.rolled = rolled;
     }
 
     @Override
     public NextAction apply(Packet packet) {
-      return doNext(createDomainRollCompletedEventStepIfNeeded(getNext(), packet), packet);
+      return doNext(
+          rolled
+              ?
+              createDomainRollCompletedEvent(DomainStatusUpdater.createStatusUpdateStep(getNext()), packet)
+              :
+              createDomainRollCompletedEventStepIfNeeded(
+                  DomainStatusUpdater.createStatusUpdateStep(getNext()), packet),
+            packet);
     }
 
   }
@@ -176,14 +190,19 @@ public NextAction apply(Packet packet) {
    */
   public static Step createDomainRollCompletedEventStepIfNeeded(Step next, Packet packet) {
     if ("true".equals(packet.remove(DOMAIN_ROLL_START_EVENT_GENERATED))) {
-      LOGGER.info(MessageKeys.DOMAIN_ROLL_COMPLETED, getDomainUid(packet));
-      return Step.chain(
-          EventHelper.createEventStep(new EventHelper.EventData(EventHelper.EventItem.DOMAIN_ROLL_COMPLETED)),
-          next);
+      return createDomainRollCompletedEvent(next, packet);
     }
     return next;
   }
 
+  private static Step createDomainRollCompletedEvent(Step next, Packet packet) {
+    LOGGER.info(MessageKeys.DOMAIN_ROLL_COMPLETED, getDomainUid(packet));
+    packet.remove(DOMAIN_ROLL_START_EVENT_GENERATED);
+    return Step.chain(
+            EventHelper.createEventStep(new EventHelper.EventData(EventHelper.EventItem.DOMAIN_ROLL_COMPLETED)),
+        next);
+  }
+
   private static String getDomainUid(Packet packet) {
     return packet.getSpi(DomainPresenceInfo.class).getDomainUid();
   }

operator/src/test/java/oracle/kubernetes/operator/helpers/AdminPodHelperTest.java

@@ -32,6 +32,7 @@
 import static oracle.kubernetes.operator.WebLogicConstants.RUNNING_STATE;
 import static oracle.kubernetes.operator.helpers.DomainIntrospectorJobTest.TEST_VOLUME_NAME;
 import static oracle.kubernetes.operator.helpers.DomainStatusMatcher.hasStatus;
+import static oracle.kubernetes.operator.helpers.EventHelper.EventItem.DOMAIN_ROLL_COMPLETED;
 import static oracle.kubernetes.operator.helpers.EventHelper.EventItem.POD_CYCLE_STARTING;
 import static oracle.kubernetes.operator.helpers.Matchers.hasContainer;
 import static oracle.kubernetes.operator.helpers.Matchers.hasEnvVar;
@@ -826,6 +827,24 @@ void whenDomainHomeChanged_podCycleEventCreatedWithCorrectNS()
     assertContainsEventWithNamespace(POD_CYCLE_STARTING, NS);
   }
 
+  @Test
+  void whenDomainHomeChanged_andRunAfterUpStep_domainRollCompletedEventCreatedWithCorrectMessage()
+      throws NoSuchMethodException, IllegalAccessException, InvocationTargetException {
+    initializeExistingPod();
+    getConfiguredDomainSpec().setDomainHome("adfgg");
+
+    testSupport.runSteps(getStepFactory(), Step.chain(getAfterUpStep(), terminalStep));
+    logRecords.clear();
+
+    assertThat("Expected Event " + DOMAIN_ROLL_COMPLETED + " expected with message not found",
+        getExpectedEventMessage(DOMAIN_ROLL_COMPLETED),
+        stringContainsInOrder("Rolling restart of domain", UID, " completed"));
+  }
+
+  Step getAfterUpStep() {
+    return new RollingHelper.AfterRollStep(null, false);
+  }
+
   @Test
   void whenDomainHomeChanged_generateExpectedLogMessage()
       throws NoSuchMethodException, IllegalAccessException, InvocationTargetException {

operator/src/test/java/oracle/kubernetes/operator/helpers/RollingHelperTest.java

@@ -1,4 +1,4 @@
-// Copyright (c) 2020, 2021, Oracle and/or its affiliates.
+// Copyright (c) 2020, 2022, Oracle and/or its affiliates.
 // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
 
 package oracle.kubernetes.operator.helpers;
@@ -222,10 +222,9 @@ void verifyThatWhenRollingIsEmpty_NoManagedServerPodsAreReplaced() {
   }
 
   @Test
-  void afterRoll_domainRollCompletedEventCreated() {
+  void whenRoll_domainRollCompletedEventCreated() {
     initializeExistingPods();
     testSupport.addToPacket(SERVERS_TO_ROLL, rolling);
-    testSupport.addToPacket(DOMAIN_ROLL_START_EVENT_GENERATED, "true");
     SERVER_NAMES.forEach(s ->
         rolling.put(s, createRollingStepAndPacket(modifyRestartVersion(createPodModel(s), "V10"), s)));
 
@@ -235,6 +234,17 @@ void afterRoll_domainRollCompletedEventCreated() {
     assertContainsEventWithMessage(DOMAIN_ROLL_COMPLETED, UID);
   }
 
+  @Test
+  void whenNoRoll_withDomainRollStartFlag_domainRollCompletedEventCreated() {
+    initializeExistingPods();
+    testSupport.addToPacket(DOMAIN_ROLL_START_EVENT_GENERATED, "true");
+
+    testSupport.runSteps(RollingHelper.rollServers(rolling, terminalStep));
+    logRecords.clear();
+
+    assertContainsEventWithMessage(DOMAIN_ROLL_COMPLETED, UID);
+  }
+
   @Test
   void afterRoll_expectedLogMessageFound() {
     initializeExistingPods();
@@ -266,9 +276,28 @@ void whenRolling_podCycleEventCreatedWithCorrectMessage() {
     logRecords.clear();
 
     SERVER_NAMES.forEach(s -> assertThat(
-        "Expected Event " + DOMAIN_ROLL_STARTING + " expected with message not found",
+        "Expected Event " + POD_CYCLE_STARTING + " expected with message not found",
         getExpectedEventMessage(POD_CYCLE_STARTING, getPodName(s), NS),
         stringContainsInOrder("Replacing ", getPodName(s), "domain restart version changed")));
+    assertThat("Expected Event " + DOMAIN_ROLL_COMPLETED + " expected with message not found",
+        getExpectedEventMessage(DOMAIN_ROLL_COMPLETED, UID, NS),
+        stringContainsInOrder("Rolling restart of domain", UID, "completed"));
+  }
+
+  @Test
+  void whenRolling_domainRollCompletedEventCreatedWithCorrectMessage() {
+    initializeExistingPods();
+    testSupport.addToPacket(SERVERS_TO_ROLL, rolling);
+    testSupport.addToPacket(DOMAIN_ROLL_START_EVENT_GENERATED, "true");
+    SERVER_NAMES.forEach(s ->
+        rolling.put(s, createRollingStepAndPacket(modifyRestartVersion(createPodModel(s), "V3"), s)));
+
+    testSupport.runSteps(RollingHelper.rollServers(rolling, terminalStep));
+    logRecords.clear();
+
+    assertThat("Expected Event " + DOMAIN_ROLL_COMPLETED + " expected with message not found",
+        getExpectedEventMessage(DOMAIN_ROLL_COMPLETED, UID, NS),
+        stringContainsInOrder("Rolling restart of domain", UID, "completed"));
   }
 
   @Test