Skip to content

Commit c1f3fd1

Browse files
maggiehe00maggiehe00
authored
Modify the pod restart testcase with image update (#3967)
* Modify the pod restart test case with image update
1 parent 8ff18aa commit c1f3fd1

File tree

4 files changed

+144
-32
lines changed

4 files changed

+144
-32
lines changed

‎integration-tests/src/test/java/oracle/weblogic/kubernetes/ItIntrospectVersion.java

Lines changed: 90 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
// Copyright (c) 2020, 2022, Oracle and/or its affiliates.
1+
// Copyright (c) 2020, 2023, Oracle and/or its affiliates.
22
// Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
33

44
package oracle.weblogic.kubernetes;
@@ -16,6 +16,7 @@
1616
import java.util.Map;
1717
import java.util.Properties;
1818
import java.util.concurrent.Callable;
19+
import java.util.logging.Level;
1920

2021
import io.kubernetes.client.custom.V1Patch;
2122
import io.kubernetes.client.openapi.models.CoreV1Event;
@@ -47,6 +48,7 @@
4748
import oracle.weblogic.kubernetes.utils.ExecResult;
4849
import oracle.weblogic.kubernetes.utils.OracleHttpClient;
4950
import org.awaitility.core.ConditionFactory;
51+
import org.awaitility.core.ConditionTimeoutException;
5052
import org.junit.jupiter.api.BeforeAll;
5153
import org.junit.jupiter.api.DisplayName;
5254
import org.junit.jupiter.api.MethodOrderer;
@@ -75,16 +77,19 @@
7577
import static oracle.weblogic.kubernetes.actions.TestActions.deleteSecret;
7678
import static oracle.weblogic.kubernetes.actions.TestActions.dockerTag;
7779
import static oracle.weblogic.kubernetes.actions.TestActions.execCommand;
80+
import static oracle.weblogic.kubernetes.actions.TestActions.getContainerRestartCount;
7881
import static oracle.weblogic.kubernetes.actions.TestActions.getCurrentIntrospectVersion;
7982
import static oracle.weblogic.kubernetes.actions.TestActions.getDomainCustomResource;
8083
import static oracle.weblogic.kubernetes.actions.TestActions.getNextIntrospectVersion;
8184
import static oracle.weblogic.kubernetes.actions.TestActions.getServiceNodePort;
8285
import static oracle.weblogic.kubernetes.actions.TestActions.getServicePort;
8386
import static oracle.weblogic.kubernetes.actions.TestActions.now;
8487
import static oracle.weblogic.kubernetes.actions.TestActions.patchDomainResourceWithNewIntrospectVersion;
88+
import static oracle.weblogic.kubernetes.actions.TestActions.patchDomainResourceWithNewRestartVersion;
8589
import static oracle.weblogic.kubernetes.actions.TestActions.scaleCluster;
8690
import static oracle.weblogic.kubernetes.actions.impl.Domain.patchDomainCustomResource;
8791
import static oracle.weblogic.kubernetes.actions.impl.Pod.getPod;
92+
import static oracle.weblogic.kubernetes.assertions.TestAssertions.podPending;
8893
import static oracle.weblogic.kubernetes.assertions.TestAssertions.podStateNotChanged;
8994
import static oracle.weblogic.kubernetes.assertions.TestAssertions.verifyRollingRestartOccurred;
9095
import static oracle.weblogic.kubernetes.utils.CommonMiiTestUtils.verifyPodsNotRolled;
@@ -94,7 +99,6 @@
9499
import static oracle.weblogic.kubernetes.utils.CommonTestUtils.testUntil;
95100
import static oracle.weblogic.kubernetes.utils.CommonTestUtils.verifyCredentials;
96101
import static oracle.weblogic.kubernetes.utils.CommonTestUtils.verifyServerCommunication;
97-
import static oracle.weblogic.kubernetes.utils.CommonTestUtils.withStandardRetryPolicy;
98102
import static oracle.weblogic.kubernetes.utils.ConfigMapUtils.createConfigMapForDomainCreation;
99103
import static oracle.weblogic.kubernetes.utils.DeployUtil.deployUsingRest;
100104
import static oracle.weblogic.kubernetes.utils.DomainUtils.createDomainAndVerify;
@@ -108,7 +112,6 @@
108112
import static oracle.weblogic.kubernetes.utils.K8sEvents.getOpGeneratedEvent;
109113
import static oracle.weblogic.kubernetes.utils.OKDUtils.createRouteForOKD;
110114
import static oracle.weblogic.kubernetes.utils.OperatorUtils.installAndVerifyOperator;
111-
import static oracle.weblogic.kubernetes.utils.PatchDomainUtils.patchDomainResource;
112115
import static oracle.weblogic.kubernetes.utils.PersistentVolumeUtils.createPV;
113116
import static oracle.weblogic.kubernetes.utils.PersistentVolumeUtils.createPVC;
114117
import static oracle.weblogic.kubernetes.utils.PodUtils.checkPodDoesNotExist;
@@ -129,6 +132,7 @@
129132
import static org.junit.jupiter.api.Assertions.assertFalse;
130133
import static org.junit.jupiter.api.Assertions.assertNotEquals;
131134
import static org.junit.jupiter.api.Assertions.assertNotNull;
135+
import static org.junit.jupiter.api.Assertions.assertThrows;
132136
import static org.junit.jupiter.api.Assertions.assertTrue;
133137

134138
/**
@@ -681,12 +685,12 @@ void testCreateNewCluster() {
681685
}
682686

683687
/**
684-
* Modify the domain scope property
685-
* From: "image: container-registry.oracle.com/middleware/weblogic:ImageTagBeingUsed" to
686-
* To: "image: container-registry.oracle.com/middleware/weblogic:DateAndTimeStamp"
687-
* e.g, From ""image: container-registry.oracle.com/middleware/weblogic:12.2.1.4"
688-
* To: "image:container-registry.oracle.com/middleware/weblogic:2021-07-08-162571383699"
689-
* Verify all the pods are restarted and back to ready state
688+
* In this test firstly we patch the running domain with an image that does not exist.
689+
* Admin server pod will be recreated but fail to get into "Ready" state
690+
* So even with a new updated restartVersion rolling restart will not be triggered in the domain
691+
* Admin server pod is in Pending state with container restart count 0
692+
* Secondly we patch the domain with a new available image
693+
* Verify rolling restart is triggered in the domain
690694
* Verify the admin server is accessible and cluster members are healthy
691695
*/
692696
@Test
@@ -721,26 +725,51 @@ void testUpdateImageName() {
721725
String imageName = domain1.getSpec().getImage();
722726
logger.info("Currently the image name used for the domain is: {0}", imageName);
723727

724-
//change image name to imageUpdate
725-
String imageTag = CommonTestUtils.getDateAndTimeStamp();
726-
String imageUpdate = KIND_REPO != null ? KIND_REPO
727-
+ (WEBLOGIC_IMAGE_NAME + ":" + imageTag).substring(TestConstants.BASE_IMAGES_REPO.length() + 1)
728-
: TEST_IMAGES_REPO + "/" + WEBLOGIC_IMAGE_NAME_DEFAULT + ":" + imageTag;
729-
getLogger().info(" The image name used for update is: {0}", imageUpdate);
730-
dockerTag(imageName, imageUpdate);
731-
dockerLoginAndPushImageToRegistry(imageUpdate);
732-
733-
StringBuffer patchStr = null;
734-
patchStr = new StringBuffer("[{");
735-
patchStr.append("\"op\": \"replace\",")
736-
.append(" \"path\": \"/spec/image\",")
737-
.append("\"value\": \"")
738-
.append(imageUpdate)
739-
.append("\"}]");
740-
logger.info("PatchStr for imageUpdate: {0}", patchStr.toString());
741-
742-
assertTrue(patchDomainResource(domainUid, introDomainNamespace, patchStr),
743-
"patchDomainCustomResource(imageUpdate) failed");
728+
//create image name used for 1st Update. This image essentially does not exist
729+
String imageTag1 = CommonTestUtils.getDateAndTimeStamp();
730+
String imageUpdate1 = KIND_REPO != null ? KIND_REPO
731+
+ (WEBLOGIC_IMAGE_NAME + ":" + imageTag1).substring(TestConstants.BASE_IMAGES_REPO.length() + 1)
732+
: TEST_IMAGES_REPO + "/" + WEBLOGIC_IMAGE_NAME_DEFAULT + ":" + imageTag1;
733+
getLogger().info(" The image name used for the 1st update is: {0}", imageUpdate1);
734+
735+
// 1st time patch the domain resource with an image that does not exist in the registry, update domain
736+
// restartVersion. After this admin server pod will be recreated but fail
737+
// to get into "Ready" state because of ImagePullBackOff error. Since admin server is stuck managed server
738+
// pods will not be recreated. Rolling restart is not triggered in the domain.
739+
patchDomainWithNewImage(imageUpdate1);
740+
String newRestartVersion = patchDomainResourceWithNewRestartVersion(domainUid, introDomainNamespace);
741+
logger.log(Level.INFO, "New restart version is {0}", newRestartVersion);
742+
logger.info("Verifying rolling restart did NOT occur for domain {0} in namespace {1}",
743+
domainUid, introDomainNamespace);
744+
//verify rolling restart is not triggered in the domain
745+
assertThrows(ConditionTimeoutException.class, () -> {
746+
verifyRollingRestartOccurred(cl1podsWithTimeStamps, 1, introDomainNamespace);
747+
});
748+
if (cluster2Created) {
749+
assertThrows(ConditionTimeoutException.class, () -> {
750+
verifyRollingRestartOccurred(cl2podsWithTimeStamps, 1, introDomainNamespace);
751+
});
752+
}
753+
754+
//verify admin server pod container restartCount is 0 before 2nd time image update
755+
assertTrue((getPodRestartCount(introDomainNamespace, adminServerPodName) == 0),
756+
String.format("Pod %s restart count does not equals to 0", adminServerPodName));
757+
//verify admin server pod is in pending state before 2nd time image update
758+
assertDoesNotThrow(() -> podPending(adminServerPodName, domainUid, introDomainNamespace),
759+
String.format("podPending failed with ApiException for pod %s in namespace %s",
760+
adminServerPodName, introDomainNamespace));
761+
762+
//create image name used for 2nd Update and make it available with proper tagging
763+
String imageTag2 = CommonTestUtils.getDateAndTimeStamp();
764+
String imageUpdate2 = KIND_REPO != null ? KIND_REPO
765+
+ (WEBLOGIC_IMAGE_NAME + ":" + imageTag1).substring(TestConstants.BASE_IMAGES_REPO.length() + 1)
766+
: TEST_IMAGES_REPO + "/" + WEBLOGIC_IMAGE_NAME_DEFAULT + ":" + imageTag2;
767+
getLogger().info(" The image name used for the 2st update is: {0}", imageUpdate2);
768+
dockerTag(imageName, imageUpdate2);
769+
dockerLoginAndPushImageToRegistry(imageUpdate2);
770+
771+
//2nd time update domain resource with available image
772+
patchDomainWithNewImage(imageUpdate2);
744773

745774
domain1 = assertDoesNotThrow(() -> getDomainCustomResource(domainUid, introDomainNamespace),
746775
String.format("getDomainCustomResource failed with ApiException when tried to get domain %s in namespace %s",
@@ -1334,4 +1363,36 @@ private void verifyConnectionBetweenClusterMembers(String serverName, List<Strin
13341363
+ "&password=" + wlsPassword + "\"",managedServerPort);
13351364
verifyServerCommunication(command, serverName, managedServerNames);
13361365
}
1366+
1367+
private void patchDomainWithNewImage(String newImage) {
1368+
// get the original domain resource before update
1369+
Domain domain1 = assertDoesNotThrow(() -> getDomainCustomResource(domainUid, introDomainNamespace),
1370+
String.format("getDomainCustomResource failed with ApiException when tried to get domain %s in namespace %s",
1371+
domainUid, introDomainNamespace));
1372+
assertNotNull(domain1, "Got null domain resource");
1373+
assertNotNull(domain1.getSpec(), domain1 + "/spec is null");
1374+
1375+
logger.info("patch the domain resource with new image");
1376+
String patchStr
1377+
= "["
1378+
+ "{\"op\": \"replace\", \"path\": \"/spec/image\", "
1379+
+ "\"value\": \"" + newImage + "\"}"
1380+
+ "]";
1381+
logger.info("Updating domain configuration using patch string: {0}\n", patchStr);
1382+
V1Patch patch = new V1Patch(patchStr);
1383+
assertTrue(patchDomainCustomResource(domainUid, introDomainNamespace, patch, V1Patch.PATCH_FORMAT_JSON_PATCH),
1384+
"Failed to patch domain");
1385+
}
1386+
1387+
private int getPodRestartCount(String nameSpace, String podName) {
1388+
int restartCount =
1389+
assertDoesNotThrow(() -> getContainerRestartCount(nameSpace, null,
1390+
podName, null),
1391+
String.format("Failed to get the restart count of the container from pod %s in namespace %s",
1392+
podName, nameSpace));
1393+
logger.info("For server pod {0} restart count is: {1}",
1394+
podName, restartCount);
1395+
return restartCount;
1396+
}
1397+
13371398
}

‎integration-tests/src/test/java/oracle/weblogic/kubernetes/assertions/TestAssertions.java

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
// Copyright (c) 2020, 2022, Oracle and/or its affiliates.
1+
// Copyright (c) 2020, 2023, Oracle and/or its affiliates.
22
// Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
33

44
package oracle.weblogic.kubernetes.assertions;
@@ -314,6 +314,18 @@ public static Callable<Boolean> podInitialized(String podName, String domainUid,
314314
return Pod.podInitialized(namespace, domainUid, podName);
315315
}
316316

317+
/**
318+
* Check if a Kubernetes pod is in pending state.
319+
*
320+
* @param podName name of the pod to check for
321+
* @param domainUid WebLogic domain uid in which the pod belongs
322+
* @param namespace name of the namespace in which to check the pod status
323+
* @return true if the pod is pending otherwise false
324+
*/
325+
public static Callable<Boolean> podPending(String podName, String domainUid, String namespace) {
326+
return Pod.podPending(namespace, domainUid, podName);
327+
}
328+
317329
/**
318330
* Check if a pod given by the podName is in Terminating state.
319331
*

‎integration-tests/src/test/java/oracle/weblogic/kubernetes/assertions/impl/Kubernetes.java

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
// Copyright (c) 2020, 2022, Oracle and/or its affiliates.
1+
// Copyright (c) 2020, 2023, Oracle and/or its affiliates.
22
// Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
33

44
package oracle.weblogic.kubernetes.assertions.impl;
@@ -135,6 +135,31 @@ public static boolean isPodRunning(String namespace, String domainUid, String po
135135
return status;
136136
}
137137

138+
/**
139+
* Checks if a pod exists in a given namespace and is in Pending state.
140+
* @param namespace in which to check for the pod running
141+
* @param domainUid the label the pod is decorated with
142+
* @param podName name of the pod to check for
143+
* @return true if pod is in pending state otherwise false
144+
* @throws ApiException when there is error in querying the cluster
145+
*/
146+
public static boolean isPodPending(String namespace, String domainUid, String podName) throws ApiException {
147+
boolean status = false;
148+
String labelSelector = null;
149+
if (domainUid != null) {
150+
labelSelector = String.format("weblogic.domainUID in (%s)", domainUid);
151+
}
152+
V1Pod pod = getPod(namespace, labelSelector, podName);
153+
if (pod != null) {
154+
getLogger().info("In the namespace {0} pod {1} is in the state {2}", namespace, podName,
155+
pod.getStatus().getPhase());
156+
status = pod.getStatus().getPhase().equals("Pending");
157+
} else {
158+
getLogger().info("Pod doesn't exist");
159+
}
160+
return status;
161+
}
162+
138163
/**
139164
* Checks if a pod exists in a given namespace and in Initialized state.
140165
* @param namespace in which to check for the pod running

‎integration-tests/src/test/java/oracle/weblogic/kubernetes/assertions/impl/Pod.java

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
// Copyright (c) 2020, 2022, Oracle and/or its affiliates.
1+
// Copyright (c) 2020, 2023, Oracle and/or its affiliates.
22
// Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
33

44
package oracle.weblogic.kubernetes.assertions.impl;
@@ -136,6 +136,20 @@ public static Callable<Boolean> podReady(String namespace, String podName, Map<S
136136
};
137137
}
138138

139+
/**
140+
* Check a given pod is in pending state.
141+
*
142+
* @param namespace name of the namespace in which to check the pod status
143+
* @param domainUid UID of the WebLogic domain
144+
* @param podName name of the pod
145+
* @return true if pod is pending otherwise false
146+
*/
147+
public static Callable<Boolean> podPending(String namespace, String domainUid, String podName) {
148+
return () -> {
149+
return Kubernetes.isPodPending(namespace, domainUid, podName);
150+
};
151+
}
152+
139153
/**
140154
* Check a given pod is in initializing status.
141155
*

0 commit comments

Comments
 (0)