Skip to content

Commit 5b90f51

Browse files
rjeberhardrjeberhard
authored
Merge pull request #1116 from oracle/opupgrade
Upgrade 2.0, 2.0.1, 2.1, 2.2 operator with running domain to develop
2 parents 24950e6 + 54d08ca commit 5b90f51

File tree

8 files changed

+361
-16
lines changed

8 files changed

+361
-16
lines changed

‎integration-tests/USECASES.MD

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -134,3 +134,11 @@ Configuration Overrides Usecases
134134
| Search log level | Use Elasticsearch Count API to query logs of level=INFO and verify that total number of logs for level=INFO is not zero and failed count is zero |
135135
| Search Operator log | Use Elasticsearch Search APIs to query Operator log info and verify that log hits for type=weblogic-operator are not empty |
136136
| Search Weblogic log | Use Elasticsearch Search APIs to query Weblogic log info and verify that log hits for Weblogic servers are not empty |
137+
138+
| Operator Upgrade | Use Case |
139+
| --- | --- |
140+
| Upgrade 2.0 operator to develop | Upgrade the 2.0 operator running a Weblogic domain to develop version of the Operator and verify apiVersion of the domain CRD shows latest version |
141+
| Upgrade 2.0.1 operator to develop | Upgrade the 2.0.1 operator running a Weblogic domain to develop version of the Operator and verify apiVersion of the domain CRD shows latest version |
142+
| Upgrade 2.1 operator to develop | Upgrade the 2.1 operator running a Weblogic domain to develop version of the Operator and verify apiVersion of the domain CRD shows latest version |
143+
| Upgrade 2.2.0 operator to develop | Upgrade the 2.2.0 operator running a Weblogic domain to develop version of the Operator and verify apiVersion of the domain CRD shows latest version |
144+
| Upgrade 2.2.1 operator to develop | Upgrade the 2.2.1 operator running a Weblogic domain to develop version of the Operator and verify apiVersion of the domain CRD shows latest version |

‎integration-tests/src/test/java/oracle/kubernetes/operator/BaseTest.java

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@
1414
import java.util.logging.SimpleFormatter;
1515
import javax.jms.Connection;
1616
import javax.jms.ConnectionFactory;
17-
1817
import oracle.kubernetes.operator.utils.Domain;
1918
import oracle.kubernetes.operator.utils.ExecCommand;
2019
import oracle.kubernetes.operator.utils.ExecResult;
@@ -74,6 +73,7 @@ public class BaseTest {
7473
private static String weblogicImageTag;
7574
private static String weblogicImageName;
7675
private static String weblogicImageServer;
76+
private static String domainApiVersion;
7777

7878
// Set QUICKTEST env var to true to run a small subset of tests.
7979
// Set SMOKETEST env var to true to run an even smaller subset of tests
@@ -121,6 +121,10 @@ public static void initialize(String appPropsFile) throws Exception {
121121
System.getenv("OCR_SERVER") != null
122122
? System.getenv("OCR_SERVER")
123123
: appProps.getProperty("OCR_SERVER");
124+
domainApiVersion =
125+
System.getenv("DOMAIN_API_VERSION") != null
126+
? System.getenv("DOMAIN_API_VERSION")
127+
: appProps.getProperty("DOMAIN_API_VERSION");
124128
maxIterationsPod =
125129
new Integer(appProps.getProperty("maxIterationsPod", "" + maxIterationsPod)).intValue();
126130
waitTimePod = new Integer(appProps.getProperty("waitTimePod", "" + waitTimePod)).intValue();
@@ -271,6 +275,10 @@ public static String getWeblogicImageServer() {
271275
return weblogicImageServer;
272276
}
273277

278+
public static String getDomainApiVersion() {
279+
return domainApiVersion;
280+
}
281+
274282
public static ExecResult cleanup() throws Exception {
275283
String cmd =
276284
"export RESULT_ROOT="

‎integration-tests/src/test/java/oracle/kubernetes/operator/ItOperatorUpgrade.java

Lines changed: 282 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,282 @@
1+
// Copyright 2019, Oracle Corporation and/or its affiliates. All rights reserved.
2+
// Licensed under the Universal Permissive License v 1.0 as shown at
3+
// http://oss.oracle.com/licenses/upl.
4+
package oracle.kubernetes.operator;
5+
6+
import static oracle.kubernetes.operator.BaseTest.DOMAININIMAGE_WLST_YAML;
7+
import static oracle.kubernetes.operator.BaseTest.OPERATOR1_YAML;
8+
import static oracle.kubernetes.operator.BaseTest.QUICKTEST;
9+
import static oracle.kubernetes.operator.BaseTest.logger;
10+
11+
import java.nio.file.Files;
12+
import java.nio.file.Paths;
13+
import java.util.ArrayList;
14+
import java.util.List;
15+
import java.util.Map;
16+
import java.util.logging.Level;
17+
import oracle.kubernetes.operator.utils.Domain;
18+
import oracle.kubernetes.operator.utils.ExecResult;
19+
import oracle.kubernetes.operator.utils.Operator;
20+
import oracle.kubernetes.operator.utils.TestUtils;
21+
import org.junit.After;
22+
import org.junit.AfterClass;
23+
import org.junit.Assume;
24+
import org.junit.FixMethodOrder;
25+
import org.junit.Test;
26+
import org.junit.runners.MethodSorters;
27+
28+
/** Operator upgrade JUnit test file testing the operator upgrade from older releases to develop. */
29+
@FixMethodOrder(MethodSorters.NAME_ASCENDING)
30+
public class ItOperatorUpgrade extends BaseTest {
31+
32+
private static final String OP_BASE_REL = "2.0";
33+
private static final String OP_TARGET_RELEASE = "weblogic-kubernetes-operator:latest";
34+
private static String OP_NS = "";
35+
private static String OP_DEP_NAME = "";
36+
private static String OP_SA = "";
37+
private static String DOM_NS = "";
38+
private static String DUID = "";
39+
private static String opUpgradeTmpDir;
40+
private Domain domain = null;
41+
private static Operator operator;
42+
43+
/**
44+
* Creates operator based on operatorRelease passed to it and then creates a Weblogic domain
45+
* controlled by that operator
46+
*
47+
* @param operatorGitRelease Git branch name of the operator release version
48+
* @param operatorRelease Operator release version from the
49+
* https://hub.docker.com/r/oracle/weblogic-kubernetes-operator/tags
50+
* @throws Exception when operator or domain creation fails
51+
*/
52+
private void setupOperatorAndDomain(String operatorGitRelease, String operatorRelease)
53+
throws Exception {
54+
logger.log(Level.INFO, "+++++++++++++++Beginning Test Setup+++++++++++++++++++++");
55+
initialize(APP_PROPS_FILE);
56+
opUpgradeTmpDir = BaseTest.getResultDir() + "/operatorupgrade";
57+
TestUtils.exec("rm -rf " + Paths.get(opUpgradeTmpDir).toString());
58+
Files.createDirectories(Paths.get(opUpgradeTmpDir));
59+
Map<String, Object> operatorMap = TestUtils.loadYaml(OPERATOR1_YAML);
60+
operatorMap.put("operatorImageName", "oracle/weblogic-kubernetes-operator");
61+
operatorMap.put("operatorImageTag", operatorRelease);
62+
operatorMap.put("operatorGitVersion", operatorGitRelease);
63+
operatorMap.put("operatorGitVersionDir", opUpgradeTmpDir);
64+
operatorMap.put("namespace", OP_NS);
65+
operatorMap.put("releaseName", OP_DEP_NAME);
66+
operatorMap.put("serviceAccount", OP_SA);
67+
List<String> dom_ns = new ArrayList<String>();
68+
dom_ns.add(DOM_NS);
69+
operatorMap.put("domainNamespaces", dom_ns);
70+
operator = TestUtils.createOperator(operatorMap, Operator.RestCertType.LEGACY);
71+
TestUtils.exec("kubectl get all --all-namespaces", true);
72+
73+
Map<String, Object> wlstDomainMap = TestUtils.loadYaml(DOMAININIMAGE_WLST_YAML);
74+
wlstDomainMap.put("domainUID", DUID);
75+
wlstDomainMap.put("namespace", DOM_NS);
76+
wlstDomainMap.put("projectRoot", opUpgradeTmpDir + "/weblogic-kubernetes-operator");
77+
domain = TestUtils.createDomain(wlstDomainMap);
78+
TestUtils.exec("kubectl get all --all-namespaces", true);
79+
domain.verifyPodsCreated();
80+
domain.verifyServicesCreated();
81+
domain.verifyServersReady();
82+
logger.log(Level.INFO, "+++++++++++++++Ending Test Setup+++++++++++++++++++++");
83+
}
84+
85+
/**
86+
* cleanup the domain and operator after every test
87+
*
88+
* @throws Exception when domain and operator cleanup fails
89+
*/
90+
@After
91+
public void cleanupOperatorAndDomain() throws Exception {
92+
if (!QUICKTEST) {
93+
logger.log(Level.INFO, "+++++++++++++++Beginning AfterTest cleanup+++++++++++++++++++++");
94+
if (domain != null) {
95+
domain.destroy();
96+
}
97+
if (operator != null) {
98+
operator.destroy();
99+
}
100+
ExecResult result = cleanup();
101+
logger.log(Level.INFO, "+++++++++++++++Done AfterTest cleanup+++++++++++++++++++++");
102+
}
103+
}
104+
105+
/**
106+
* Releases k8s cluster lease, archives result, pv directories
107+
*
108+
* @throws Exception when deleting pv directories or other tearDown tasks fail.
109+
*/
110+
@AfterClass
111+
public static void staticUnPrepare() throws Exception {
112+
if (!QUICKTEST) {
113+
logger.info("+++++++++++++++++++++++++++++++++---------------------------------+");
114+
logger.info("BEGIN");
115+
logger.info("Run once, release cluster lease");
116+
tearDown(new Object() {}.getClass().getEnclosingClass().getSimpleName());
117+
logger.info("SUCCESS");
118+
}
119+
}
120+
121+
/**
122+
* Test for upgrading Operator from release 2.0 to develop branch
123+
*
124+
* @throws Exception when upgrade fails
125+
*/
126+
@Test
127+
public void testOperatorUpgradeFrom2_0() throws Exception {
128+
Assume.assumeFalse(QUICKTEST);
129+
String testMethod = new Object() {}.getClass().getEnclosingMethod().getName();
130+
logTestBegin(testMethod);
131+
OP_NS = "weblogic-operator20";
132+
OP_DEP_NAME = "operator-upgrade20";
133+
OP_SA = "operator-sa20";
134+
DOM_NS = "weblogic-domain20";
135+
DUID = "operatordomain20";
136+
setupOperatorAndDomain("2.0", "2.0");
137+
upgradeOperator(true);
138+
logger.info("SUCCESS - " + testMethod);
139+
}
140+
141+
/**
142+
* Test for upgrading Operator from release 2.0.1 to develop branch
143+
*
144+
* @throws Exception when upgrade fails
145+
*/
146+
@Test
147+
public void testOperatorUpgradeFrom2_0_1() throws Exception {
148+
Assume.assumeFalse(QUICKTEST);
149+
String testMethod = new Object() {}.getClass().getEnclosingMethod().getName();
150+
logTestBegin(testMethod);
151+
OP_NS = "weblogic-operator201";
152+
OP_DEP_NAME = "operator-upgrade201";
153+
OP_SA = "operator-sa201";
154+
DOM_NS = "weblogic-domain201";
155+
DUID = "operatordomain201";
156+
setupOperatorAndDomain("release/2.0.1", "2.0.1");
157+
upgradeOperator(true);
158+
logger.info("SUCCESS - " + testMethod);
159+
}
160+
161+
/**
162+
* Test for upgrading Operator from release 2.1 to develop branch
163+
*
164+
* @throws Exception when upgrade fails
165+
*/
166+
@Test
167+
public void testOperatorUpgradeFrom2_1() throws Exception {
168+
Assume.assumeFalse(QUICKTEST);
169+
String testMethod = new Object() {}.getClass().getEnclosingMethod().getName();
170+
logTestBegin(testMethod);
171+
OP_NS = "weblogic-operator21";
172+
OP_DEP_NAME = "operator-upgrade21";
173+
OP_SA = "operator-sa21";
174+
DOM_NS = "weblogic-domain21";
175+
DUID = "operatordomain21";
176+
setupOperatorAndDomain("release/2.1", "2.1");
177+
upgradeOperator(false);
178+
logger.info("SUCCESS - " + testMethod);
179+
}
180+
181+
/**
182+
* Test for upgrading Operator from release 2.2.0 to develop branch
183+
*
184+
* @throws Exception when upgrade fails
185+
*/
186+
@Test
187+
public void testOperatorUpgradeFrom2_2_0() throws Exception {
188+
Assume.assumeFalse(QUICKTEST);
189+
String testMethod = new Object() {}.getClass().getEnclosingMethod().getName();
190+
logTestBegin(testMethod);
191+
OP_NS = "weblogic-operator220";
192+
OP_DEP_NAME = "operator-upgrade220";
193+
OP_SA = "operator-sa220";
194+
DOM_NS = "weblogic-domain220";
195+
DUID = "operatordomain220";
196+
setupOperatorAndDomain("release/2.2", "2.2.0");
197+
upgradeOperator(false);
198+
logger.info("SUCCESS - " + testMethod);
199+
}
200+
201+
/**
202+
* Test for upgrading Operator from release 2.2.1 to develop branch
203+
*
204+
* @throws Exception when upgrade fails
205+
*/
206+
@Test
207+
public void testOperatorUpgradeFrom2_2_1() throws Exception {
208+
Assume.assumeFalse(QUICKTEST);
209+
String testMethod = new Object() {}.getClass().getEnclosingMethod().getName();
210+
logTestBegin(testMethod);
211+
OP_NS = "weblogic-operator221";
212+
OP_DEP_NAME = "operator-upgrade221";
213+
OP_SA = "operator-sa221";
214+
DOM_NS = "weblogic-domain221";
215+
DUID = "operatordomain221";
216+
setupOperatorAndDomain("release/2.2.1", "2.2.1");
217+
upgradeOperator(false);
218+
logger.info("SUCCESS - " + testMethod);
219+
}
220+
221+
/**
222+
* Upgrades operator to develop branch by using the helm upgrade
223+
*
224+
* @param restart boolean parameter used to determine if a restart of domain is checked
225+
* @throws Exception when upgrade fails or basic usecase testing or scaling fails.
226+
*/
227+
private void upgradeOperator(boolean restart) throws Exception {
228+
operator.callHelmUpgrade("image=" + OP_TARGET_RELEASE);
229+
if (restart) {
230+
checkDomainRollingRestarted();
231+
}
232+
checkOperatorVersion();
233+
testBasicUseCases(domain);
234+
testClusterScaling(operator, domain);
235+
}
236+
237+
/**
238+
* checks the expected version of the upgraded operator in a loop. In Jenkins it takes nearly 8
239+
* minutes to show the updated value of the domain CRD
240+
*
241+
* @throws Exception when version does not match
242+
*/
243+
private void checkOperatorVersion() throws Exception {
244+
boolean result = false;
245+
logger.log(
246+
Level.INFO,
247+
"Checking for the domain apiVersion "
248+
+ getDomainApiVersion()
249+
+ " in a loop for up to 15 minutes");
250+
for (int i = 0; i < 900; i = i + 10) {
251+
ExecResult exec =
252+
TestUtils.exec(
253+
"kubectl get domain -n " + DOM_NS + " " + DUID + " -o jsonpath={.apiVersion}", true);
254+
if (exec.stdout().contains(getDomainApiVersion())) {
255+
logger.log(Level.INFO, "Got the expected apiVersion");
256+
result = true;
257+
break;
258+
}
259+
Thread.sleep(1000 * 10);
260+
}
261+
if (!result) {
262+
throw new RuntimeException("FAILURE: Didn't get the expected operator version");
263+
}
264+
}
265+
266+
/**
267+
* Check whether the weblogic server instances are rolling restarted
268+
*
269+
* @throws Exception If restart fails or not restarted
270+
*/
271+
private void checkDomainRollingRestarted() throws Exception {
272+
domain.verifyAdminServerRestarted();
273+
TestUtils.checkPodReady(DUID + "-" + domain.getAdminServerName(), DOM_NS);
274+
for (int i = 2; i >= 1; i--) {
275+
logger.info(
276+
"Checking if managed server pod(" + DUID + "--managed-server" + i + ") is restarted");
277+
TestUtils.checkPodTerminating(DUID + "-managed-server" + i, DOM_NS);
278+
TestUtils.checkPodCreated(DUID + "-managed-server" + i, DOM_NS);
279+
TestUtils.checkPodReady(DUID + "-managed-server" + i, DOM_NS);
280+
}
281+
}
282+
}

‎integration-tests/src/test/java/oracle/kubernetes/operator/utils/Domain.java

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,6 @@
2929
import javax.jms.QueueConnectionFactory;
3030
import javax.naming.Context;
3131
import javax.naming.InitialContext;
32-
3332
import oracle.kubernetes.operator.BaseTest;
3433
import org.yaml.snakeyaml.Yaml;
3534

@@ -116,7 +115,7 @@ public void verifyDomainCreated() throws Exception {
116115
*/
117116
public void verifyPodsCreated() throws Exception {
118117
// check admin pod
119-
logger.info("Checking if admin pod(" + domainUid + "-" + adminServerName + ") is Running");
118+
logger.info("Checking if admin pod(" + domainUid + "-" + adminServerName + ") is Created");
120119
TestUtils.checkPodCreated(domainUid + "-" + adminServerName, domainNS);
121120

122121
if (!serverStartPolicy.equals("ADMIN_ONLY")) {
@@ -128,7 +127,7 @@ public void verifyPodsCreated() throws Exception {
128127
+ "-"
129128
+ managedServerNameBase
130129
+ i
131-
+ ") is Running");
130+
+ ") is Created");
132131
TestUtils.checkPodCreated(domainUid + "-" + managedServerNameBase + i, domainNS);
133132
}
134133
}
@@ -189,13 +188,14 @@ public void verifyServicesCreated(boolean precreateService) throws Exception {
189188
*/
190189
public void verifyServersReady() throws Exception {
191190
// check admin pod
192-
logger.info("Checking if admin server is Running");
191+
logger.info("Checking if admin server is Running and Ready");
193192
TestUtils.checkPodReady(domainUid + "-" + adminServerName, domainNS);
194193

195194
if (!serverStartPolicy.equals("ADMIN_ONLY")) {
196195
// check managed server pods
197196
for (int i = 1; i <= initialManagedServerReplicas; i++) {
198-
logger.info("Checking if managed server (" + managedServerNameBase + i + ") is Running");
197+
logger.info(
198+
"Checking if managed server (" + managedServerNameBase + i + ") is Running and Ready");
199199
TestUtils.checkPodReady(domainUid + "-" + managedServerNameBase + i, domainNS);
200200
}
201201
} else {
@@ -1389,8 +1389,18 @@ protected void initialize(Map<String, Object> inputDomainMap) throws Exception {
13891389
this.projectRoot = BaseTest.getProjectRoot();
13901390

13911391
// copy samples to RESULT_DIR
1392-
TestUtils.exec(
1393-
"cp -rf " + BaseTest.getProjectRoot() + "/kubernetes/samples " + BaseTest.getResultDir());
1392+
if (domainMap.containsKey("projectRoot")) {
1393+
TestUtils.exec(
1394+
"cp -rf "
1395+
+ domainMap.get("projectRoot")
1396+
+ "/kubernetes/samples "
1397+
+ BaseTest.getResultDir(),
1398+
true);
1399+
} else {
1400+
TestUtils.exec(
1401+
"cp -rf " + BaseTest.getProjectRoot() + "/kubernetes/samples " + BaseTest.getResultDir(),
1402+
true);
1403+
}
13941404

13951405
this.voyager =
13961406
(System.getenv("LB_TYPE") != null && System.getenv("LB_TYPE").equalsIgnoreCase("VOYAGER"))

0 commit comments

Comments
 (0)