Skip to content

Commit c3e8350

Browse files
authored
Merge pull request #1131 from oracle/pmackin-OWLS-71966
WIP: OWLS-71966
2 parents a20dad2 + 011c37b commit c3e8350

File tree

31 files changed

+1743
-284
lines changed

31 files changed

+1743
-284
lines changed

docs-source/content/userguide/managing-domains/domain-lifecycle/restarting.md

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -199,3 +199,14 @@ If you've created a new image that is not rolling compatible, and you've changed
199199
the image, or the Kubernetes resources that register your domain with the operator. For example, your servers are caching information from an external database and you've modified the contents of the database.
200200

201201
In these cases, you must manually initiate a restart.
202+
203+
* **Managed Coherence Servers safe shutdown**.
204+
205+
If the domain is configured to use a Coherence cluster, then you will need to increase the Kubernetes graceful timeout value.
206+
When a server is shut down, Coherence needs time to recover partitions and rebalance the cluster before it is safe to shut down a second server.
207+
Using the Kubernetes graceful termination feature, the operator will automatically wait until the Coherence HAStatus MBean attribute
208+
indicates that it is safe to shut down the server. However, after the graceful termination timeout expires, the pod will be deleted regardless.
209+
Therefore, it is important to set the domain YAML `timeoutSeconds` to a large enough value to prevent the server from shutting down before
210+
Coherence is safe. Furthermore, if the operator is not able to access the Coherence MBean, then the server will not be shut down
211+
until the domain `timeoutSeconds` expires. To minimize any possibility of cache data loss, you should increase the `timeoutSeconds`
212+
value to a large number, for example, 15 minutes.
Lines changed: 219 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,219 @@
1+
// Copyright 2019, Oracle Corporation and/or its affiliates. All rights reserved.
2+
// Licensed under the Universal Permissive License v 1.0 as shown at
3+
// http://oss.oracle.com/licenses/upl.
4+
5+
package oracle.kubernetes.operator;
6+
7+
import java.util.HashMap;
8+
import java.util.Map;
9+
10+
import oracle.kubernetes.operator.utils.Domain;
11+
import oracle.kubernetes.operator.utils.Operator;
12+
import oracle.kubernetes.operator.utils.TestUtils;
13+
import org.junit.AfterClass;
14+
import org.junit.Assert;
15+
import org.junit.Assume;
16+
import org.junit.BeforeClass;
17+
import org.junit.FixMethodOrder;
18+
import org.junit.Test;
19+
import org.junit.runners.MethodSorters;
20+
21+
/**
22+
* This class contains Coherence related integration tests.
23+
*/
24+
@FixMethodOrder(MethodSorters.NAME_ASCENDING)
25+
public class ItCoherenceTests extends BaseTest {
26+
27+
private static Domain domain = null;
28+
private static Operator operator1 = null;
29+
30+
private static final String PROXY_CLIENT_SCRIPT = "buildRunProxyClient.sh";
31+
private static final String PROXY_CLIENT_APP_NAME = "coherence-proxy-client";
32+
private static final String PROXY_SERVER_APP_NAME = "coherence-proxy-server";
33+
private static final String OP_CACHE_LOAD = "load";
34+
private static final String OP_CACHE_VALIDATE = "validate";
35+
private static final String PROXY_PORT = "9000";
36+
37+
/**
38+
* This method gets called only once before any of the test methods are executed. It does the
39+
* initialization of the integration test properties defined in OperatorIT.properties and setting
40+
* the resultRoot, pvRoot and projectRoot attributes. Also, create the operator.
41+
*
42+
* @throws Exception exception
43+
*/
44+
@BeforeClass
45+
public static void staticPrepare() throws Exception {
46+
// initialize test properties and create the directories
47+
if (!QUICKTEST) {
48+
initialize(APP_PROPS_FILE);
49+
operator1 = TestUtils.createOperator(OPERATOR1_YAML);
50+
}
51+
}
52+
53+
/**
54+
* Releases k8s cluster lease, archives result, pv directories and destroy the operator.
55+
*
56+
* @throws Exception exception
57+
*/
58+
@AfterClass
59+
public static void staticUnPrepare() throws Exception {
60+
if (!QUICKTEST) {
61+
operator1.destroy();
62+
tearDown(new Object() {}.getClass().getEnclosingClass().getSimpleName());
63+
}
64+
}
65+
66+
@Test
67+
public void testRollingRestart() throws Exception {
68+
Assume.assumeFalse(QUICKTEST);
69+
70+
String testMethodName = new Object() {}.getClass().getEnclosingMethod().getName();
71+
logTestBegin(testMethodName);
72+
73+
domain = createDomain();
74+
Assert.assertNotNull(domain);
75+
76+
try {
77+
// Build and run the proxy client on the admin VM to load the cache
78+
copyAndExecuteProxyClientInPod(OP_CACHE_LOAD);
79+
80+
// Do the rolling restart
81+
restartDomainByChangingEnvProperty();
82+
83+
// Build and run the proxy client on the admin VM to validate the cache
84+
copyAndExecuteProxyClientInPod(OP_CACHE_VALIDATE);
85+
} finally {
86+
destroyDomain();
87+
}
88+
logger.info("SUCCESS - " + testMethodName);
89+
}
90+
91+
/**
92+
* Since the coherence.jar is not open source, we need to build the proxy client on the admin VM, which has the
93+
* coherence.jar. Copy the shell script file and all coherence app files over to the admin pod.
94+
* Then run the script to build the proxy client and run the proxy test.
95+
*
96+
* @param cacheOp - cache operation
97+
*/
98+
private static void copyAndExecuteProxyClientInPod(String cacheOp) {
99+
try {
100+
final String adminServerPod = domain.getDomainUid() + "-" + domain.getAdminServerName();
101+
102+
final String domainNS = domain.getDomainNs();
103+
104+
// Use the proxy running on Managed Server 1, get the internal POD IP
105+
final String podName = domain.getManagedSeverPodName(1);
106+
final String ProxyIP = TestUtils.getPodIP(domainNS, "", podName);
107+
108+
String cohAppLocationOnHost = BaseTest.getAppLocationOnHost() + "/" + PROXY_CLIENT_APP_NAME;
109+
String cohAppLocationInPod = BaseTest.getAppLocationInPod() + "/" + PROXY_CLIENT_APP_NAME;
110+
final String cohScriptPathOnHost = cohAppLocationOnHost + "/" + PROXY_CLIENT_SCRIPT;
111+
final String cohScriptPathInPod = cohAppLocationInPod + "/" + PROXY_CLIENT_SCRIPT;
112+
final String successMarker = "CACHE-SUCCESS";
113+
114+
logger.info("Copying files to admin pod for App " + PROXY_CLIENT_APP_NAME);
115+
116+
// Create app dir in the admin pod
117+
StringBuffer mkdirCmd = new StringBuffer(" -- bash -c 'mkdir -p ");
118+
mkdirCmd.append(cohAppLocationInPod).append("'");
119+
TestUtils.kubectlexec(adminServerPod, domainNS, mkdirCmd.toString());
120+
121+
// Copy shell script to the pod
122+
TestUtils.copyFileViaCat(cohScriptPathOnHost, cohScriptPathInPod, adminServerPod, domainNS);
123+
124+
// Copy all App files to the admin pod
125+
TestUtils.copyAppFilesToPod(
126+
cohAppLocationOnHost, cohAppLocationInPod, adminServerPod, domainNS);
127+
128+
logger.info(
129+
"Executing script "
130+
+ PROXY_CLIENT_SCRIPT
131+
+ " for App "
132+
+ PROXY_CLIENT_APP_NAME
133+
+ " in the admin pod");
134+
135+
// Run the script to on the admin pod (note first arg is app directory is applocation in pod)
136+
domain.callShellScriptInAdminPod(
137+
successMarker, cohScriptPathInPod, cohAppLocationInPod, cacheOp, ProxyIP, PROXY_PORT);
138+
} catch (Exception e) {
139+
throw new RuntimeException(e);
140+
}
141+
}
142+
143+
/**
144+
* Create the domain.
145+
*
146+
* @return domain
147+
* @throws Exception exception
148+
*/
149+
private Domain createDomain() throws Exception {
150+
151+
Map<String, String> envMap = new HashMap<>();
152+
153+
// Set this ENV var with the WDT archive so that it is included in the image build.
154+
envMap.put("CUSTOM_WDT_ARCHIVE", buildProxyServerWdtZip());
155+
156+
// create domain
157+
Map<String, Object> domainMap = TestUtils.loadYaml(DOMAININIMAGE_WDT_YAML);
158+
domainMap.put("namespace", "test1");
159+
domainMap.put("domainUID", "coh");
160+
domainMap.put("additionalEnvMap", envMap);
161+
162+
domainMap.put(
163+
"customWdtTemplate",
164+
BaseTest.getProjectRoot()
165+
+ "/integration-tests/src/test/resources/wdt/coh-wdt-config.yaml");
166+
Domain domain = TestUtils.createDomain(domainMap);
167+
domain.verifyDomainCreated();
168+
return domain;
169+
}
170+
171+
/**
172+
* Destroy the domain.
173+
*
174+
* @throws Exception exception
175+
*/
176+
private static void destroyDomain() throws Exception {
177+
if (domain != null) {
178+
domain.destroy();
179+
}
180+
}
181+
182+
/**
183+
* Modify the domain scope env property on the domain resource using kubectl apply -f domain.yaml
184+
* Verify that all the server pods in the domain got re-started. The property tested is: env:
185+
* "-Dweblogic.StdoutDebugEnabled=false"--> "-Dweblogic.StdoutDebugEnabled=true"
186+
*
187+
* @throws Exception exception
188+
*/
189+
private void restartDomainByChangingEnvProperty() throws Exception {
190+
191+
// The default cmd loop sleep is too long and we could miss states like terminating. Change
192+
// the
193+
// sleep and iterations
194+
//
195+
setWaitTimePod(2);
196+
setMaxIterationsPod(150);
197+
198+
domain.verifyDomainServerPodRestart(
199+
"\"-Dweblogic.StdoutDebugEnabled=false\"", "\"-Dweblogic.StdoutDebugEnabled=true\"");
200+
}
201+
202+
/**
203+
* Build the WDT zip that contains the Coherence proxy server.
204+
*
205+
* @return the WDT zip path
206+
*/
207+
private static String buildProxyServerWdtZip() {
208+
209+
// Build the proxy server gar file
210+
String garPath = getResultDir() + "/coh-proxy-server.gar";
211+
String cohAppLocationOnHost = BaseTest.getAppLocationOnHost() + "/" + PROXY_SERVER_APP_NAME;
212+
TestUtils.buildJarArchive(garPath, cohAppLocationOnHost);
213+
214+
// Build the WDT zip
215+
String wdtArchivePath = getResultDir() + "/coh-wdt-archive.zip";
216+
TestUtils.buildWdtZip(wdtArchivePath, new String[] {garPath}, getResultDir());
217+
return wdtArchivePath;
218+
}
219+
}

integration-tests/src/test/java/oracle/kubernetes/operator/ItElasticLogging.java

Lines changed: 12 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -37,10 +37,10 @@ public class ItElasticLogging extends BaseTest {
3737
private static final String elasticStackYamlLoc =
3838
"kubernetes/samples/scripts/elasticsearch-and-kibana/elasticsearch_and_kibana.yaml";
3939
private final String loggingJarRepos =
40-
"https://github.com/oracle/weblogic-logging-exporter/releases/download/v0.1.1";
40+
"https://github.com/oracle/weblogic-logging-exporter/releases/download/v0.1.1";
4141
private final String wlsLoggingExpJar = "weblogic-logging-exporter-0.1.1.jar";
4242
private final String snakeyamlJarRepos =
43-
"https://repo1.maven.org/maven2/org/yaml/snakeyaml/1.23";
43+
"https://repo1.maven.org/maven2/org/yaml/snakeyaml/1.23";
4444
private final String snakeyamlJar = "snakeyaml-1.23.jar";
4545
private static Operator operator;
4646
private static Domain domain;
@@ -224,13 +224,13 @@ public void testWeblogicLogSearch() throws Exception {
224224
* @throws Exception exception
225225
*/
226226
@Test
227-
public void testWLSLoggingExporter() throws Exception {
227+
public void testWlsLoggingExporter() throws Exception {
228228
Assume.assumeFalse(QUICKTEST);
229229
String testMethodName = new Object() {}.getClass().getEnclosingMethod().getName();
230230
logTestBegin(testMethodName);
231231

232232
// Download Weblogic logging exporter
233-
downloadWLSLoggingExporterJars();
233+
downloadWlsLoggingExporterJars();
234234
// Copy required resources to all wls server pods
235235
copyResourceFilesToAllPods();
236236

@@ -265,7 +265,7 @@ private static void verifyLoggingExpReady(String index) throws Exception {
265265
Assume.assumeNotNull(indexStatus);
266266
Assume.assumeNotNull(indexName);
267267

268-
if(!index.equalsIgnoreCase(kibanaIndexKey)) {
268+
if (!index.equalsIgnoreCase(kibanaIndexKey)) {
269269
// Add the logstash and wls index name to a Map
270270
testVarMap.put(index, indexName);
271271
}
@@ -278,16 +278,14 @@ private static void verifyLoggingExpReady(String index) throws Exception {
278278
String[] indexNameArr =
279279
indexName.split(System.getProperty("line.separator"));
280280

281-
for(int i = 0; i < indexStatusArr.length; i++) {
282-
logger.info("Health status of " + indexNameArr[i] +
283-
" is: " + healthStatusArr[i]);
284-
logger.info("Index status of " + indexNameArr[i] +
285-
" is: " + indexStatusArr[i]);
281+
for (int i = 0; i < indexStatusArr.length; i++) {
282+
logger.info("Health status of " + indexNameArr[i] + " is: " + healthStatusArr[i]);
283+
logger.info("Index status of " + indexNameArr[i] + " is: " + indexStatusArr[i]);
286284
// Verify that the health status of index
287285
Assume.assumeTrue(
288286
index + " is not ready!",
289-
healthStatusArr[i].trim().equalsIgnoreCase("yellow") ||
290-
healthStatusArr[i].trim().equalsIgnoreCase("green"));
287+
healthStatusArr[i].trim().equalsIgnoreCase("yellow")
288+
|| healthStatusArr[i].trim().equalsIgnoreCase("green"));
291289
// Verify that the index is open for use
292290
Assume.assumeTrue(index + " index is not open!",
293291
indexStatusArr[i].trim().equalsIgnoreCase("open"));
@@ -395,7 +393,7 @@ private String execSearchQuery(String queryCriteria, String index) throws Except
395393
return result.stdout();
396394
}
397395

398-
private void downloadWLSLoggingExporterJars() throws Exception {
396+
private void downloadWlsLoggingExporterJars() throws Exception {
399397
File loggingJatReposDir = new File(loggingExpArchiveLoc);
400398

401399
if (loggingJatReposDir.list().length == 0) {
@@ -455,8 +453,7 @@ private void copyResourceFilesToAllPods() throws Exception {
455453

456454
private void copyResourceFilesToOnePod(String serverName, String domainNS)
457455
throws Exception {
458-
String resourceDir =
459-
BaseTest.getProjectRoot() + "/integration-tests/src/test/resources";
456+
String resourceDir = BaseTest.getProjectRoot() + "/integration-tests/src/test/resources";
460457
String testResourceDir = resourceDir + "/loggingexporter";
461458
final String loggingYamlFile = "WebLogicLoggingExporter.yaml";
462459

integration-tests/src/test/java/oracle/kubernetes/operator/ItOperator.java

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -226,10 +226,10 @@ public void testTwoDomainsManagedByTwoOperators() throws Exception {
226226
if (!domainUidsToBeDeleted.equals("")) {
227227
logger.info("About to delete domains: " + domainUidsToBeDeleted);
228228
TestUtils.deleteWeblogicDomainResources(domainUidsToBeDeleted);
229-
if(domain1 != null) {
229+
if (domain1 != null) {
230230
TestUtils.verifyAfterDeletion(domain1);
231231
}
232-
if(domain2 != null) {
232+
if (domain2 != null) {
233233
TestUtils.verifyAfterDeletion(domain2);
234234
}
235235
}
@@ -312,11 +312,11 @@ public void testTwoDomainsManagedByOneOperatorSharingPV() throws Exception {
312312
if (!domainUidsToBeDeleted.equals("")) {
313313
logger.info("About to delete domains: " + domainUidsToBeDeleted);
314314
TestUtils.deleteWeblogicDomainResources(domainUidsToBeDeleted);
315-
if(domain1 != null) {
315+
if (domain1 != null) {
316316
TestUtils.verifyAfterDeletion(domain1);
317317
}
318-
if(domain2 != null) {
319-
TestUtils.verifyAfterDeletion(domain2);
318+
if (domain2 != null) {
319+
TestUtils.verifyAfterDeletion(domain2);
320320
}
321321
}
322322
}

0 commit comments

Comments
 (0)