Skip to content

Commit 01c0c08

Browse files
hzhao-githubhzhao-github
committed
Completed JIRA OWLS-69023 - Verify WLS Logging Exporter using ELK Elasticsearch & Kibana in Operator Env jenkins-ignore
1 parent efb1d34 commit 01c0c08

File tree

6 files changed

+374
-1
lines changed

6 files changed

+374
-1
lines changed

‎integration-tests/USECASES.MD

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -101,4 +101,10 @@ Configuration Overrides Usecases
101101
| Sticky Session | Use Case |
102102
| --- | --- |
103103
| Server affinity | Use a web application deployed on Weblogic cluster to track HTTP session. Test server-affinity by sending two HTTP requests to Weblogic and verify that all requests are directed to same Weblogic server |
104-
| Session state isolation | Verify that values saved in a client session state are not visible to another client |
104+
| Session state isolation | Verify that values saved in a client session state are not visible to another client |
105+
106+
| Logging with Elastic Stack | Use Case |
107+
| --- | --- |
108+
| Search log level | Use Elasticsearch Count API to query logs of level=INFO and verify that total number of logs for level=INFO is not zero and failed count is zero |
109+
| Search Operator log | Use Elasticsearch Search APIs to query Operator log info and verify that log hits for type=weblogic-operator are not empty |
110+
| Search Weblogic log | Use Elasticsearch Search APIs to query Weblogic log info and verify that log hits for Weblogic servers are not empty |

‎integration-tests/src/test/java/oracle/kubernetes/operator/BaseTest.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ public class BaseTest {
3535
public static final String OPERATOR2_YAML = "operator2.yaml";
3636
public static final String OPERATORBC_YAML = "operator_bc.yaml";
3737
public static final String OPERATOR_CHAIN_YAML = "operator_chain.yaml";
38+
public static final String OPERATOR1_ELK_YAML = "operator_elk.yaml";
3839

3940
// file used to customize domain properties for domain, PV and LB inputs yaml
4041
public static final String DOMAINONPV_WLST_YAML = "domainonpvwlst.yaml";

‎integration-tests/src/test/java/oracle/kubernetes/operator/ITElasticLogging.java

Lines changed: 287 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,287 @@
1+
// Copyright 2019, Oracle Corporation and/or its affiliates. All rights reserved.
2+
// Licensed under the Universal Permissive License v 1.0 as shown at
3+
// http://oss.oracle.com/licenses/upl.
4+
5+
package oracle.kubernetes.operator;
6+
7+
import java.util.Map;
8+
import java.util.regex.Matcher;
9+
import java.util.regex.Pattern;
10+
import oracle.kubernetes.operator.utils.Domain;
11+
import oracle.kubernetes.operator.utils.ExecResult;
12+
import oracle.kubernetes.operator.utils.Operator;
13+
import oracle.kubernetes.operator.utils.TestUtils;
14+
import org.junit.AfterClass;
15+
import org.junit.Assume;
16+
import org.junit.BeforeClass;
17+
import org.junit.FixMethodOrder;
18+
import org.junit.Test;
19+
import org.junit.runners.MethodSorters;
20+
21+
/**
22+
* Simple JUnit test file used for testing Operator.
23+
*
24+
* <p>This test is used for testing operator working with Elastic Stack
25+
*/
26+
@FixMethodOrder(MethodSorters.NAME_ASCENDING)
27+
public class ITElasticLogging extends BaseTest {
28+
private static Operator operator;
29+
private static Domain domain;
30+
private static String k8sExecCmdPrefix;
31+
private static String elasticSearchURL;
32+
private static Map<String, Object> testVarMap;
33+
34+
/**
35+
* This method gets called only once before any of the test methods are executed. It does the
36+
* initialization of the integration test properties defined in OperatorIT.properties and setting
37+
* the resultRoot, pvRoot and projectRoot attributes. It installs Elastic Stack, verifies Elastic
38+
* Stack is ready to use, creates an operator and a Weblogic domain
39+
*
40+
* @throws Exception
41+
*/
42+
@BeforeClass
43+
public static void staticPrepare() throws Exception {
44+
if (!QUICKTEST) {
45+
// initialize test properties and create the directories
46+
initialize(APP_PROPS_FILE);
47+
48+
// Install Elastic Stack
49+
String cmd =
50+
"kubectl apply -f "
51+
+ getProjectRoot()
52+
+ "/kubernetes/samples/scripts/elasticsearch-and-kibana/elasticsearch_and_kibana.yaml";
53+
logger.info("Command to install Elastic search and Kibana: " + cmd);
54+
TestUtils.exec(cmd);
55+
56+
// Create operator-elk
57+
if (operator == null) {
58+
logger.info("Creating Operator & waiting for the script to complete execution");
59+
operator = TestUtils.createOperator(OPERATOR1_ELK_YAML, "2/2");
60+
}
61+
62+
// create domain
63+
if (domain == null) {
64+
logger.info("Creating WLS Domain & waiting for the script to complete execution");
65+
domain = TestUtils.createDomain(DOMAINONPV_WLST_YAML);
66+
domain.verifyDomainCreated();
67+
}
68+
69+
// Get Elasticsearch host and port from yaml file and build Elasticsearch URL
70+
testVarMap = TestUtils.loadYaml(OPERATOR1_ELK_YAML);
71+
String operatorPodName = operator.getOperatorPodName();
72+
elasticSearchURL =
73+
"http://"
74+
+ testVarMap.get("elasticSearchHost")
75+
+ ":"
76+
+ testVarMap.get("elasticSearchPort");
77+
Assume.assumeFalse(
78+
"Got null when building Elasticsearch URL", elasticSearchURL.contains("null"));
79+
80+
// Create the prefix of k8s exec command
81+
StringBuffer k8sExecCmdPrefixBuff =
82+
new StringBuffer("kubectl exec -it ")
83+
.append(operatorPodName)
84+
.append(" -n ")
85+
.append(operator.getOperatorNamespace())
86+
.append(" -- /bin/bash -c ")
87+
.append("'curl ")
88+
.append(elasticSearchURL);
89+
k8sExecCmdPrefix = k8sExecCmdPrefixBuff.toString();
90+
91+
// Verify that Elastic Stack is ready to use
92+
verifyElasticStackReady();
93+
}
94+
}
95+
96+
/**
97+
* Releases k8s cluster lease, archives result, pv directories and uninstall Elastic Stack
98+
*
99+
* @throws Exception
100+
*/
101+
@AfterClass
102+
public static void staticUnPrepare() throws Exception {
103+
if (!QUICKTEST) {
104+
logger.info("+++++++++++++++++++++++++++++++++---------------------------------+");
105+
logger.info("BEGIN");
106+
logger.info("Run once, release cluster lease");
107+
108+
tearDown(new Object() {}.getClass().getEnclosingClass().getSimpleName());
109+
110+
// Uninstall Elastic Stack
111+
String cmd =
112+
"kubectl delete -f "
113+
+ getProjectRoot()
114+
+ "/kubernetes/samples/scripts/elasticsearch-and-kibana/elasticsearch_and_kibana.yaml";
115+
logger.info("Command to uninstall Elastic Stack: " + cmd);
116+
TestUtils.exec(cmd);
117+
118+
logger.info("SUCCESS");
119+
}
120+
}
121+
122+
/**
123+
* Use Elasticsearch Count API to query logs of level=INFO. Verify that total number of logs for
124+
* level=INFO is not zero and failed count is zero
125+
*
126+
* @throws Exception
127+
*/
128+
@Test
129+
public void testLogLevelSearch() throws Exception {
130+
Assume.assumeFalse(QUICKTEST);
131+
String testMethodName = new Object() {}.getClass().getEnclosingMethod().getName();
132+
logTestBegin(testMethodName);
133+
134+
// Verify that number of logs is not zero and failed count is zero
135+
String regex = ".*count\":(\\d+),.*failed\":(\\d+)";
136+
String queryCriteria = "/_count?q=level:INFO";
137+
verifySearchResults(queryCriteria, regex, true);
138+
139+
logger.info("SUCCESS - " + testMethodName);
140+
}
141+
142+
/**
143+
* Use Elasticsearch Search APIs to query Operator log info. Verify that log hits for
144+
* type=weblogic-operator are not empty
145+
*
146+
* @throws Exception
147+
*/
148+
@Test
149+
public void testOperatorLogSearch() throws Exception {
150+
Assume.assumeFalse(QUICKTEST);
151+
String testMethodName = new Object() {}.getClass().getEnclosingMethod().getName();
152+
logTestBegin(testMethodName);
153+
154+
// Verify that log hits for Operator are not empty
155+
String regex = ".*took\":(\\d+),.*hits\":\\{(.+)\\}";
156+
String queryCriteria = "/_search?q=type:weblogic-operator";
157+
verifySearchResults(queryCriteria, regex, false);
158+
159+
logger.info("SUCCESS - " + testMethodName);
160+
}
161+
162+
/**
163+
* Use Elasticsearch Search APIs to query Weblogic log info. Verify that log hits for Weblogic
164+
* servers are not empty
165+
*
166+
* @throws Exception
167+
*/
168+
@Test
169+
public void testWeblogicLogSearch() throws Exception {
170+
Assume.assumeFalse(QUICKTEST);
171+
String testMethodName = new Object() {}.getClass().getEnclosingMethod().getName();
172+
logTestBegin(testMethodName);
173+
174+
Map<String, Object> domainMap = domain.getDomainMap();
175+
String domainUid = domain.getDomainUid();
176+
String adminServerName = (String) domainMap.get("adminServerName");
177+
String adminServerPodName = domainUid + "-" + adminServerName;
178+
String managedServerNameBase = domainMap.get("managedServerNameBase").toString();
179+
String managedServerPodName = domainUid + "-" + managedServerNameBase + "1";
180+
181+
// Verify that log hits for admin server are not empty
182+
String regex = ".*took\":(\\d+),.*hits\":\\{(.+)\\}";
183+
String queryCriteria = "/_search?q=log:" + adminServerPodName + " | grep RUNNING";
184+
verifySearchResults(queryCriteria, regex, false);
185+
186+
// Verify that log hits for managed server are not empty
187+
queryCriteria = "/_search?q=log:" + managedServerPodName + " | grep RUNNING";
188+
verifySearchResults(queryCriteria, regex, false);
189+
190+
logger.info("SUCCESS - " + testMethodName);
191+
}
192+
193+
private void verifySearchResults(String queryCriteria, String regex, boolean checkCount)
194+
throws Exception {
195+
String results = execElasticStackQuery(queryCriteria);
196+
197+
int count = -1;
198+
int failedCount = -1;
199+
String hits = "";
200+
Pattern pattern = Pattern.compile(regex);
201+
Matcher matcher = pattern.matcher(results);
202+
if (matcher.find()) {
203+
count = Integer.parseInt(matcher.group(1));
204+
if (checkCount) {
205+
failedCount = Integer.parseInt(matcher.group(2));
206+
} else {
207+
hits = matcher.group(2);
208+
}
209+
}
210+
211+
Assume.assumeTrue("Total count of logs should be more than 0!", count > 0);
212+
logger.info("Total count of logs: " + count);
213+
if (checkCount) {
214+
Assume.assumeTrue("Total failed count should be 0!", failedCount == 0);
215+
logger.info("Total failed count: " + failedCount);
216+
} else {
217+
Assume.assumeFalse("Total failed count should be 0!", hits.isEmpty());
218+
}
219+
}
220+
221+
private String execElasticStackQuery(String queryCriteria) throws Exception {
222+
StringBuffer k8sExecCmdPrefixBuff = new StringBuffer(k8sExecCmdPrefix);
223+
int offset = k8sExecCmdPrefixBuff.indexOf("http");
224+
k8sExecCmdPrefixBuff.insert(offset, " -X GET ");
225+
String indexName = (String) testVarMap.get("indexName");
226+
String cmd =
227+
k8sExecCmdPrefixBuff
228+
.append("/")
229+
.append(indexName)
230+
.append(queryCriteria)
231+
.append("'")
232+
.toString();
233+
logger.info("Command to search: " + cmd);
234+
ExecResult result = TestUtils.exec(cmd);
235+
236+
return result.stdout();
237+
}
238+
239+
private static void verifyElasticStackReady() throws Exception {
240+
// Get Logstash info
241+
String healthStatus = execElasticStackStatusCheck("*logstash*", "$1");
242+
String indexStatus = execElasticStackStatusCheck("*logstash*", "$2");
243+
String indexName = execElasticStackStatusCheck("*logstash*", "$3");
244+
245+
// Verify that the health status of Logstash
246+
Assume.assumeNotNull(healthStatus);
247+
Assume.assumeTrue(
248+
"Logstash is not ready!",
249+
healthStatus.equalsIgnoreCase("yellow") || healthStatus.equalsIgnoreCase("green"));
250+
// Verify that the index is open for use
251+
Assume.assumeNotNull(indexStatus);
252+
Assume.assumeTrue("Logstash index is not open!", indexStatus.equalsIgnoreCase("open"));
253+
// Add the index name to a Map
254+
Assume.assumeNotNull(indexName);
255+
testVarMap.put("indexName", indexName);
256+
257+
// Get Kibana info
258+
healthStatus = execElasticStackStatusCheck("*kibana*", "$1");
259+
indexStatus = execElasticStackStatusCheck("*kibana*", "$2");
260+
261+
// Verify that the health status of Kibana
262+
Assume.assumeNotNull(healthStatus);
263+
Assume.assumeTrue(
264+
"Kibana is not ready!",
265+
healthStatus.equalsIgnoreCase("yellow") || healthStatus.equalsIgnoreCase("green"));
266+
// Verify that the index is open for use
267+
Assume.assumeNotNull(indexStatus);
268+
Assume.assumeTrue("Kibana index is not open!", indexStatus.equalsIgnoreCase("open"));
269+
}
270+
271+
private static String execElasticStackStatusCheck(String indexName, String varLoc)
272+
throws Exception {
273+
StringBuffer k8sExecCmdPrefixBuff = new StringBuffer(k8sExecCmdPrefix);
274+
String cmd =
275+
k8sExecCmdPrefixBuff
276+
.append("/_cat/indices/")
277+
.append(indexName)
278+
.append(" | awk '\\''{ print ")
279+
.append(varLoc)
280+
.append(" }'\\'")
281+
.toString();
282+
logger.info("Command to exec Elastic Stack status check: " + cmd);
283+
ExecResult result = TestUtils.exec(cmd);
284+
285+
return result.stdout();
286+
}
287+
}

‎integration-tests/src/test/java/oracle/kubernetes/operator/utils/Operator.java

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -142,6 +142,18 @@ public void verifyOperatorReady() throws Exception {
142142
TestUtils.checkPodReady("", operatorNS);
143143
}
144144

145+
/**
146+
* verifies operator pod is ready
147+
*
148+
* @param containerNum - container number in a pod
149+
* @throws Exception
150+
*/
151+
public void verifyOperatorReady(String containerNum) throws Exception {
152+
logger.info("Checking if Operator pod is Ready");
153+
// empty string for pod name as there is only one pod
154+
TestUtils.checkPodReady("", operatorNS, containerNum);
155+
}
156+
145157
/**
146158
* Start operator and makes sure it is deployed and ready
147159
*
@@ -566,4 +578,21 @@ public String getUserProjectsDir() {
566578
public RESTCertType getRestCertType() {
567579
return restCertType;
568580
}
581+
582+
/**
583+
* Retrieve Operator pod name
584+
*
585+
* @return Operator pod name
586+
* @throws Exception
587+
*/
588+
public String getOperatorPodName() throws Exception {
589+
String cmd =
590+
"kubectl get pod -n "
591+
+ getOperatorNamespace()
592+
+ " -o jsonpath=\"{.items[0].metadata.name}\"";
593+
logger.info("Command to query Operator pod name: " + cmd);
594+
ExecResult result = TestUtils.exec(cmd);
595+
596+
return result.stdout();
597+
}
569598
}

0 commit comments

Comments
 (0)