Skip to content

Commit 42350d6

Browse files
rjeberhardrjeberhard
authored
Merge pull request #1064 from oracle/logging-elastic-tests
completed OWLS-74156 - test automation for WLS Logging using Elastic Stack in Operator Env
2 parents 0256d80 + 4977581 commit 42350d6

File tree

6 files changed

+380
-1
lines changed

6 files changed

+380
-1
lines changed

‎integration-tests/USECASES.MD

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -101,4 +101,10 @@ Configuration Overrides Usecases
101101
| Sticky Session | Use Case |
102102
| --- | --- |
103103
| Server affinity | Use a web application deployed on Weblogic cluster to track HTTP session. Test server-affinity by sending two HTTP requests to Weblogic and verify that all requests are directed to same Weblogic server |
104-
| Session state isolation | Verify that values saved in a client session state are not visible to another client |
104+
| Session state isolation | Verify that values saved in a client session state are not visible to another client |
105+
106+
| Logging with Elastic Stack | Use Case |
107+
| --- | --- |
108+
| Search log level | Use Elasticsearch Count API to query logs of level=INFO and verify that total number of logs for level=INFO is not zero and failed count is zero |
109+
| Search Operator log | Use Elasticsearch Search APIs to query Operator log info and verify that log hits for type=weblogic-operator are not empty |
110+
| Search Weblogic log | Use Elasticsearch Search APIs to query Weblogic log info and verify that log hits for Weblogic servers are not empty |

‎integration-tests/src/test/java/oracle/kubernetes/operator/BaseTest.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ public class BaseTest {
3535
public static final String OPERATOR2_YAML = "operator2.yaml";
3636
public static final String OPERATORBC_YAML = "operator_bc.yaml";
3737
public static final String OPERATOR_CHAIN_YAML = "operator_chain.yaml";
38+
public static final String OPERATOR1_ELK_YAML = "operator_elk.yaml";
3839

3940
// file used to customize domain properties for domain, PV and LB inputs yaml
4041
public static final String DOMAINONPV_WLST_YAML = "domainonpvwlst.yaml";

‎integration-tests/src/test/java/oracle/kubernetes/operator/ITElasticLogging.java

Lines changed: 293 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,293 @@
1+
// Copyright 2019, Oracle Corporation and/or its affiliates. All rights reserved.
2+
// Licensed under the Universal Permissive License v 1.0 as shown at
3+
// http://oss.oracle.com/licenses/upl.
4+
5+
package oracle.kubernetes.operator;
6+
7+
import java.util.Map;
8+
import java.util.regex.Matcher;
9+
import java.util.regex.Pattern;
10+
import oracle.kubernetes.operator.utils.Domain;
11+
import oracle.kubernetes.operator.utils.ExecResult;
12+
import oracle.kubernetes.operator.utils.Operator;
13+
import oracle.kubernetes.operator.utils.TestUtils;
14+
import org.junit.AfterClass;
15+
import org.junit.Assume;
16+
import org.junit.BeforeClass;
17+
import org.junit.FixMethodOrder;
18+
import org.junit.Test;
19+
import org.junit.runners.MethodSorters;
20+
21+
/**
22+
* Simple JUnit test file used for testing Operator.
23+
*
24+
* <p>This test is used for testing operator working with Elastic Stack
25+
*/
26+
@FixMethodOrder(MethodSorters.NAME_ASCENDING)
27+
public class ITElasticLogging extends BaseTest {
28+
private static Operator operator;
29+
private static Domain domain;
30+
private static String k8sExecCmdPrefix;
31+
private static String elasticSearchURL;
32+
private static Map<String, Object> testVarMap;
33+
private static final String elasticStackYamlLoc =
34+
"kubernetes/samples/scripts/elasticsearch-and-kibana/elasticsearch_and_kibana.yaml";
35+
36+
/**
37+
* This method gets called only once before any of the test methods are executed. It does the
38+
* initialization of the integration test properties defined in OperatorIT.properties and setting
39+
* the resultRoot, pvRoot and projectRoot attributes. It installs Elastic Stack, verifies Elastic
40+
* Stack is ready to use, creates an operator and a Weblogic domain
41+
*
42+
* @throws Exception
43+
*/
44+
@BeforeClass
45+
public static void staticPrepare() throws Exception {
46+
if (!QUICKTEST) {
47+
// initialize test properties and create the directories
48+
initialize(APP_PROPS_FILE);
49+
50+
// Install Elastic Stack
51+
StringBuffer cmd =
52+
new StringBuffer("kubectl apply -f ")
53+
.append(getProjectRoot())
54+
.append("/")
55+
.append(elasticStackYamlLoc);
56+
logger.info("Command to Install Elastic Stack: " + cmd.toString());
57+
TestUtils.exec(cmd.toString());
58+
59+
// Create operator-elk
60+
if (operator == null) {
61+
logger.info("Creating Operator & waiting for the script to complete execution");
62+
operator = TestUtils.createOperator(OPERATOR1_ELK_YAML, "2/2");
63+
}
64+
65+
// create domain
66+
if (domain == null) {
67+
logger.info("Creating WLS Domain & waiting for the script to complete execution");
68+
domain = TestUtils.createDomain(DOMAINONPV_WLST_YAML);
69+
domain.verifyDomainCreated();
70+
}
71+
72+
// Get Elasticsearch host and port from yaml file and build Elasticsearch URL
73+
testVarMap = TestUtils.loadYaml(OPERATOR1_ELK_YAML);
74+
String operatorPodName = operator.getOperatorPodName();
75+
StringBuffer elasticSearchURLBuff =
76+
new StringBuffer("http://")
77+
.append(testVarMap.get("elasticSearchHost"))
78+
.append(":")
79+
.append(testVarMap.get("elasticSearchPort"));
80+
elasticSearchURL = elasticSearchURLBuff.toString();
81+
Assume.assumeFalse(
82+
"Got null when building Elasticsearch URL", elasticSearchURL.contains("null"));
83+
84+
// Create the prefix of k8s exec command
85+
StringBuffer k8sExecCmdPrefixBuff =
86+
new StringBuffer("kubectl exec -it ")
87+
.append(operatorPodName)
88+
.append(" -n ")
89+
.append(operator.getOperatorNamespace())
90+
.append(" -- /bin/bash -c ")
91+
.append("'curl ")
92+
.append(elasticSearchURL);
93+
k8sExecCmdPrefix = k8sExecCmdPrefixBuff.toString();
94+
95+
// Verify that Elastic Stack is ready to use
96+
verifyElasticStackReady();
97+
}
98+
}
99+
100+
/**
101+
* Releases k8s cluster lease, archives result, pv directories and uninstall Elastic Stack
102+
*
103+
* @throws Exception
104+
*/
105+
@AfterClass
106+
public static void staticUnPrepare() throws Exception {
107+
if (!QUICKTEST) {
108+
logger.info("+++++++++++++++++++++++++++++++++---------------------------------+");
109+
logger.info("BEGIN");
110+
logger.info("Run once, release cluster lease");
111+
112+
// Uninstall Elastic Stack
113+
StringBuffer cmd =
114+
new StringBuffer("kubectl delete -f ")
115+
.append(getProjectRoot())
116+
.append("/")
117+
.append(elasticStackYamlLoc);
118+
logger.info("Command to uninstall Elastic Stack: " + cmd.toString());
119+
TestUtils.exec(cmd.toString());
120+
121+
tearDown(new Object() {}.getClass().getEnclosingClass().getSimpleName());
122+
123+
logger.info("SUCCESS");
124+
}
125+
}
126+
127+
/**
128+
* Use Elasticsearch Count API to query logs of level=INFO. Verify that total number of logs for
129+
* level=INFO is not zero and failed count is zero
130+
*
131+
* @throws Exception
132+
*/
133+
@Test
134+
public void testLogLevelSearch() throws Exception {
135+
Assume.assumeFalse(QUICKTEST);
136+
String testMethodName = new Object() {}.getClass().getEnclosingMethod().getName();
137+
logTestBegin(testMethodName);
138+
139+
// Verify that number of logs is not zero and failed count is zero
140+
String regex = ".*count\":(\\d+),.*failed\":(\\d+)";
141+
String queryCriteria = "/_count?q=level:INFO";
142+
verifySearchResults(queryCriteria, regex, true);
143+
144+
logger.info("SUCCESS - " + testMethodName);
145+
}
146+
147+
/**
148+
* Use Elasticsearch Search APIs to query Operator log info. Verify that log hits for
149+
* type=weblogic-operator are not empty
150+
*
151+
* @throws Exception
152+
*/
153+
@Test
154+
public void testOperatorLogSearch() throws Exception {
155+
Assume.assumeFalse(QUICKTEST);
156+
String testMethodName = new Object() {}.getClass().getEnclosingMethod().getName();
157+
logTestBegin(testMethodName);
158+
159+
// Verify that log hits for Operator are not empty
160+
String regex = ".*took\":(\\d+),.*hits\":\\{(.+)\\}";
161+
String queryCriteria = "/_search?q=type:weblogic-operator";
162+
verifySearchResults(queryCriteria, regex, false);
163+
164+
logger.info("SUCCESS - " + testMethodName);
165+
}
166+
167+
/**
168+
* Use Elasticsearch Search APIs to query Weblogic log info. Verify that log hits for Weblogic
169+
* servers are not empty
170+
*
171+
* @throws Exception
172+
*/
173+
@Test
174+
public void testWeblogicLogSearch() throws Exception {
175+
Assume.assumeFalse(QUICKTEST);
176+
String testMethodName = new Object() {}.getClass().getEnclosingMethod().getName();
177+
logTestBegin(testMethodName);
178+
179+
Map<String, Object> domainMap = domain.getDomainMap();
180+
String domainUid = domain.getDomainUid();
181+
String adminServerName = (String) domainMap.get("adminServerName");
182+
String adminServerPodName = domainUid + "-" + adminServerName;
183+
String managedServerNameBase = domainMap.get("managedServerNameBase").toString();
184+
String managedServerPodName = domainUid + "-" + managedServerNameBase + "1";
185+
186+
// Verify that log hits for admin server are not empty
187+
String regex = ".*took\":(\\d+),.*hits\":\\{(.+)\\}";
188+
String queryCriteria = "/_search?q=log:" + adminServerPodName + " | grep RUNNING";
189+
verifySearchResults(queryCriteria, regex, false);
190+
191+
// Verify that log hits for managed server are not empty
192+
queryCriteria = "/_search?q=log:" + managedServerPodName + " | grep RUNNING";
193+
verifySearchResults(queryCriteria, regex, false);
194+
195+
logger.info("SUCCESS - " + testMethodName);
196+
}
197+
198+
private void verifySearchResults(String queryCriteria, String regex, boolean checkCount)
199+
throws Exception {
200+
String results = execElasticStackQuery(queryCriteria);
201+
202+
int count = -1;
203+
int failedCount = -1;
204+
String hits = "";
205+
Pattern pattern = Pattern.compile(regex);
206+
Matcher matcher = pattern.matcher(results);
207+
if (matcher.find()) {
208+
count = Integer.parseInt(matcher.group(1));
209+
if (checkCount) {
210+
failedCount = Integer.parseInt(matcher.group(2));
211+
} else {
212+
hits = matcher.group(2);
213+
}
214+
}
215+
216+
Assume.assumeTrue("Total count of logs should be more than 0!", count > 0);
217+
logger.info("Total count of logs: " + count);
218+
if (checkCount) {
219+
Assume.assumeTrue("Total failed count should be 0!", failedCount == 0);
220+
logger.info("Total failed count: " + failedCount);
221+
} else {
222+
Assume.assumeFalse("Total hits of search is empty!", hits.isEmpty());
223+
}
224+
}
225+
226+
private String execElasticStackQuery(String queryCriteria) throws Exception {
227+
StringBuffer k8sExecCmdPrefixBuff = new StringBuffer(k8sExecCmdPrefix);
228+
int offset = k8sExecCmdPrefixBuff.indexOf("http");
229+
k8sExecCmdPrefixBuff.insert(offset, " -X GET ");
230+
String indexName = (String) testVarMap.get("indexName");
231+
String cmd =
232+
k8sExecCmdPrefixBuff
233+
.append("/")
234+
.append(indexName)
235+
.append(queryCriteria)
236+
.append("'")
237+
.toString();
238+
logger.info("Command to search: " + cmd);
239+
ExecResult result = TestUtils.exec(cmd);
240+
241+
return result.stdout();
242+
}
243+
244+
private static void verifyElasticStackReady() throws Exception {
245+
// Get Logstash info
246+
String healthStatus = execElasticStackStatusCheck("*logstash*", "$1");
247+
String indexStatus = execElasticStackStatusCheck("*logstash*", "$2");
248+
String indexName = execElasticStackStatusCheck("*logstash*", "$3");
249+
250+
// Verify that the health status of Logstash
251+
Assume.assumeNotNull(healthStatus);
252+
Assume.assumeTrue(
253+
"Logstash is not ready!",
254+
healthStatus.equalsIgnoreCase("yellow") || healthStatus.equalsIgnoreCase("green"));
255+
// Verify that the index is open for use
256+
Assume.assumeNotNull(indexStatus);
257+
Assume.assumeTrue("Logstash index is not open!", indexStatus.equalsIgnoreCase("open"));
258+
// Add the index name to a Map
259+
Assume.assumeNotNull(indexName);
260+
testVarMap.put("indexName", indexName);
261+
262+
// Get Kibana info
263+
healthStatus = execElasticStackStatusCheck("*kibana*", "$1");
264+
indexStatus = execElasticStackStatusCheck("*kibana*", "$2");
265+
266+
// Verify that the health status of Kibana
267+
Assume.assumeNotNull(healthStatus);
268+
Assume.assumeTrue(
269+
"Kibana is not ready!",
270+
healthStatus.equalsIgnoreCase("yellow") || healthStatus.equalsIgnoreCase("green"));
271+
// Verify that the index is open for use
272+
Assume.assumeNotNull(indexStatus);
273+
Assume.assumeTrue("Kibana index is not open!", indexStatus.equalsIgnoreCase("open"));
274+
}
275+
276+
private static String execElasticStackStatusCheck(String indexName, String varLoc)
277+
throws Exception {
278+
StringBuffer k8sExecCmdPrefixBuff = new StringBuffer(k8sExecCmdPrefix);
279+
String cmd =
280+
k8sExecCmdPrefixBuff
281+
.append("/_cat/indices/")
282+
.append(indexName)
283+
.append(" | awk '\\''{ print ")
284+
.append(varLoc)
285+
.append(" }'\\'")
286+
.toString();
287+
logger.info("Command to exec Elastic Stack status check: " + cmd);
288+
ExecResult result = TestUtils.exec(cmd);
289+
logger.info("Results: " + result.stdout());
290+
291+
return result.stdout();
292+
}
293+
}

‎integration-tests/src/test/java/oracle/kubernetes/operator/utils/Operator.java

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -142,6 +142,18 @@ public void verifyOperatorReady() throws Exception {
142142
TestUtils.checkPodReady("", operatorNS);
143143
}
144144

145+
/**
146+
* verifies operator pod is ready
147+
*
148+
* @param containerNum - container number in a pod
149+
* @throws Exception
150+
*/
151+
public void verifyOperatorReady(String containerNum) throws Exception {
152+
logger.info("Checking if Operator pod is Ready");
153+
// empty string for pod name as there is only one pod
154+
TestUtils.checkPodReady("", operatorNS, containerNum);
155+
}
156+
145157
/**
146158
* Start operator and makes sure it is deployed and ready
147159
*
@@ -566,4 +578,21 @@ public String getUserProjectsDir() {
566578
public RESTCertType getRestCertType() {
567579
return restCertType;
568580
}
581+
582+
/**
583+
* Retrieve Operator pod name
584+
*
585+
* @return Operator pod name
586+
* @throws Exception
587+
*/
588+
public String getOperatorPodName() throws Exception {
589+
String cmd =
590+
"kubectl get pod -n "
591+
+ getOperatorNamespace()
592+
+ " -o jsonpath=\"{.items[0].metadata.name}\"";
593+
logger.info("Command to query Operator pod name: " + cmd);
594+
ExecResult result = TestUtils.exec(cmd);
595+
596+
return result.stdout();
597+
}
569598
}

0 commit comments

Comments
 (0)