Skip to content

Commit 67cb02b

Browse files
rjeberhardrjeberhard
committed
Merge branch 'elk-sample-op-oci' into 'main'
Added Integration tests for Elastic-stack sample on Opeator See merge request weblogic-cloud/weblogic-kubernetes-operator!4258
2 parents 25480ba + dc0d7a8 commit 67cb02b

File tree

2 files changed

+279
-3
lines changed

2 files changed

+279
-3
lines changed

‎integration-tests/src/test/java/oracle/weblogic/kubernetes/ItElasticLoggingSample.java

Lines changed: 272 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,272 @@
1+
// Copyright (c) 2023, Oracle and/or its affiliates.
2+
// Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
3+
4+
package oracle.weblogic.kubernetes;
5+
6+
import java.nio.file.Files;
7+
import java.nio.file.Path;
8+
import java.nio.file.Paths;
9+
import java.nio.file.StandardCopyOption;
10+
import java.util.HashMap;
11+
import java.util.List;
12+
import java.util.Map;
13+
import java.util.regex.Matcher;
14+
import java.util.regex.Pattern;
15+
16+
import oracle.weblogic.kubernetes.actions.impl.OperatorParams;
17+
import oracle.weblogic.kubernetes.actions.impl.primitive.Command;
18+
import oracle.weblogic.kubernetes.actions.impl.primitive.CommandParams;
19+
import oracle.weblogic.kubernetes.actions.impl.primitive.HelmParams;
20+
import oracle.weblogic.kubernetes.annotations.IntegrationTest;
21+
import oracle.weblogic.kubernetes.annotations.Namespaces;
22+
import oracle.weblogic.kubernetes.logging.LoggingFacade;
23+
import oracle.weblogic.kubernetes.utils.ExecResult;
24+
import org.junit.jupiter.api.AfterAll;
25+
import org.junit.jupiter.api.BeforeAll;
26+
import org.junit.jupiter.api.DisplayName;
27+
import org.junit.jupiter.api.Tag;
28+
import org.junit.jupiter.api.Test;
29+
30+
import static oracle.weblogic.kubernetes.TestConstants.BUSYBOX_IMAGE;
31+
import static oracle.weblogic.kubernetes.TestConstants.BUSYBOX_TAG;
32+
import static oracle.weblogic.kubernetes.TestConstants.ELASTICSEARCH_HTTP_PORT;
33+
import static oracle.weblogic.kubernetes.TestConstants.ELASTICSEARCH_IMAGE;
34+
import static oracle.weblogic.kubernetes.TestConstants.KIBANA_IMAGE;
35+
import static oracle.weblogic.kubernetes.TestConstants.KIBANA_INDEX_KEY;
36+
import static oracle.weblogic.kubernetes.TestConstants.KUBERNETES_CLI;
37+
import static oracle.weblogic.kubernetes.TestConstants.LOGSTASH_INDEX_KEY;
38+
import static oracle.weblogic.kubernetes.TestConstants.OPERATOR_CHART_DIR;
39+
import static oracle.weblogic.kubernetes.TestConstants.OPERATOR_RELEASE_NAME;
40+
import static oracle.weblogic.kubernetes.actions.ActionConstants.ITTESTS_DIR;
41+
import static oracle.weblogic.kubernetes.actions.ActionConstants.WORK_DIR;
42+
import static oracle.weblogic.kubernetes.actions.TestActions.execCommand;
43+
import static oracle.weblogic.kubernetes.actions.TestActions.getOperatorPodName;
44+
import static oracle.weblogic.kubernetes.assertions.TestAssertions.operatorIsReady;
45+
import static oracle.weblogic.kubernetes.utils.CommonTestUtils.testUntil;
46+
import static oracle.weblogic.kubernetes.utils.CommonTestUtils.withStandardRetryPolicy;
47+
import static oracle.weblogic.kubernetes.utils.FileUtils.replaceStringInFile;
48+
import static oracle.weblogic.kubernetes.utils.LoggingExporterUtils.verifyLoggingExporterReady;
49+
import static oracle.weblogic.kubernetes.utils.OperatorUtils.installAndVerifyOperator;
50+
import static oracle.weblogic.kubernetes.utils.OperatorUtils.upgradeAndVerifyOperator;
51+
import static oracle.weblogic.kubernetes.utils.ThreadSafeLogger.getLogger;
52+
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
53+
import static org.junit.jupiter.api.Assertions.assertEquals;
54+
import static org.junit.jupiter.api.Assertions.assertNotNull;
55+
import static org.junit.jupiter.api.Assertions.assertTrue;
56+
import static org.junit.jupiter.api.Assertions.fail;
57+
58+
/**
59+
* To add integration tests for ELK Stack sample on Operator, this test does
60+
* 1. Install and start Elasticsearch/Kibana using sample script
61+
* kubernetes/samples/scripts/elasticsearch-and-kibana/elasticsearch_and_kibana.yaml.
62+
* 2. Install and start Operators with ELK Stack enabled,
63+
* 3. Verify that ELK Stack is ready to use by checking the index status of
64+
* Kibana and Logstash created in the Operator pod successfully.
65+
* 4. Verify that Elasticsearch collects data from Operator logs and
66+
* stores them in its repository correctly.
67+
*/
68+
@DisplayName("ELK Stack sample to test to use Elasticsearch API to query Operator logs")
69+
@IntegrationTest
70+
@Tag("kind-parallel")
71+
class ItElasticLoggingSample {
72+
// constants for namespaces
73+
private static String domainNamespace = null;
74+
private static String opNamespace = null;
75+
76+
// constants for ELK stack
77+
static String elasticSearchHost;
78+
static String elasticSearchNs = "default";
79+
private static String sourceELKConfigFile =
80+
ITTESTS_DIR + "/../kubernetes/samples/scripts/elasticsearch-and-kibana/elasticsearch_and_kibana.yaml";
81+
private static String destELKConfigFile = WORK_DIR + "elasticsearch_and_kibana.yaml";
82+
83+
// constants for test
84+
private static String k8sExecCmdPrefix;
85+
private static Map<String, String> testVarMap;
86+
87+
private static LoggingFacade logger = null;
88+
89+
/**
90+
* Install Elasticsearch, Kibana and Operator.
91+
*
92+
* @param namespaces list of namespaces created by the IntegrationTestWatcher by the
93+
* JUnit engine parameter resolution mechanism.
94+
*/
95+
@BeforeAll
96+
public static void init(@Namespaces(2) List<String> namespaces) {
97+
logger = getLogger();
98+
99+
// get a new unique opNamespace
100+
logger.info("Assigning a unique namespace for Operator");
101+
assertNotNull(namespaces.get(0), "Namespace list is null");
102+
opNamespace = namespaces.get(0);
103+
104+
// get a new unique domainNamespace
105+
logger.info("Assigning a unique namespace for Domain");
106+
assertNotNull(namespaces.get(1), "Namespace list is null");
107+
domainNamespace = namespaces.get(1);
108+
109+
// copy original ELK config file to work dir
110+
Path sourceELKConfigFilePath = Paths.get(sourceELKConfigFile);
111+
Path destELKConfigFilePath = Paths.get(destELKConfigFile);
112+
assertDoesNotThrow(() -> Files.copy(sourceELKConfigFilePath, destELKConfigFilePath,
113+
StandardCopyOption.REPLACE_EXISTING)," Failed to copy files");
114+
115+
// reploce the location for busybox image
116+
assertDoesNotThrow(() -> replaceStringInFile(destELKConfigFilePath.toString(),
117+
"busybox", BUSYBOX_IMAGE + ":" + BUSYBOX_TAG),
118+
"Failed to replace String: " + BUSYBOX_IMAGE + ":" + BUSYBOX_TAG);
119+
120+
// reploce the location for ELK stack image
121+
assertDoesNotThrow(() -> replaceStringInFile(destELKConfigFilePath.toString(),
122+
"elasticsearch:7.8.1", ELASTICSEARCH_IMAGE),"Failed to replace String: " + ELASTICSEARCH_IMAGE);
123+
assertDoesNotThrow(() -> replaceStringInFile(destELKConfigFilePath.toString(),
124+
"kibana:7.8.1", KIBANA_IMAGE),"Failed to replace String: " + KIBANA_IMAGE);
125+
126+
// install and verify Elasticsearch and Kibana;
127+
elasticSearchHost = "elasticsearch." + elasticSearchNs + ".svc.cluster.local";
128+
logger.info("install and verify Elasticsearch and Kibana");
129+
130+
CommandParams params = new CommandParams().defaults();
131+
params.command(KUBERNETES_CLI + " apply -f " + destELKConfigFilePath);
132+
boolean result = Command.withParams(params).execute();
133+
assertTrue(result, "Failed to install Elasticsearch and Kibana");
134+
135+
// install and verify Operator
136+
installAndVerifyOperator(opNamespace, opNamespace + "-sa",
137+
false, 0, true, domainNamespace);
138+
139+
// upgrade to latest operator
140+
HelmParams upgradeHelmParams = new HelmParams()
141+
.releaseName(OPERATOR_RELEASE_NAME)
142+
.namespace(opNamespace)
143+
.chartDir(OPERATOR_CHART_DIR);
144+
145+
// build operator chart values
146+
OperatorParams opParams = new OperatorParams()
147+
.helmParams(upgradeHelmParams)
148+
.elkIntegrationEnabled(true)
149+
.elasticSearchHost(elasticSearchHost)
150+
.elasticSearchPort(9200)
151+
.createLogStashConfigMap(true);
152+
153+
assertTrue(upgradeAndVerifyOperator(opNamespace, opParams),
154+
String.format("Failed to upgrade operator in namespace %s", opNamespace));
155+
156+
// wait for the operator to be ready
157+
logger.info("Wait for the operator pod is ready in namespace {0}", opNamespace);
158+
testUntil(
159+
assertDoesNotThrow(() -> operatorIsReady(opNamespace),
160+
"operatorIsReady failed with ApiException"),
161+
logger,
162+
"operator to be running in namespace {0}",
163+
opNamespace);
164+
165+
StringBuffer elasticsearchUrlBuff =
166+
new StringBuffer("curl http://")
167+
.append(elasticSearchHost)
168+
.append(":")
169+
.append(ELASTICSEARCH_HTTP_PORT);
170+
k8sExecCmdPrefix = elasticsearchUrlBuff.toString();
171+
logger.info("Elasticsearch URL {0}", k8sExecCmdPrefix);
172+
173+
// Verify that ELK Stack is ready to use
174+
testVarMap = new HashMap<String, String>();
175+
testVarMap = verifyLoggingExporterReady(opNamespace, elasticSearchNs, null, LOGSTASH_INDEX_KEY);
176+
Map<String, String> kibanaMap = verifyLoggingExporterReady(opNamespace, elasticSearchNs, null, KIBANA_INDEX_KEY);
177+
178+
// merge testVarMap and kibanaMap
179+
testVarMap.putAll(kibanaMap);
180+
}
181+
182+
/**
183+
* Uninstall ELK Stack.
184+
*/
185+
@AfterAll
186+
void tearDown() {
187+
// uninstall ELK Stack
188+
logger.info("uninstall and verify Elasticsearch and Kibana");
189+
190+
Path destELKConfigFilePath = Paths.get(destELKConfigFile);
191+
CommandParams params = new CommandParams().defaults();
192+
params.command(KUBERNETES_CLI + " delete -f " + destELKConfigFilePath);
193+
boolean result = Command.withParams(params).execute();
194+
assertTrue(result, "Failed to uninstall Elasticsearch and Kibana");
195+
}
196+
197+
/**
198+
* Use Elasticsearch Count API to query Operator logs of level=INFO.
199+
* Verify that total number of logs for level=INFO is not zero and failed count is zero.
200+
*/
201+
@Test
202+
@DisplayName("Use Elasticsearch Count API to query Operator logs of level=INFO and verify")
203+
void testOpLogLevelSearch() {
204+
// Verify that number of logs is not zero and failed count is zero
205+
String regex = ".*count\":(\\d+),.*failed\":(\\d+)";
206+
String queryCriteria = "/_count?q=level:INFO";
207+
208+
// verify log level query results
209+
withStandardRetryPolicy.untilAsserted(
210+
() -> assertTrue(verifyCountsHitsInSearchResults(queryCriteria, regex, LOGSTASH_INDEX_KEY),
211+
"Query logs of level=INFO failed"));
212+
213+
logger.info("Query logs of level=INFO succeeded");
214+
}
215+
216+
private boolean verifyCountsHitsInSearchResults(String queryCriteria, String regex, String index) {
217+
boolean testResult = false;
218+
int count = -1;
219+
int failedCount = -1;
220+
221+
ExecResult results = execSearchQuery(queryCriteria, index);
222+
if (results.exitValue() == 0) {
223+
Pattern pattern = Pattern.compile(regex, Pattern.DOTALL | Pattern.MULTILINE);
224+
Matcher matcher = pattern.matcher(results.stdout());
225+
if (matcher.find()) {
226+
count = Integer.parseInt(matcher.group(1));
227+
failedCount = Integer.parseInt(matcher.group(2));
228+
}
229+
230+
// verify that total count > 0 and failed count = 0 in Operator log
231+
logger.info("Total count of logs: " + count);
232+
assertTrue(count > 0, "Total count of logs should be more than 0!");
233+
logger.info("Total failed count: " + failedCount);
234+
assertEquals(0, failedCount, "Total failed count should be 0!");
235+
testResult = true;
236+
} else {
237+
fail("Failed to verify total count and failed count in Operator log" + results.stderr());
238+
}
239+
240+
return testResult;
241+
}
242+
243+
private ExecResult execSearchQuery(String queryCriteria, String index) {
244+
String operatorPodName = assertDoesNotThrow(
245+
() -> getOperatorPodName(OPERATOR_RELEASE_NAME, opNamespace));
246+
assertTrue(operatorPodName != null && !operatorPodName.isEmpty(), "Failed to get Operator pad name");
247+
logger.info("Operator pod name " + operatorPodName);
248+
249+
int waittime = 5;
250+
String indexName = testVarMap.get(index);
251+
StringBuffer curlOptions = new StringBuffer(" --connect-timeout " + waittime)
252+
.append(" --max-time " + waittime)
253+
.append(" -X GET ");
254+
StringBuffer k8sExecCmdPrefixBuff = new StringBuffer(k8sExecCmdPrefix);
255+
int offset = k8sExecCmdPrefixBuff.indexOf("http");
256+
k8sExecCmdPrefixBuff.insert(offset, curlOptions);
257+
String cmd = k8sExecCmdPrefixBuff
258+
.append("/")
259+
.append(indexName)
260+
.append(queryCriteria)
261+
.toString();
262+
logger.info("Exec command {0} in Operator pod {1}", cmd, operatorPodName);
263+
264+
ExecResult execResult = assertDoesNotThrow(
265+
() -> execCommand(opNamespace, operatorPodName, null, true,
266+
"/bin/sh", "-c", cmd));
267+
assertNotNull(execResult, "curl command returns null");
268+
logger.info("Search query returns " + execResult.stdout());
269+
270+
return execResult;
271+
}
272+
}

‎kubernetes/samples/scripts/elasticsearch-and-kibana/elasticsearch_and_kibana.yaml

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright (c) 2018, 2021, Oracle and/or its affiliates.
1+
# Copyright (c) 2018, 2023, Oracle and/or its affiliates.
22
# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
33

44
# When a user installs the WebLogic operator Helm chart, the user can set
@@ -50,13 +50,17 @@ spec:
5050
privileged: true
5151
containers:
5252
- name: "elasticsearch"
53-
image: "elasticsearch:6.8.23"
53+
image: "elasticsearch:7.8.1"
5454
ports:
5555
- containerPort: 9200
5656
- containerPort: 9300
5757
env:
58+
- name: discovery.type
59+
value: single-node
5860
- name: ES_JAVA_OPTS
5961
value: -Xms1024m -Xmx1024m
62+
- name: bootstrap.memory_lock
63+
value: "false"
6064

6165
---
6266
kind: "Service"
@@ -97,7 +101,7 @@ spec:
97101
spec:
98102
containers:
99103
- name: "kibana"
100-
image: "kibana:6.8.23"
104+
image: "kibana:7.8.1"
101105
ports:
102106
- containerPort: 5601
103107

0 commit comments

Comments
 (0)