Skip to content

Commit beb0223

Browse files
martijnvgmartijnvg
committed
Added logsdb rolling upgrade test that uses match_only_text field.
1 parent 1d285b1 commit beb0223

File tree

2 files changed

+253
-0
lines changed

2 files changed

+253
-0
lines changed

‎build-tools-internal/src/main/resources/checkstyle_suppressions.xml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737
<suppress files="qa[/\\]rolling-upgrade[/\\]src[/\\]javaRestTest[/\\]java[/\\]org[/\\]elasticsearch[/\\]upgrades[/\\]TsdbIT.java" checks="LineLength" />
3838
<suppress files="qa[/\\]rolling-upgrade[/\\]src[/\\]javaRestTest[/\\]java[/\\]org[/\\]elasticsearch[/\\]upgrades[/\\]TsdbIndexingRollingUpgradeIT.java" checks="LineLength" />
3939
<suppress files="qa[/\\]rolling-upgrade[/\\]src[/\\]javaRestTest[/\\]java[/\\]org[/\\]elasticsearch[/\\]upgrades[/\\]LogsdbIndexingRollingUpgradeIT.java" checks="LineLength" />
40+
<suppress files="plugin[/\\]logsdb[/\\]qa[/\\]rolling-upgrade[/\\]src[/\\]javaRestTest[/\\]java[/\\]org[/\\]elasticsearch[/\\]upgrades[/\\]MatchOnlyTextRollingUpgradeIT.java" checks="LineLength" />
4041

4142
<!-- Gradle requires inputs to be seriablizable -->
4243
<suppress files="build-tools-internal[/\\]src[/\\]main[/\\]java[/\\]org[/\\]elasticsearch[/\\]gradle[/\\]internal[/\\]precommit[/\\]TestingConventionRule.java" checks="RegexpSinglelineJava" />

‎x-pack/plugin/logsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/MatchOnlyTextRollingUpgradeIT.java‎

Lines changed: 252 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,252 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.upgrades;
11+
12+
import com.carrotsearch.randomizedtesting.annotations.Name;
13+
14+
import org.elasticsearch.client.Request;
15+
import org.elasticsearch.client.Response;
16+
import org.elasticsearch.client.ResponseException;
17+
import org.elasticsearch.common.network.NetworkAddress;
18+
import org.elasticsearch.common.time.DateFormatter;
19+
import org.elasticsearch.common.time.FormatNames;
20+
import org.elasticsearch.common.xcontent.XContentHelper;
21+
import org.elasticsearch.test.rest.ObjectPath;
22+
import org.elasticsearch.xcontent.XContentType;
23+
24+
import java.io.IOException;
25+
import java.io.InputStream;
26+
import java.time.Instant;
27+
import java.util.List;
28+
import java.util.Locale;
29+
import java.util.Map;
30+
31+
import static org.elasticsearch.upgrades.LogsIndexModeRollingUpgradeIT.enableLogsdbByDefault;
32+
import static org.elasticsearch.upgrades.LogsIndexModeRollingUpgradeIT.getWriteBackingIndex;
33+
import static org.hamcrest.Matchers.containsString;
34+
import static org.hamcrest.Matchers.equalTo;
35+
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
36+
import static org.hamcrest.Matchers.notNullValue;
37+
38+
public class MatchOnlyTextRollingUpgradeIT extends AbstractRollingUpgradeWithSecurityTestCase {
39+
40+
static String BULK_ITEM_TEMPLATE =
41+
"""
42+
{"@timestamp": "$now", "host.name": "$host", "method": "$method", "ip": "$ip", "message": "$message", "length": $length, "factor": $factor}
43+
""";
44+
45+
private static final String TEMPLATE = """
46+
{
47+
"mappings": {
48+
"properties": {
49+
"@timestamp" : {
50+
"type": "date"
51+
},
52+
"method": {
53+
"type": "keyword"
54+
},
55+
"message": {
56+
"type": "match_only_text"
57+
},
58+
"ip": {
59+
"type": "ip"
60+
},
61+
"length": {
62+
"type": "long"
63+
},
64+
"factor": {
65+
"type": "double"
66+
}
67+
}
68+
}
69+
}""";
70+
71+
public MatchOnlyTextRollingUpgradeIT(@Name("upgradedNodes") int upgradedNodes) {
72+
super(upgradedNodes);
73+
}
74+
75+
public void testIndexing() throws Exception {
76+
String dataStreamName = "logs-bwc-test";
77+
if (isOldCluster()) {
78+
startTrial();
79+
enableLogsdbByDefault();
80+
createTemplate(dataStreamName, getClass().getSimpleName().toLowerCase(Locale.ROOT), TEMPLATE);
81+
82+
Instant startTime = Instant.now().minusSeconds(60 * 60);
83+
bulkIndex(dataStreamName, 4, 1024, startTime);
84+
85+
String firstBackingIndex = getWriteBackingIndex(client(), dataStreamName, 0);
86+
var settings = (Map<?, ?>) getIndexSettingsWithDefaults(firstBackingIndex).get(firstBackingIndex);
87+
assertThat(((Map<?, ?>) settings.get("settings")).get("index.mode"), equalTo("logsdb"));
88+
assertThat(((Map<?, ?>) settings.get("defaults")).get("index.mapping.source.mode"), equalTo("SYNTHETIC"));
89+
90+
ensureGreen(dataStreamName);
91+
search(dataStreamName);
92+
query(dataStreamName);
93+
} else if (isMixedCluster()) {
94+
Instant startTime = Instant.now().minusSeconds(60 * 30);
95+
bulkIndex(dataStreamName, 4, 1024, startTime);
96+
97+
ensureGreen(dataStreamName);
98+
search(dataStreamName);
99+
query(dataStreamName);
100+
} else if (isUpgradedCluster()) {
101+
ensureGreen(dataStreamName);
102+
Instant startTime = Instant.now();
103+
bulkIndex(dataStreamName, 4, 1024, startTime);
104+
search(dataStreamName);
105+
query(dataStreamName);
106+
107+
var forceMergeRequest = new Request("POST", "/" + dataStreamName + "/_forcemerge");
108+
forceMergeRequest.addParameter("max_num_segments", "1");
109+
assertOK(client().performRequest(forceMergeRequest));
110+
111+
ensureGreen(dataStreamName);
112+
search(dataStreamName);
113+
query(dataStreamName);
114+
}
115+
}
116+
117+
static void createTemplate(String dataStreamName, String id, String template) throws IOException {
118+
final String INDEX_TEMPLATE = """
119+
{
120+
"index_patterns": ["$DATASTREAM"],
121+
"template": $TEMPLATE,
122+
"data_stream": {
123+
}
124+
}""";
125+
var putIndexTemplateRequest = new Request("POST", "/_index_template/" + id);
126+
putIndexTemplateRequest.setJsonEntity(INDEX_TEMPLATE.replace("$TEMPLATE", template).replace("$DATASTREAM", dataStreamName));
127+
assertOK(client().performRequest(putIndexTemplateRequest));
128+
}
129+
130+
static String bulkIndex(String dataStreamName, int numRequest, int numDocs, Instant startTime) throws Exception {
131+
String firstIndex = null;
132+
for (int i = 0; i < numRequest; i++) {
133+
var bulkRequest = new Request("POST", "/" + dataStreamName + "/_bulk");
134+
StringBuilder requestBody = new StringBuilder();
135+
for (int j = 0; j < numDocs; j++) {
136+
String hostName = "host" + j % 50; // Not realistic, but makes asserting search / query response easier.
137+
String methodName = "method" + j % 5;
138+
String ip = NetworkAddress.format(randomIp(true));
139+
String param = "chicken" + randomInt(5);
140+
String message = "the quick brown fox jumps over the " + param;
141+
long length = randomLong();
142+
double factor = randomDouble();
143+
144+
requestBody.append("{\"create\": {}}");
145+
requestBody.append('\n');
146+
requestBody.append(
147+
BULK_ITEM_TEMPLATE.replace("$now", formatInstant(startTime))
148+
.replace("$host", hostName)
149+
.replace("$method", methodName)
150+
.replace("$ip", ip)
151+
.replace("$message", message)
152+
.replace("$length", Long.toString(length))
153+
.replace("$factor", Double.toString(factor))
154+
);
155+
requestBody.append('\n');
156+
157+
startTime = startTime.plusMillis(1);
158+
}
159+
bulkRequest.setJsonEntity(requestBody.toString());
160+
bulkRequest.addParameter("refresh", "true");
161+
var response = client().performRequest(bulkRequest);
162+
assertOK(response);
163+
var responseBody = entityAsMap(response);
164+
assertThat("errors in response:\n " + responseBody, responseBody.get("errors"), equalTo(false));
165+
if (firstIndex == null) {
166+
firstIndex = (String) ((Map<?, ?>) ((Map<?, ?>) ((List<?>) responseBody.get("items")).get(0)).get("create")).get("_index");
167+
}
168+
}
169+
return firstIndex;
170+
}
171+
172+
void search(String dataStreamName) throws Exception {
173+
var searchRequest = new Request("POST", "/" + dataStreamName + "/_search");
174+
searchRequest.addParameter("pretty", "true");
175+
searchRequest.setJsonEntity("""
176+
{
177+
"size": 500,
178+
"query": {
179+
"match_phrase": {
180+
"message": "chicken"
181+
}
182+
}
183+
}
184+
""".replace("chicken", "chicken" + randomInt(5)));
185+
var response = client().performRequest(searchRequest);
186+
assertOK(response);
187+
var responseBody = entityAsMap(response);
188+
logger.info("{}", responseBody);
189+
190+
Integer totalCount = ObjectPath.evaluate(responseBody, "hits.total.value");
191+
assertThat(totalCount, greaterThanOrEqualTo(512));
192+
}
193+
194+
void query(String dataStreamName) throws Exception {
195+
var queryRequest = new Request("POST", "/_query");
196+
queryRequest.addParameter("pretty", "true");
197+
queryRequest.setJsonEntity("""
198+
{
199+
"query": "FROM $ds | STATS max(length), max(factor) BY message | SORT message | LIMIT 5"
200+
}
201+
""".replace("$ds", dataStreamName));
202+
var response = client().performRequest(queryRequest);
203+
assertOK(response);
204+
var responseBody = entityAsMap(response);
205+
logger.info("{}", responseBody);
206+
207+
String column1 = ObjectPath.evaluate(responseBody, "columns.0.name");
208+
String column2 = ObjectPath.evaluate(responseBody, "columns.1.name");
209+
String column3 = ObjectPath.evaluate(responseBody, "columns.2.name");
210+
assertThat(column1, equalTo("max(length)"));
211+
assertThat(column2, equalTo("max(factor)"));
212+
assertThat(column3, equalTo("message"));
213+
214+
String key = ObjectPath.evaluate(responseBody, "values.0.2");
215+
assertThat(key, equalTo("the quick brown fox jumps over the chicken0"));
216+
Long maxRx = ObjectPath.evaluate(responseBody, "values.0.0");
217+
assertThat(maxRx, notNullValue());
218+
Double maxTx = ObjectPath.evaluate(responseBody, "values.0.1");
219+
assertThat(maxTx, notNullValue());
220+
}
221+
222+
protected static void startTrial() throws IOException {
223+
Request startTrial = new Request("POST", "/_license/start_trial");
224+
startTrial.addParameter("acknowledge", "true");
225+
try {
226+
assertOK(client().performRequest(startTrial));
227+
} catch (ResponseException e) {
228+
var responseBody = entityAsMap(e.getResponse());
229+
String error = ObjectPath.evaluate(responseBody, "error_message");
230+
assertThat(error, containsString("Trial was already activated."));
231+
}
232+
}
233+
234+
static Map<String, Object> getIndexSettingsWithDefaults(String index) throws IOException {
235+
Request request = new Request("GET", "/" + index + "/_settings");
236+
request.addParameter("flat_settings", "true");
237+
request.addParameter("include_defaults", "true");
238+
Response response = client().performRequest(request);
239+
try (InputStream is = response.getEntity().getContent()) {
240+
return XContentHelper.convertToMap(
241+
XContentType.fromMediaType(response.getEntity().getContentType().getValue()).xContent(),
242+
is,
243+
true
244+
);
245+
}
246+
}
247+
248+
static String formatInstant(Instant instant) {
249+
return DateFormatter.forPattern(FormatNames.STRICT_DATE_OPTIONAL_TIME.getName()).format(instant);
250+
}
251+
252+
}

0 commit comments

Comments
 (0)