Skip to content

Commit f63e9e5

Browse files
martijnvgmartijnvg
authored
Remove DocumentParserContext#removeLastIgnoredField(...) (#132644)
This is no longer needed as offsets of leaf array fields are stored in a doc values offset field that is controlled by field mapper.
1 parent d036f8b commit f63e9e5

File tree

3 files changed

+246
-18
lines changed

3 files changed

+246
-18
lines changed

‎server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java‎

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -702,8 +702,6 @@ private static void parseNonDynamicArray(
702702
boolean supportStoringArrayOffsets = mapper != null && mapper.supportStoringArrayOffsets();
703703
String fullPath = context.path().pathAsText(arrayFieldName);
704704

705-
// Check if we need to record the array source. This only applies to synthetic source.
706-
boolean canRemoveSingleLeafElement = false;
707705
if (context.canAddIgnoredField() && supportStoringArrayOffsets == false) {
708706
Mapper.SourceKeepMode mode = Mapper.SourceKeepMode.NONE;
709707
boolean objectWithFallbackSyntheticSource = false;
@@ -721,13 +719,6 @@ private static void parseNonDynamicArray(
721719
}
722720
boolean copyToFieldHasValuesInDocument = context.isWithinCopyTo() == false && context.isCopyToDestinationField(fullPath);
723721

724-
canRemoveSingleLeafElement = mapper instanceof FieldMapper
725-
&& mode == Mapper.SourceKeepMode.ARRAYS
726-
&& context.inArrayScope() == false
727-
&& mapper.leafName().equals(NOOP_FIELD_MAPPER_NAME) == false
728-
&& fieldWithFallbackSyntheticSource == false
729-
&& copyToFieldHasValuesInDocument == false;
730-
731722
if (objectWithFallbackSyntheticSource
732723
|| fieldWithFallbackSyntheticSource
733724
|| fieldWithStoredArraySource
@@ -776,9 +767,6 @@ private static void parseNonDynamicArray(
776767
&& context.isImmediateParentAnArray()) {
777768
context.getOffSetContext().maybeRecordEmptyArray(mapper.getOffsetFieldName());
778769
}
779-
if (elements <= 1 && canRemoveSingleLeafElement) {
780-
context.removeLastIgnoredField(fullPath);
781-
}
782770
postProcessDynamicArrayMapping(context, lastFieldName);
783771
}
784772

‎server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -335,12 +335,6 @@ public final void addIgnoredField(IgnoredSourceFieldMapper.NameValue values) {
335335
}
336336
}
337337

338-
final void removeLastIgnoredField(String name) {
339-
if (ignoredFieldValues.isEmpty() == false && ignoredFieldValues.getLast().name().equals(name)) {
340-
ignoredFieldValues.removeLast();
341-
}
342-
}
343-
344338
/**
345339
* Return the collection of values for fields that have been ignored so far.
346340
*/

‎x-pack/plugin/logsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/SyntheticSourceRollingUpgradeIT.java‎

Lines changed: 246 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,246 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.upgrades;
11+
12+
import com.carrotsearch.randomizedtesting.annotations.Name;
13+
14+
import org.elasticsearch.client.Request;
15+
import org.elasticsearch.test.rest.ObjectPath;
16+
import org.hamcrest.Matchers;
17+
18+
import java.time.Instant;
19+
import java.util.Arrays;
20+
import java.util.List;
21+
import java.util.Locale;
22+
import java.util.Map;
23+
import java.util.stream.Collectors;
24+
25+
import static org.elasticsearch.upgrades.LogsIndexModeRollingUpgradeIT.enableLogsdbByDefault;
26+
import static org.elasticsearch.upgrades.LogsIndexModeRollingUpgradeIT.getWriteBackingIndex;
27+
import static org.elasticsearch.upgrades.MatchOnlyTextRollingUpgradeIT.createTemplate;
28+
import static org.elasticsearch.upgrades.MatchOnlyTextRollingUpgradeIT.formatInstant;
29+
import static org.elasticsearch.upgrades.MatchOnlyTextRollingUpgradeIT.getIndexSettingsWithDefaults;
30+
import static org.elasticsearch.upgrades.MatchOnlyTextRollingUpgradeIT.startTrial;
31+
import static org.hamcrest.Matchers.both;
32+
import static org.hamcrest.Matchers.emptyIterable;
33+
import static org.hamcrest.Matchers.equalTo;
34+
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
35+
import static org.hamcrest.Matchers.lessThanOrEqualTo;
36+
import static org.hamcrest.Matchers.not;
37+
38+
public class SyntheticSourceRollingUpgradeIT extends AbstractRollingUpgradeWithSecurityTestCase {
39+
40+
static String BULK_ITEM_TEMPLATE = """
41+
{"@timestamp": "$now", "field1": "$field1", "field2": $field2, "field3": $field3, "field4": $field4}
42+
""";
43+
44+
private static final String TEMPLATE = """
45+
{
46+
"mappings": {
47+
"properties": {
48+
"@timestamp" : {
49+
"type": "date"
50+
},
51+
"field1": {
52+
"type": "keyword"
53+
},
54+
"field2": {
55+
"type": "keyword"
56+
},
57+
"field3": {
58+
"type": "long"
59+
},
60+
"field4": {
61+
"type": "long"
62+
}
63+
}
64+
}
65+
}""";
66+
67+
private static final Integer[] VALUES = new Integer[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 };
68+
69+
public SyntheticSourceRollingUpgradeIT(@Name("upgradedNodes") int upgradedNodes) {
70+
super(upgradedNodes);
71+
}
72+
73+
public void testIndexing() throws Exception {
74+
String dataStreamName = "logs-bwc-test";
75+
if (isOldCluster()) {
76+
startTrial();
77+
enableLogsdbByDefault();
78+
createTemplate(dataStreamName, getClass().getSimpleName().toLowerCase(Locale.ROOT), TEMPLATE);
79+
80+
Instant startTime = Instant.now().minusSeconds(60 * 60);
81+
bulkIndex(dataStreamName, 4, 1024, startTime);
82+
83+
String firstBackingIndex = getWriteBackingIndex(client(), dataStreamName, 0);
84+
var settings = (Map<?, ?>) getIndexSettingsWithDefaults(firstBackingIndex).get(firstBackingIndex);
85+
assertThat(((Map<?, ?>) settings.get("settings")).get("index.mode"), equalTo("logsdb"));
86+
assertThat(((Map<?, ?>) settings.get("defaults")).get("index.mapping.source.mode"), equalTo("SYNTHETIC"));
87+
88+
ensureGreen(dataStreamName);
89+
search(dataStreamName);
90+
query(dataStreamName);
91+
} else if (isMixedCluster()) {
92+
Instant startTime = Instant.now().minusSeconds(60 * 30);
93+
bulkIndex(dataStreamName, 4, 1024, startTime);
94+
95+
ensureGreen(dataStreamName);
96+
search(dataStreamName);
97+
query(dataStreamName);
98+
} else if (isUpgradedCluster()) {
99+
ensureGreen(dataStreamName);
100+
Instant startTime = Instant.now();
101+
bulkIndex(dataStreamName, 4, 1024, startTime);
102+
search(dataStreamName);
103+
query(dataStreamName);
104+
105+
var forceMergeRequest = new Request("POST", "/" + dataStreamName + "/_forcemerge");
106+
forceMergeRequest.addParameter("max_num_segments", "1");
107+
assertOK(client().performRequest(forceMergeRequest));
108+
109+
ensureGreen(dataStreamName);
110+
search(dataStreamName);
111+
query(dataStreamName);
112+
}
113+
}
114+
115+
static String bulkIndex(String dataStreamName, int numRequest, int numDocs, Instant startTime) throws Exception {
116+
String firstIndex = null;
117+
for (int i = 0; i < numRequest; i++) {
118+
var bulkRequest = new Request("POST", "/" + dataStreamName + "/_bulk");
119+
StringBuilder requestBody = new StringBuilder();
120+
for (int j = 0; j < numDocs; j++) {
121+
String field1 = Integer.toString(randomFrom(VALUES));
122+
var randomArray = randomArray(1, 3, Integer[]::new, () -> randomFrom(VALUES));
123+
String field2 = Arrays.stream(randomArray).map(s -> "\"" + s + "\"").collect(Collectors.joining(","));
124+
int field3 = randomFrom(VALUES);
125+
String field4 = Arrays.stream(randomArray).map(String::valueOf).collect(Collectors.joining(","));
126+
127+
requestBody.append("{\"create\": {}}");
128+
requestBody.append('\n');
129+
requestBody.append(
130+
BULK_ITEM_TEMPLATE.replace("$now", formatInstant(startTime))
131+
.replace("$field1", field1)
132+
.replace("$field2", "[" + field2 + "]")
133+
.replace("$field3", Long.toString(field3))
134+
.replace("$field4", "[" + field4 + "]")
135+
);
136+
requestBody.append('\n');
137+
138+
startTime = startTime.plusMillis(1);
139+
}
140+
bulkRequest.setJsonEntity(requestBody.toString());
141+
bulkRequest.addParameter("refresh", "true");
142+
var response = client().performRequest(bulkRequest);
143+
assertOK(response);
144+
var responseBody = entityAsMap(response);
145+
assertThat("errors in response:\n " + responseBody, responseBody.get("errors"), equalTo(false));
146+
if (firstIndex == null) {
147+
firstIndex = (String) ((Map<?, ?>) ((Map<?, ?>) ((List<?>) responseBody.get("items")).get(0)).get("create")).get("_index");
148+
}
149+
}
150+
return firstIndex;
151+
}
152+
153+
void search(String dataStreamName) throws Exception {
154+
var searchRequest = new Request("POST", "/" + dataStreamName + "/_search");
155+
searchRequest.addParameter("pretty", "true");
156+
searchRequest.setJsonEntity("""
157+
{
158+
"size": 500
159+
}
160+
""");
161+
var response = client().performRequest(searchRequest);
162+
assertOK(response);
163+
var responseBody = entityAsMap(response);
164+
logger.info("{}", responseBody);
165+
assertThat(ObjectPath.evaluate(responseBody, "_shards.failed"), Matchers.equalTo(0));
166+
Integer totalCount = ObjectPath.evaluate(responseBody, "hits.total.value");
167+
assertThat(totalCount, greaterThanOrEqualTo(512));
168+
169+
Map<?, ?> firstSource = ObjectPath.evaluate(responseBody, "hits.hits.0._source");
170+
Integer field1 = Integer.valueOf((String) firstSource.get("field1"));
171+
assertThat(field1, both(greaterThanOrEqualTo(0)).and(lessThanOrEqualTo(9)));
172+
List<?> field2 = (List<?>) firstSource.get("field2");
173+
assertThat(field2, not(emptyIterable()));
174+
for (var e : field2) {
175+
Integer value = Integer.valueOf((String) e);
176+
assertThat(value, both(greaterThanOrEqualTo(0)).and(lessThanOrEqualTo(9)));
177+
}
178+
Integer field3 = (Integer) firstSource.get("field3");
179+
assertThat(field3, both(greaterThanOrEqualTo(0)).and(lessThanOrEqualTo(9)));
180+
List<?> field4 = (List<?>) firstSource.get("field4");
181+
assertThat(field4, not(emptyIterable()));
182+
for (var e : field4) {
183+
Integer value = (Integer) e;
184+
assertThat(value, both(greaterThanOrEqualTo(0)).and(lessThanOrEqualTo(9)));
185+
}
186+
}
187+
188+
void query(String dataStreamName) throws Exception {
189+
var queryRequest = new Request("POST", "/_query");
190+
queryRequest.addParameter("pretty", "true");
191+
queryRequest.setJsonEntity("""
192+
{
193+
"query": "FROM $ds | SORT @timestamp | KEEP field1,field2,field3,field4 | LIMIT 5"
194+
}
195+
""".replace("$ds", dataStreamName));
196+
var response = client().performRequest(queryRequest);
197+
assertOK(response);
198+
var responseBody = entityAsMap(response);
199+
logger.info("{}", responseBody);
200+
201+
String column1 = ObjectPath.evaluate(responseBody, "columns.0.name");
202+
String column2 = ObjectPath.evaluate(responseBody, "columns.1.name");
203+
String column3 = ObjectPath.evaluate(responseBody, "columns.2.name");
204+
String column4 = ObjectPath.evaluate(responseBody, "columns.3.name");
205+
assertThat(column1, equalTo("field1"));
206+
assertThat(column2, equalTo("field2"));
207+
assertThat(column3, equalTo("field3"));
208+
assertThat(column4, equalTo("field4"));
209+
210+
{
211+
var field1 = Integer.valueOf(ObjectPath.evaluate(responseBody, "values.0.0"));
212+
assertThat(field1, both(greaterThanOrEqualTo(0)).and(lessThanOrEqualTo(9)));
213+
}
214+
{
215+
var object = ObjectPath.evaluate(responseBody, "values.0.1");
216+
if (object instanceof List<?> field2) {
217+
assertThat(field2, not(emptyIterable()));
218+
for (var e : field2) {
219+
Integer value = Integer.valueOf((String) e);
220+
assertThat(value, both(greaterThanOrEqualTo(0)).and(lessThanOrEqualTo(9)));
221+
}
222+
} else {
223+
Integer field2 = Integer.valueOf((String) object);
224+
assertThat(field2, both(greaterThanOrEqualTo(0)).and(lessThanOrEqualTo(9)));
225+
}
226+
}
227+
{
228+
Integer field3 = ObjectPath.evaluate(responseBody, "values.0.2");
229+
assertThat(field3, both(greaterThanOrEqualTo(0)).and(lessThanOrEqualTo(9)));
230+
}
231+
{
232+
var object = ObjectPath.evaluate(responseBody, "values.0.3");
233+
if (object instanceof List<?> field4) {
234+
assertThat(field4, not(emptyIterable()));
235+
for (var e : field4) {
236+
Integer value = (Integer) e;
237+
assertThat(value, both(greaterThanOrEqualTo(0)).and(lessThanOrEqualTo(9)));
238+
}
239+
} else {
240+
Integer field4 = (Integer) object;
241+
assertThat(field4, both(greaterThanOrEqualTo(0)).and(lessThanOrEqualTo(9)));
242+
}
243+
}
244+
}
245+
246+
}

0 commit comments

Comments
 (0)