diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index 15e7ff88350b6..1a248f2dd501e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -702,8 +702,6 @@ private static void parseNonDynamicArray( boolean supportStoringArrayOffsets = mapper != null && mapper.supportStoringArrayOffsets(); String fullPath = context.path().pathAsText(arrayFieldName); - // Check if we need to record the array source. This only applies to synthetic source. - boolean canRemoveSingleLeafElement = false; if (context.canAddIgnoredField() && supportStoringArrayOffsets == false) { Mapper.SourceKeepMode mode = Mapper.SourceKeepMode.NONE; boolean objectWithFallbackSyntheticSource = false; @@ -721,13 +719,6 @@ private static void parseNonDynamicArray( } boolean copyToFieldHasValuesInDocument = context.isWithinCopyTo() == false && context.isCopyToDestinationField(fullPath); - canRemoveSingleLeafElement = mapper instanceof FieldMapper - && mode == Mapper.SourceKeepMode.ARRAYS - && context.inArrayScope() == false - && mapper.leafName().equals(NOOP_FIELD_MAPPER_NAME) == false - && fieldWithFallbackSyntheticSource == false - && copyToFieldHasValuesInDocument == false; - if (objectWithFallbackSyntheticSource || fieldWithFallbackSyntheticSource || fieldWithStoredArraySource @@ -776,9 +767,6 @@ private static void parseNonDynamicArray( && context.isImmediateParentAnArray()) { context.getOffSetContext().maybeRecordEmptyArray(mapper.getOffsetFieldName()); } - if (elements <= 1 && canRemoveSingleLeafElement) { - context.removeLastIgnoredField(fullPath); - } postProcessDynamicArrayMapping(context, lastFieldName); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java index b77c0426c23d4..69faec23d2d48 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParserContext.java @@ -335,12 +335,6 @@ public final void addIgnoredField(IgnoredSourceFieldMapper.NameValue values) { } } - final void removeLastIgnoredField(String name) { - if (ignoredFieldValues.isEmpty() == false && ignoredFieldValues.getLast().name().equals(name)) { - ignoredFieldValues.removeLast(); - } - } - /** * Return the collection of values for fields that have been ignored so far. */ diff --git a/x-pack/plugin/logsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/SyntheticSourceRollingUpgradeIT.java b/x-pack/plugin/logsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/SyntheticSourceRollingUpgradeIT.java new file mode 100644 index 0000000000000..182beee18f591 --- /dev/null +++ b/x-pack/plugin/logsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/SyntheticSourceRollingUpgradeIT.java @@ -0,0 +1,246 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.upgrades; + +import com.carrotsearch.randomizedtesting.annotations.Name; + +import org.elasticsearch.client.Request; +import org.elasticsearch.test.rest.ObjectPath; +import org.hamcrest.Matchers; + +import java.time.Instant; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.elasticsearch.upgrades.LogsIndexModeRollingUpgradeIT.enableLogsdbByDefault; +import static org.elasticsearch.upgrades.LogsIndexModeRollingUpgradeIT.getWriteBackingIndex; +import static org.elasticsearch.upgrades.MatchOnlyTextRollingUpgradeIT.createTemplate; +import static org.elasticsearch.upgrades.MatchOnlyTextRollingUpgradeIT.formatInstant; +import static org.elasticsearch.upgrades.MatchOnlyTextRollingUpgradeIT.getIndexSettingsWithDefaults; +import static org.elasticsearch.upgrades.MatchOnlyTextRollingUpgradeIT.startTrial; +import static org.hamcrest.Matchers.both; +import static org.hamcrest.Matchers.emptyIterable; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.lessThanOrEqualTo; +import static org.hamcrest.Matchers.not; + +public class SyntheticSourceRollingUpgradeIT extends AbstractRollingUpgradeWithSecurityTestCase { + + static String BULK_ITEM_TEMPLATE = """ + {"@timestamp": "$now", "field1": "$field1", "field2": $field2, "field3": $field3, "field4": $field4} + """; + + private static final String TEMPLATE = """ + { + "mappings": { + "properties": { + "@timestamp" : { + "type": "date" + }, + "field1": { + "type": "keyword" + }, + "field2": { + "type": "keyword" + }, + "field3": { + "type": "long" + }, + "field4": { + "type": "long" + } + } + } + }"""; + + private static final Integer[] VALUES = new Integer[] { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }; + + public SyntheticSourceRollingUpgradeIT(@Name("upgradedNodes") int upgradedNodes) { + super(upgradedNodes); + } + + public void testIndexing() throws Exception { + String dataStreamName = "logs-bwc-test"; + if (isOldCluster()) { + startTrial(); + enableLogsdbByDefault(); + createTemplate(dataStreamName, getClass().getSimpleName().toLowerCase(Locale.ROOT), TEMPLATE); + + Instant startTime = Instant.now().minusSeconds(60 * 60); + bulkIndex(dataStreamName, 4, 1024, startTime); + + String firstBackingIndex = getWriteBackingIndex(client(), dataStreamName, 0); + var settings = (Map) getIndexSettingsWithDefaults(firstBackingIndex).get(firstBackingIndex); + assertThat(((Map) settings.get("settings")).get("index.mode"), equalTo("logsdb")); + assertThat(((Map) settings.get("defaults")).get("index.mapping.source.mode"), equalTo("SYNTHETIC")); + + ensureGreen(dataStreamName); + search(dataStreamName); + query(dataStreamName); + } else if (isMixedCluster()) { + Instant startTime = Instant.now().minusSeconds(60 * 30); + bulkIndex(dataStreamName, 4, 1024, startTime); + + ensureGreen(dataStreamName); + search(dataStreamName); + query(dataStreamName); + } else if (isUpgradedCluster()) { + ensureGreen(dataStreamName); + Instant startTime = Instant.now(); + bulkIndex(dataStreamName, 4, 1024, startTime); + search(dataStreamName); + query(dataStreamName); + + var forceMergeRequest = new Request("POST", "/" + dataStreamName + "/_forcemerge"); + forceMergeRequest.addParameter("max_num_segments", "1"); + assertOK(client().performRequest(forceMergeRequest)); + + ensureGreen(dataStreamName); + search(dataStreamName); + query(dataStreamName); + } + } + + static String bulkIndex(String dataStreamName, int numRequest, int numDocs, Instant startTime) throws Exception { + String firstIndex = null; + for (int i = 0; i < numRequest; i++) { + var bulkRequest = new Request("POST", "/" + dataStreamName + "/_bulk"); + StringBuilder requestBody = new StringBuilder(); + for (int j = 0; j < numDocs; j++) { + String field1 = Integer.toString(randomFrom(VALUES)); + var randomArray = randomArray(1, 3, Integer[]::new, () -> randomFrom(VALUES)); + String field2 = Arrays.stream(randomArray).map(s -> "\"" + s + "\"").collect(Collectors.joining(",")); + int field3 = randomFrom(VALUES); + String field4 = Arrays.stream(randomArray).map(String::valueOf).collect(Collectors.joining(",")); + + requestBody.append("{\"create\": {}}"); + requestBody.append('\n'); + requestBody.append( + BULK_ITEM_TEMPLATE.replace("$now", formatInstant(startTime)) + .replace("$field1", field1) + .replace("$field2", "[" + field2 + "]") + .replace("$field3", Long.toString(field3)) + .replace("$field4", "[" + field4 + "]") + ); + requestBody.append('\n'); + + startTime = startTime.plusMillis(1); + } + bulkRequest.setJsonEntity(requestBody.toString()); + bulkRequest.addParameter("refresh", "true"); + var response = client().performRequest(bulkRequest); + assertOK(response); + var responseBody = entityAsMap(response); + assertThat("errors in response:\n " + responseBody, responseBody.get("errors"), equalTo(false)); + if (firstIndex == null) { + firstIndex = (String) ((Map) ((Map) ((List) responseBody.get("items")).get(0)).get("create")).get("_index"); + } + } + return firstIndex; + } + + void search(String dataStreamName) throws Exception { + var searchRequest = new Request("POST", "/" + dataStreamName + "/_search"); + searchRequest.addParameter("pretty", "true"); + searchRequest.setJsonEntity(""" + { + "size": 500 + } + """); + var response = client().performRequest(searchRequest); + assertOK(response); + var responseBody = entityAsMap(response); + logger.info("{}", responseBody); + assertThat(ObjectPath.evaluate(responseBody, "_shards.failed"), Matchers.equalTo(0)); + Integer totalCount = ObjectPath.evaluate(responseBody, "hits.total.value"); + assertThat(totalCount, greaterThanOrEqualTo(512)); + + Map firstSource = ObjectPath.evaluate(responseBody, "hits.hits.0._source"); + Integer field1 = Integer.valueOf((String) firstSource.get("field1")); + assertThat(field1, both(greaterThanOrEqualTo(0)).and(lessThanOrEqualTo(9))); + List field2 = (List) firstSource.get("field2"); + assertThat(field2, not(emptyIterable())); + for (var e : field2) { + Integer value = Integer.valueOf((String) e); + assertThat(value, both(greaterThanOrEqualTo(0)).and(lessThanOrEqualTo(9))); + } + Integer field3 = (Integer) firstSource.get("field3"); + assertThat(field3, both(greaterThanOrEqualTo(0)).and(lessThanOrEqualTo(9))); + List field4 = (List) firstSource.get("field4"); + assertThat(field4, not(emptyIterable())); + for (var e : field4) { + Integer value = (Integer) e; + assertThat(value, both(greaterThanOrEqualTo(0)).and(lessThanOrEqualTo(9))); + } + } + + void query(String dataStreamName) throws Exception { + var queryRequest = new Request("POST", "/_query"); + queryRequest.addParameter("pretty", "true"); + queryRequest.setJsonEntity(""" + { + "query": "FROM $ds | SORT @timestamp | KEEP field1,field2,field3,field4 | LIMIT 5" + } + """.replace("$ds", dataStreamName)); + var response = client().performRequest(queryRequest); + assertOK(response); + var responseBody = entityAsMap(response); + logger.info("{}", responseBody); + + String column1 = ObjectPath.evaluate(responseBody, "columns.0.name"); + String column2 = ObjectPath.evaluate(responseBody, "columns.1.name"); + String column3 = ObjectPath.evaluate(responseBody, "columns.2.name"); + String column4 = ObjectPath.evaluate(responseBody, "columns.3.name"); + assertThat(column1, equalTo("field1")); + assertThat(column2, equalTo("field2")); + assertThat(column3, equalTo("field3")); + assertThat(column4, equalTo("field4")); + + { + var field1 = Integer.valueOf(ObjectPath.evaluate(responseBody, "values.0.0")); + assertThat(field1, both(greaterThanOrEqualTo(0)).and(lessThanOrEqualTo(9))); + } + { + var object = ObjectPath.evaluate(responseBody, "values.0.1"); + if (object instanceof List field2) { + assertThat(field2, not(emptyIterable())); + for (var e : field2) { + Integer value = Integer.valueOf((String) e); + assertThat(value, both(greaterThanOrEqualTo(0)).and(lessThanOrEqualTo(9))); + } + } else { + Integer field2 = Integer.valueOf((String) object); + assertThat(field2, both(greaterThanOrEqualTo(0)).and(lessThanOrEqualTo(9))); + } + } + { + Integer field3 = ObjectPath.evaluate(responseBody, "values.0.2"); + assertThat(field3, both(greaterThanOrEqualTo(0)).and(lessThanOrEqualTo(9))); + } + { + var object = ObjectPath.evaluate(responseBody, "values.0.3"); + if (object instanceof List field4) { + assertThat(field4, not(emptyIterable())); + for (var e : field4) { + Integer value = (Integer) e; + assertThat(value, both(greaterThanOrEqualTo(0)).and(lessThanOrEqualTo(9))); + } + } else { + Integer field4 = (Integer) object; + assertThat(field4, both(greaterThanOrEqualTo(0)).and(lessThanOrEqualTo(9))); + } + } + } + +}