From 004d04ce764964bf189c370406e9f2fe2e72cece Mon Sep 17 00:00:00 2001 From: Dmitry Kubikov Date: Tue, 2 Sep 2025 12:57:57 -0700 Subject: [PATCH 01/23] Added keyword multi field with ignore_above to match only text bwc tests --- .../index/mapper/KeywordFieldMapper.java | 1 + .../org/elasticsearch/test/ESTestCase.java | 13 +- .../MatchOnlyTextRollingUpgradeIT.java | 169 +++++++++++------- 3 files changed, 114 insertions(+), 69 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index b6d9215db030b..91a680a187376 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -482,6 +482,7 @@ public KeywordFieldMapper build(MapperBuilderContext context) { indexCreatedVersion, IndexVersions.SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_KEYWORD ); + offsetsFieldName = null; return new KeywordFieldMapper( leafName(), fieldtype, diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java index 43243e431db48..0fd0cbf494013 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java @@ -20,7 +20,6 @@ import com.carrotsearch.randomizedtesting.generators.RandomPicks; import com.carrotsearch.randomizedtesting.generators.RandomStrings; import com.carrotsearch.randomizedtesting.rules.TestRuleAdapter; - import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -1242,6 +1241,18 @@ public static String randomAlphaOfLength(int codeUnits) { return RandomizedTest.randomAsciiOfLength(codeUnits); } + /** + * Generates a string containing a random number of random length alphas, all delimited by space. + */ + public static String randomAlphasDelimitedBySpace(int maxAlphas, int minCodeUnits, int maxCodeUnits) { + int numAlphas = randomIntBetween(1, maxAlphas); + List alphas = new ArrayList<>(numAlphas); + for (int i = 0; i < numAlphas; i++) { + alphas.add(randomAlphaOfLengthBetween(minCodeUnits, maxCodeUnits)); + } + return String.join(" ", alphas); + } + /** * Generate a random string containing only alphanumeric characters. * The locale for the string is {@link Locale#ROOT}. diff --git a/x-pack/plugin/logsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/MatchOnlyTextRollingUpgradeIT.java b/x-pack/plugin/logsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/MatchOnlyTextRollingUpgradeIT.java index df4ef4db1b29d..d4e6755f07743 100644 --- a/x-pack/plugin/logsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/MatchOnlyTextRollingUpgradeIT.java +++ b/x-pack/plugin/logsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/MatchOnlyTextRollingUpgradeIT.java @@ -10,7 +10,6 @@ package org.elasticsearch.upgrades; import com.carrotsearch.randomizedtesting.annotations.Name; - import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; @@ -37,11 +36,19 @@ public class MatchOnlyTextRollingUpgradeIT extends AbstractRollingUpgradeWithSecurityTestCase { + private static final String DATA_STREAM = "logs-bwc-test"; + + private static final int IGNORE_ABOVE_MAX = 256; + private static final int NUM_REQUESTS = 4; + private static final int NUM_DOCS_PER_REQUEST = 1024; + static String BULK_ITEM_TEMPLATE = """ + { "create": {} } {"@timestamp": "$now", "host.name": "$host", "method": "$method", "ip": "$ip", "message": "$message", "length": $length, "factor": $factor} """; + // "ignore_above": "$IGNORE_ABOVE" private static final String TEMPLATE = """ { "mappings": { @@ -53,7 +60,14 @@ public class MatchOnlyTextRollingUpgradeIT extends AbstractRollingUpgradeWithSec "type": "keyword" }, "message": { - "type": "match_only_text" + "type": "match_only_text", + "fields": { + "keyword": { + "type": "keyword", + "store": true, + "ignore_above": "$IGNORE_ABOVE" + } + } }, "ip": { "type": "ip" @@ -73,50 +87,67 @@ public MatchOnlyTextRollingUpgradeIT(@Name("upgradedNodes") int upgradedNodes) { } public void testIndexing() throws Exception { - String dataStreamName = "logs-bwc-test"; + if (isOldCluster()) { + // given - enable logsdb and create a template startTrial(); enableLogsdbByDefault(); - createTemplate(dataStreamName, getClass().getSimpleName().toLowerCase(Locale.ROOT), TEMPLATE); + String templateId = getClass().getSimpleName().toLowerCase(Locale.ROOT); + createTemplate(DATA_STREAM, templateId, prepareTemplate()); - Instant startTime = Instant.now().minusSeconds(60 * 60); - bulkIndex(dataStreamName, 4, 1024, startTime); + // when - index some documents + bulkIndex(NUM_REQUESTS, NUM_DOCS_PER_REQUEST); - String firstBackingIndex = getWriteBackingIndex(client(), dataStreamName, 0); + // then - verify that logsdb and synthetic source are both enabled + String firstBackingIndex = getWriteBackingIndex(client(), DATA_STREAM, 0); var settings = (Map) getIndexSettingsWithDefaults(firstBackingIndex).get(firstBackingIndex); assertThat(((Map) settings.get("settings")).get("index.mode"), equalTo("logsdb")); assertThat(((Map) settings.get("defaults")).get("index.mapping.source.mode"), equalTo("SYNTHETIC")); - ensureGreen(dataStreamName); - search(dataStreamName); - query(dataStreamName); + // then continued - verify that the created data stream using the created template + LogsdbIndexingRollingUpgradeIT.assertDataStream(DATA_STREAM, templateId); + + // when/then - run some queries and verify results + ensureGreen(DATA_STREAM); + search(DATA_STREAM); + query(DATA_STREAM); + } else if (isMixedCluster()) { - Instant startTime = Instant.now().minusSeconds(60 * 30); - bulkIndex(dataStreamName, 4, 1024, startTime); + // when + bulkIndex(NUM_REQUESTS, NUM_DOCS_PER_REQUEST); + + // when/then + ensureGreen(DATA_STREAM); + search(DATA_STREAM); + query(DATA_STREAM); - ensureGreen(dataStreamName); - search(dataStreamName); - query(dataStreamName); } else if (isUpgradedCluster()) { - ensureGreen(dataStreamName); - Instant startTime = Instant.now(); - bulkIndex(dataStreamName, 4, 1024, startTime); - search(dataStreamName); - query(dataStreamName); + // when/then + ensureGreen(DATA_STREAM); + bulkIndex(NUM_REQUESTS, NUM_DOCS_PER_REQUEST); + search(DATA_STREAM); + query(DATA_STREAM); - var forceMergeRequest = new Request("POST", "/" + dataStreamName + "/_forcemerge"); + // when/then continued - force merge all shard segments into one + var forceMergeRequest = new Request("POST", "/" + DATA_STREAM + "/_forcemerge"); forceMergeRequest.addParameter("max_num_segments", "1"); assertOK(client().performRequest(forceMergeRequest)); - ensureGreen(dataStreamName); - search(dataStreamName); - query(dataStreamName); + // then continued + ensureGreen(DATA_STREAM); + search(DATA_STREAM); + query(DATA_STREAM); } } + private String prepareTemplate() { + return TEMPLATE.replace("$IGNORE_ABOVE", String.valueOf(randomInt(IGNORE_ABOVE_MAX))); + } + static void createTemplate(String dataStreamName, String id, String template) throws IOException { final String INDEX_TEMPLATE = """ { + "priority": 500, "index_patterns": ["$DATASTREAM"], "template": $TEMPLATE, "data_stream": { @@ -127,46 +158,52 @@ static void createTemplate(String dataStreamName, String id, String template) th assertOK(client().performRequest(putIndexTemplateRequest)); } - static String bulkIndex(String dataStreamName, int numRequest, int numDocs, Instant startTime) throws Exception { + private void bulkIndex(int numRequest, int numDocs) throws Exception { String firstIndex = null; + Instant startTime = Instant.now().minusSeconds(60 * 60); + for (int i = 0; i < numRequest; i++) { - var bulkRequest = new Request("POST", "/" + dataStreamName + "/_bulk"); - StringBuilder requestBody = new StringBuilder(); - for (int j = 0; j < numDocs; j++) { - String hostName = "host" + j % 50; // Not realistic, but makes asserting search / query response easier. - String methodName = "method" + j % 5; - String ip = NetworkAddress.format(randomIp(true)); - String param = "chicken" + randomInt(5); - String message = "the quick brown fox jumps over the " + param; - long length = randomLong(); - double factor = randomDouble(); - - requestBody.append("{\"create\": {}}"); - requestBody.append('\n'); - requestBody.append( - BULK_ITEM_TEMPLATE.replace("$now", formatInstant(startTime)) - .replace("$host", hostName) - .replace("$method", methodName) - .replace("$ip", ip) - .replace("$message", message) - .replace("$length", Long.toString(length)) - .replace("$factor", Double.toString(factor)) - ); - requestBody.append('\n'); - - startTime = startTime.plusMillis(1); - } - bulkRequest.setJsonEntity(requestBody.toString()); + var bulkRequest = new Request("POST", "/" + DATA_STREAM + "/_bulk"); + bulkRequest.setJsonEntity(bulkIndexRequestBody(numDocs, startTime)); bulkRequest.addParameter("refresh", "true"); + var response = client().performRequest(bulkRequest); - assertOK(response); var responseBody = entityAsMap(response); + + assertOK(response); assertThat("errors in response:\n " + responseBody, responseBody.get("errors"), equalTo(false)); if (firstIndex == null) { firstIndex = (String) ((Map) ((Map) ((List) responseBody.get("items")).get(0)).get("create")).get("_index"); } } - return firstIndex; + } + + private String bulkIndexRequestBody(int numDocs, Instant startTime) { + StringBuilder requestBody = new StringBuilder(); + + for (int j = 0; j < numDocs; j++) { + String hostName = "host" + j % 50; // Not realistic, but makes asserting search / query response easier. + String methodName = "method" + j % 5; + String ip = NetworkAddress.format(randomIp(true)); + String message = randomAlphasDelimitedBySpace(10, 1, 15); + long length = randomLong(); + double factor = randomDouble(); + + requestBody.append( + BULK_ITEM_TEMPLATE.replace("$now", formatInstant(startTime)) + .replace("$host", hostName) + .replace("$method", methodName) + .replace("$ip", ip) + .replace("$message", message) + .replace("$length", Long.toString(length)) + .replace("$factor", Double.toString(factor)) + ); + requestBody.append('\n'); + + startTime = startTime.plusMillis(1); + } + + return requestBody.toString(); } void search(String dataStreamName) throws Exception { @@ -174,34 +211,30 @@ void search(String dataStreamName) throws Exception { searchRequest.addParameter("pretty", "true"); searchRequest.setJsonEntity(""" { - "size": 500, - "query": { - "match_phrase": { - "message": "chicken" - } - } + "size": 500 } - """.replace("chicken", "chicken" + randomInt(5))); + """); var response = client().performRequest(searchRequest); assertOK(response); var responseBody = entityAsMap(response); logger.info("{}", responseBody); Integer totalCount = ObjectPath.evaluate(responseBody, "hits.total.value"); - assertThat(totalCount, greaterThanOrEqualTo(512)); + assertThat(totalCount, greaterThanOrEqualTo(NUM_REQUESTS * NUM_DOCS_PER_REQUEST)); } - void query(String dataStreamName) throws Exception { + private void query(String dataStreamName) throws Exception { var queryRequest = new Request("POST", "/_query"); queryRequest.addParameter("pretty", "true"); queryRequest.setJsonEntity(""" { - "query": "FROM $ds | STATS max(length), max(factor) BY message | SORT message | LIMIT 5" + "query": "FROM $ds | STATS max(length), max(factor) BY message.keyword | SORT message.keyword | LIMIT 5" } """.replace("$ds", dataStreamName)); var response = client().performRequest(queryRequest); assertOK(response); var responseBody = entityAsMap(response); + logger.info("potato response body"); logger.info("{}", responseBody); String column1 = ObjectPath.evaluate(responseBody, "columns.0.name"); @@ -209,14 +242,14 @@ void query(String dataStreamName) throws Exception { String column3 = ObjectPath.evaluate(responseBody, "columns.2.name"); assertThat(column1, equalTo("max(length)")); assertThat(column2, equalTo("max(factor)")); - assertThat(column3, equalTo("message")); + assertThat(column3, equalTo("message.keyword")); - String key = ObjectPath.evaluate(responseBody, "values.0.2"); - assertThat(key, equalTo("the quick brown fox jumps over the chicken0")); Long maxRx = ObjectPath.evaluate(responseBody, "values.0.0"); - assertThat(maxRx, notNullValue()); Double maxTx = ObjectPath.evaluate(responseBody, "values.0.1"); + String key = ObjectPath.evaluate(responseBody, "values.0.2"); assertThat(maxTx, notNullValue()); + assertThat(maxRx, notNullValue()); + assertThat(key, equalTo("banana boy")); } protected static void startTrial() throws IOException { From be849ea5f28975e3f812cc704310588f19138517 Mon Sep 17 00:00:00 2001 From: Dmitry Kubikov Date: Wed, 3 Sep 2025 16:24:52 -0700 Subject: [PATCH 02/23] Rework ignore_above --- .../icu/ICUCollationKeywordFieldMapper.java | 8 +- .../elasticsearch/index/IndexSettings.java | 17 +- .../index/mapper/FieldMapper.java | 17 + .../index/mapper/KeywordFieldMapper.java | 44 +-- .../index/mapper/TextFieldMapper.java | 4 +- .../flattened/FlattenedFieldMapper.java | 33 +- .../MatchOnlyTextRollingUpgradeIT.java | 7 +- .../upgrades/TextRollingUpgradeIT.java | 296 ++++++++++++++++++ .../wildcard/mapper/WildcardFieldMapper.java | 26 +- 9 files changed, 370 insertions(+), 82 deletions(-) create mode 100644 x-pack/plugin/logsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/TextRollingUpgradeIT.java diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapper.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapper.java index 6b5c43ac3daa6..cd80164ff2c03 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapper.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapper.java @@ -13,7 +13,6 @@ import com.ibm.icu.text.RawCollationKey; import com.ibm.icu.text.RuleBasedCollator; import com.ibm.icu.util.ULocale; - import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.SortedSetDocValuesField; @@ -250,12 +249,7 @@ public static class Builder extends FieldMapper.Builder { false ).acceptsNull(); - final Parameter ignoreAbove = Parameter.intParam("ignore_above", true, m -> toType(m).ignoreAbove, Integer.MAX_VALUE) - .addValidator(v -> { - if (v < 0) { - throw new IllegalArgumentException("[ignore_above] must be positive, got [" + v + "]"); - } - }); + final Parameter ignoreAbove = Parameter.ignoreAboveParam(m -> toType(m).ignoreAbove); final Parameter nullValue = Parameter.stringParam("null_value", false, m -> toType(m).nullValue, null).acceptsNull(); public Builder(String name) { diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index 636c30027bd0b..8f1f6d4ef6b2f 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -803,6 +803,9 @@ public Iterator> settings() { * occupy at most 4 bytes. */ + private static final int IGNORE_ABOVE_DEFAULT = Integer.MAX_VALUE; + private static final int IGNORE_ABOVE_DEFAULT_LOGSDB = 8191; + public static final Setting IGNORE_ABOVE_SETTING = Setting.intSetting( "index.mapping.ignore_above", IndexSettings::getIgnoreAboveDefaultValue, @@ -813,11 +816,17 @@ public Iterator> settings() { ); private static String getIgnoreAboveDefaultValue(final Settings settings) { - if (IndexSettings.MODE.get(settings) == IndexMode.LOGSDB - && IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings).onOrAfter(IndexVersions.ENABLE_IGNORE_ABOVE_LOGSDB)) { - return "8191"; + return String.valueOf( + getIgnoreAboveDefaultValue(IndexSettings.MODE.get(settings), IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings)) + ); + } + + public static int getIgnoreAboveDefaultValue(final IndexMode indexMode, final IndexVersion indexCreatedVersion) { + if (indexMode == IndexMode.LOGSDB + && (indexCreatedVersion != null && indexCreatedVersion.onOrAfter(IndexVersions.ENABLE_IGNORE_ABOVE_LOGSDB))) { + return IGNORE_ABOVE_DEFAULT_LOGSDB; } else { - return String.valueOf(Integer.MAX_VALUE); + return IGNORE_ABOVE_DEFAULT; } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index 34362e25bfece..048f5a2254cbe 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -1337,6 +1337,23 @@ public static Parameter normsParam(Function initi .setMergeValidator((prev, curr, c) -> prev == curr || (prev && curr == false)); } + public static Parameter ignoreAboveParam(Function initializer) { + return ignoreAboveParam(initializer, null, null); + } + + public static Parameter ignoreAboveParam( + Function initializer, + final IndexMode indexMode, + final IndexVersion indexCreatedVersion + ) { + final int defaultValue = IndexSettings.getIgnoreAboveDefaultValue(indexMode, indexCreatedVersion); + return Parameter.intParam("ignore_above", true, initializer, defaultValue).addValidator(v -> { + if (v < 0) { + throw new IllegalArgumentException("[ignore_above] must be positive, got [" + v + "]"); + } + }); + } + /** * Defines a script parameter * @param initializer retrieves the equivalent parameter from an existing FieldMapper for use in merges diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 91a680a187376..d6702ffe5c154 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -89,7 +89,6 @@ import static org.apache.lucene.index.IndexWriter.MAX_TERM_LENGTH; import static org.elasticsearch.core.Strings.format; -import static org.elasticsearch.index.IndexSettings.IGNORE_ABOVE_SETTING; import static org.elasticsearch.index.IndexSettings.USE_DOC_VALUES_SKIPPER; import static org.elasticsearch.index.mapper.FieldArrayContext.getOffsetsFieldName; @@ -179,7 +178,6 @@ public static final class Builder extends FieldMapper.DimensionBuilder { false ); private final Parameter ignoreAbove; - private final int ignoreAboveDefault; private final IndexSortConfig indexSortConfig; private final IndexMode indexMode; private final Parameter indexOptions = TextParams.keywordIndexOptions(m -> toType(m).indexOptions); @@ -218,7 +216,6 @@ public Builder(final String name, final MappingParserContext mappingParserContex name, mappingParserContext.getIndexAnalyzers(), mappingParserContext.scriptCompiler(), - IGNORE_ABOVE_SETTING.get(mappingParserContext.getSettings()), mappingParserContext.getIndexSettings().getIndexVersionCreated(), mappingParserContext.getIndexSettings().getMode(), mappingParserContext.getIndexSettings().getIndexSortConfig(), @@ -232,29 +229,16 @@ public Builder(final String name, final MappingParserContext mappingParserContex String name, IndexAnalyzers indexAnalyzers, ScriptCompiler scriptCompiler, - int ignoreAboveDefault, IndexVersion indexCreatedVersion, SourceKeepMode sourceKeepMode ) { - this( - name, - indexAnalyzers, - scriptCompiler, - ignoreAboveDefault, - indexCreatedVersion, - IndexMode.STANDARD, - null, - false, - false, - sourceKeepMode - ); + this(name, indexAnalyzers, scriptCompiler, indexCreatedVersion, IndexMode.STANDARD, null, false, false, sourceKeepMode); } private Builder( String name, IndexAnalyzers indexAnalyzers, ScriptCompiler scriptCompiler, - int ignoreAboveDefault, IndexVersion indexCreatedVersion, IndexMode indexMode, IndexSortConfig indexSortConfig, @@ -288,13 +272,7 @@ private Builder( ); } }).precludesParameters(normalizer); - this.ignoreAboveDefault = ignoreAboveDefault; - this.ignoreAbove = Parameter.intParam("ignore_above", true, m -> toType(m).fieldType().ignoreAbove(), ignoreAboveDefault) - .addValidator(v -> { - if (v < 0) { - throw new IllegalArgumentException("[ignore_above] must be positive, got [" + v + "]"); - } - }); + this.ignoreAbove = Parameter.ignoreAboveParam(m -> toType(m).fieldType().ignoreAbove(), indexMode, indexCreatedVersion); this.indexSortConfig = indexSortConfig; this.indexMode = indexMode; this.enableDocValuesSkipper = enableDocValuesSkipper; @@ -303,7 +281,7 @@ private Builder( } public Builder(String name, IndexVersion indexCreatedVersion) { - this(name, null, ScriptCompiler.NONE, Integer.MAX_VALUE, indexCreatedVersion, SourceKeepMode.NONE); + this(name, null, ScriptCompiler.NONE, indexCreatedVersion, SourceKeepMode.NONE); } public static Builder buildWithDocValuesSkipper( @@ -316,7 +294,6 @@ public static Builder buildWithDocValuesSkipper( name, null, ScriptCompiler.NONE, - Integer.MAX_VALUE, indexCreatedVersion, indexMode, // Sort config is used to decide if DocValueSkippers can be used. Since skippers are forced, a sort config is not needed. @@ -482,7 +459,6 @@ public KeywordFieldMapper build(MapperBuilderContext context) { indexCreatedVersion, IndexVersions.SYNTHETIC_SOURCE_STORE_ARRAYS_NATIVELY_KEYWORD ); - offsetsFieldName = null; return new KeywordFieldMapper( leafName(), fieldtype, @@ -539,6 +515,7 @@ private static boolean indexSortConfigByHostName(final IndexSortConfig indexSort public static final class KeywordFieldType extends StringFieldType { private final int ignoreAbove; + private final int ignoreAboveDefaultValue; private final String nullValue; private final NamedAnalyzer normalizer; private final boolean eagerGlobalOrdinals; @@ -570,6 +547,7 @@ public KeywordFieldType( this.eagerGlobalOrdinals = builder.eagerGlobalOrdinals.getValue(); this.normalizer = normalizer; this.ignoreAbove = builder.ignoreAbove.getValue(); + this.ignoreAboveDefaultValue = builder.ignoreAbove.getDefaultValue(); this.nullValue = builder.nullValue.getValue(); this.scriptValues = builder.scriptValues(); this.isDimension = builder.dimension.getValue(); @@ -584,6 +562,7 @@ public KeywordFieldType(String name, boolean isIndexed, boolean hasDocValues, Ma super(name, isIndexed, false, hasDocValues, TextSearchInfo.SIMPLE_MATCH_ONLY, meta); this.normalizer = Lucene.KEYWORD_ANALYZER; this.ignoreAbove = Integer.MAX_VALUE; + this.ignoreAboveDefaultValue = Integer.MAX_VALUE; this.nullValue = null; this.eagerGlobalOrdinals = false; this.scriptValues = null; @@ -610,6 +589,7 @@ public KeywordFieldType(String name, FieldType fieldType) { ); this.normalizer = Lucene.KEYWORD_ANALYZER; this.ignoreAbove = Integer.MAX_VALUE; + this.ignoreAboveDefaultValue = Integer.MAX_VALUE; this.nullValue = null; this.eagerGlobalOrdinals = false; this.scriptValues = null; @@ -625,6 +605,7 @@ public KeywordFieldType(String name, NamedAnalyzer analyzer) { super(name, true, false, true, textSearchInfo(Defaults.FIELD_TYPE, null, analyzer, analyzer), Collections.emptyMap()); this.normalizer = Lucene.KEYWORD_ANALYZER; this.ignoreAbove = Integer.MAX_VALUE; + this.ignoreAboveDefaultValue = Integer.MAX_VALUE; this.nullValue = null; this.eagerGlobalOrdinals = false; this.scriptValues = null; @@ -1065,6 +1046,10 @@ public int ignoreAbove() { return ignoreAbove; } + public boolean isIgnoreAboveSet() { + return ignoreAbove != ignoreAboveDefaultValue; + } + @Override public boolean isDimension() { return isDimension; @@ -1124,7 +1109,6 @@ public String originalName() { private final boolean isSyntheticSource; private final IndexAnalyzers indexAnalyzers; - private final int ignoreAboveDefault; private final IndexMode indexMode; private final IndexSortConfig indexSortConfig; private final boolean enableDocValuesSkipper; @@ -1156,7 +1140,6 @@ private KeywordFieldMapper( this.scriptCompiler = builder.scriptCompiler; this.indexCreatedVersion = builder.indexCreatedVersion; this.isSyntheticSource = isSyntheticSource; - this.ignoreAboveDefault = builder.ignoreAboveDefault; this.indexMode = builder.indexMode; this.indexSortConfig = builder.indexSortConfig; this.enableDocValuesSkipper = builder.enableDocValuesSkipper; @@ -1312,7 +1295,6 @@ public FieldMapper.Builder getMergeBuilder() { leafName(), indexAnalyzers, scriptCompiler, - ignoreAboveDefault, indexCreatedVersion, indexMode, indexSortConfig, @@ -1386,7 +1368,7 @@ protected BytesRef preserve(BytesRef value) { } } - if (fieldType().ignoreAbove != Integer.MAX_VALUE) { + if (fieldType().isIgnoreAboveSet()) { layers.add(new CompositeSyntheticFieldLoader.StoredFieldLayer(originalName) { @Override protected void writeValue(Object value, XContentBuilder b) throws IOException { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index 8a586d68177e0..572e40135897d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -1021,7 +1021,7 @@ public boolean isAggregatable() { * A delegate by definition must have doc_values or be stored so most of the time it can be used for loading. */ public boolean canUseSyntheticSourceDelegateForLoading() { - return syntheticSourceDelegate != null && syntheticSourceDelegate.ignoreAbove() == Integer.MAX_VALUE; + return syntheticSourceDelegate != null && syntheticSourceDelegate.isIgnoreAboveSet() == false; } /** @@ -1029,7 +1029,7 @@ public boolean canUseSyntheticSourceDelegateForLoading() { */ public boolean canUseSyntheticSourceDelegateForQuerying() { return syntheticSourceDelegate != null - && syntheticSourceDelegate.ignoreAbove() == Integer.MAX_VALUE + && syntheticSourceDelegate.isIgnoreAboveSet() == false && syntheticSourceDelegate.isIndexed(); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java index f70a47caa080c..8d2a586335cec 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java @@ -38,6 +38,8 @@ import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.core.Nullable; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.fielddata.FieldData; import org.elasticsearch.index.fielddata.FieldDataContext; @@ -82,8 +84,6 @@ import java.util.Set; import java.util.function.Function; -import static org.elasticsearch.index.IndexSettings.IGNORE_ABOVE_SETTING; - /** * A field mapper that accepts a JSON object and flattens it into a single field. This data type * can be a useful alternative to an 'object' mapping when the object has a large, unknown set @@ -124,7 +124,6 @@ private static Builder builder(Mapper in) { return ((FlattenedFieldMapper) in).builder; } - private final int ignoreAboveDefault; private final int ignoreAbove; public static class Builder extends FieldMapper.Builder { @@ -152,7 +151,6 @@ public static class Builder extends FieldMapper.Builder { m -> builder(m).eagerGlobalOrdinals.get(), false ); - private final int ignoreAboveDefault; private final Parameter ignoreAbove; private final Parameter indexOptions = TextParams.keywordIndexOptions(m -> builder(m).indexOptions.get()); @@ -180,23 +178,22 @@ public static class Builder extends FieldMapper.Builder { private final Parameter> meta = Parameter.metaParam(); + private final IndexMode indexMode; + private final IndexVersion indexCreatedVersion; + public static FieldMapper.Parameter> dimensionsParam(Function> initializer) { return FieldMapper.Parameter.stringArrayParam(TIME_SERIES_DIMENSIONS_ARRAY_PARAM, false, initializer); } public Builder(final String name) { - this(name, Integer.MAX_VALUE); + this(name, IndexMode.STANDARD, IndexVersion.current()); } - private Builder(String name, int ignoreAboveDefault) { + private Builder(String name, IndexMode indexMode, IndexVersion indexCreatedVersion) { super(name); - this.ignoreAboveDefault = ignoreAboveDefault; - this.ignoreAbove = Parameter.intParam("ignore_above", true, m -> builder(m).ignoreAbove.get(), ignoreAboveDefault) - .addValidator(v -> { - if (v < 0) { - throw new IllegalArgumentException("[ignore_above] must be positive, got [" + v + "]"); - } - }); + this.indexMode = indexMode; + this.indexCreatedVersion = indexCreatedVersion; + this.ignoreAbove = Parameter.ignoreAboveParam(m -> builder(m).ignoreAbove.get(), indexMode, indexCreatedVersion); this.dimensions.precludesParameters(ignoreAbove); } @@ -235,11 +232,13 @@ public FlattenedFieldMapper build(MapperBuilderContext context) { dimensions.get(), ignoreAbove.getValue() ); - return new FlattenedFieldMapper(leafName(), ft, builderParams(this, context), ignoreAboveDefault, this); + return new FlattenedFieldMapper(leafName(), ft, builderParams(this, context), this); } } - public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n, IGNORE_ABOVE_SETTING.get(c.getSettings()))); + public static final TypeParser PARSER = new TypeParser( + (n, c) -> new Builder(n, c.getIndexSettings().getMode(), c.indexVersionCreated()) + ); /** * A field type that represents the values under a particular JSON key, used @@ -832,11 +831,9 @@ private FlattenedFieldMapper( String leafName, MappedFieldType mappedFieldType, BuilderParams builderParams, - int ignoreAboveDefault, Builder builder ) { super(leafName, mappedFieldType, builderParams); - this.ignoreAboveDefault = ignoreAboveDefault; this.builder = builder; this.ignoreAbove = builder.ignoreAbove.get(); this.fieldParser = new FlattenedFieldParser( @@ -905,7 +902,7 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(leafName(), ignoreAboveDefault).init(this); + return new Builder(leafName(), builder.indexMode, builder.indexCreatedVersion).init(this); } @Override diff --git a/x-pack/plugin/logsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/MatchOnlyTextRollingUpgradeIT.java b/x-pack/plugin/logsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/MatchOnlyTextRollingUpgradeIT.java index d4e6755f07743..0f054fc816754 100644 --- a/x-pack/plugin/logsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/MatchOnlyTextRollingUpgradeIT.java +++ b/x-pack/plugin/logsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/MatchOnlyTextRollingUpgradeIT.java @@ -64,8 +64,7 @@ public class MatchOnlyTextRollingUpgradeIT extends AbstractRollingUpgradeWithSec "fields": { "keyword": { "type": "keyword", - "store": true, - "ignore_above": "$IGNORE_ABOVE" + "store": true } } }, @@ -228,7 +227,7 @@ private void query(String dataStreamName) throws Exception { queryRequest.addParameter("pretty", "true"); queryRequest.setJsonEntity(""" { - "query": "FROM $ds | STATS max(length), max(factor) BY message.keyword | SORT message.keyword | LIMIT 5" + "query": "FROM $ds | STATS max(length), max(factor) BY message | SORT message | LIMIT 5" } """.replace("$ds", dataStreamName)); var response = client().performRequest(queryRequest); @@ -242,7 +241,7 @@ private void query(String dataStreamName) throws Exception { String column3 = ObjectPath.evaluate(responseBody, "columns.2.name"); assertThat(column1, equalTo("max(length)")); assertThat(column2, equalTo("max(factor)")); - assertThat(column3, equalTo("message.keyword")); + assertThat(column3, equalTo("message")); Long maxRx = ObjectPath.evaluate(responseBody, "values.0.0"); Double maxTx = ObjectPath.evaluate(responseBody, "values.0.1"); diff --git a/x-pack/plugin/logsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/TextRollingUpgradeIT.java b/x-pack/plugin/logsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/TextRollingUpgradeIT.java new file mode 100644 index 0000000000000..73ad0fd128800 --- /dev/null +++ b/x-pack/plugin/logsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/TextRollingUpgradeIT.java @@ -0,0 +1,296 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.upgrades; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.common.network.NetworkAddress; +import org.elasticsearch.common.time.DateFormatter; +import org.elasticsearch.common.time.FormatNames; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.test.rest.ObjectPath; +import org.elasticsearch.xcontent.XContentType; + +import java.io.IOException; +import java.io.InputStream; +import java.time.Instant; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import static org.elasticsearch.upgrades.StandardToLogsDbIndexModeRollingUpgradeIT.enableLogsdbByDefault; +import static org.elasticsearch.upgrades.StandardToLogsDbIndexModeRollingUpgradeIT.getWriteBackingIndex; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; +import static org.hamcrest.Matchers.notNullValue; + +public class TextRollingUpgradeIT extends AbstractRollingUpgradeWithSecurityTestCase { + + private static final String DATA_STREAM = "logs-bwc-test"; + + private static final int IGNORE_ABOVE_MAX = 256; + private static final int NUM_REQUESTS = 4; + private static final int NUM_DOCS_PER_REQUEST = 1024; + + static String BULK_ITEM_TEMPLATE = + """ + { "create": {} } + {"@timestamp": "$now", "host.name": "$host", "method": "$method", "ip": "$ip", "message": "$message", "length": $length, "factor": $factor} + """; + + private static final String TEMPLATE = """ + { + "mappings": { + "properties": { + "@timestamp" : { + "type": "date" + }, + "method": { + "type": "keyword" + }, + "message": { + "type": "text", + "fields": { + "keyword": { + "ignore_above": $IGNORE_ABOVE, + "type": "keyword" + } + } + }, + "ip": { + "type": "ip" + }, + "length": { + "type": "long" + }, + "factor": { + "type": "double" + } + } + } + }"""; + + // when sorted, this message will appear at the top + private String smallestMessage; + + public TextRollingUpgradeIT(@Name("upgradedNodes") int upgradedNodes) { + super(upgradedNodes); + } + + public void testIndexing() throws Exception { + + if (isOldCluster()) { + // given - enable logsdb and create a template + startTrial(); + enableLogsdbByDefault(); + String templateId = getClass().getSimpleName().toLowerCase(Locale.ROOT); + createTemplate(DATA_STREAM, templateId, prepareTemplate()); + + // when - index some documents + bulkIndex(NUM_REQUESTS, NUM_DOCS_PER_REQUEST); + + // then - verify that logsdb and synthetic source are both enabled + String firstBackingIndex = getWriteBackingIndex(client(), DATA_STREAM, 0); + var settings = (Map) getIndexSettingsWithDefaults(firstBackingIndex).get(firstBackingIndex); + assertThat(((Map) settings.get("settings")).get("index.mode"), equalTo("logsdb")); + assertThat(((Map) settings.get("defaults")).get("index.mapping.source.mode"), equalTo("SYNTHETIC")); + + // then continued - verify that the created data stream using the created template + LogsdbIndexingRollingUpgradeIT.assertDataStream(DATA_STREAM, templateId); + + // when/then - run some queries and verify results + ensureGreen(DATA_STREAM); + search(DATA_STREAM); + query(DATA_STREAM); + + } else if (isMixedCluster()) { + // when + bulkIndex(NUM_REQUESTS, NUM_DOCS_PER_REQUEST); + + // when/then + ensureGreen(DATA_STREAM); + search(DATA_STREAM); + query(DATA_STREAM); + + } else if (isUpgradedCluster()) { + // when/then + ensureGreen(DATA_STREAM); + bulkIndex(NUM_REQUESTS, NUM_DOCS_PER_REQUEST); + search(DATA_STREAM); + query(DATA_STREAM); + + // when/then continued - force merge all shard segments into one + var forceMergeRequest = new Request("POST", "/" + DATA_STREAM + "/_forcemerge"); + forceMergeRequest.addParameter("max_num_segments", "1"); + assertOK(client().performRequest(forceMergeRequest)); + + // then continued + ensureGreen(DATA_STREAM); + search(DATA_STREAM); + query(DATA_STREAM); + } + } + + private String prepareTemplate() { + boolean shouldSetIgnoreAbove = randomBoolean(); + if (shouldSetIgnoreAbove) { + return TEMPLATE.replace("$IGNORE_ABOVE", String.valueOf(randomInt(IGNORE_ABOVE_MAX))); + } + + // removes the entire line that defines ignore_above + return TEMPLATE.replaceAll("(?m)^\\s*\"ignore_above\":\\s*\\$IGNORE_ABOVE\\s*,?\\s*\\n?", ""); + } + + static void createTemplate(String dataStreamName, String id, String template) throws IOException { + final String INDEX_TEMPLATE = """ + { + "priority": 500, + "index_patterns": ["$DATASTREAM"], + "template": $TEMPLATE, + "data_stream": { + } + }"""; + var putIndexTemplateRequest = new Request("POST", "/_index_template/" + id); + putIndexTemplateRequest.setJsonEntity(INDEX_TEMPLATE.replace("$TEMPLATE", template).replace("$DATASTREAM", dataStreamName)); + assertOK(client().performRequest(putIndexTemplateRequest)); + } + + private void bulkIndex(int numRequest, int numDocs) throws Exception { + String firstIndex = null; + Instant startTime = Instant.now().minusSeconds(60 * 60); + + for (int i = 0; i < numRequest; i++) { + var bulkRequest = new Request("POST", "/" + DATA_STREAM + "/_bulk"); + bulkRequest.setJsonEntity(bulkIndexRequestBody(numDocs, startTime)); + bulkRequest.addParameter("refresh", "true"); + + var response = client().performRequest(bulkRequest); + var responseBody = entityAsMap(response); + + assertOK(response); + assertThat("errors in response:\n " + responseBody, responseBody.get("errors"), equalTo(false)); + if (firstIndex == null) { + firstIndex = (String) ((Map) ((Map) ((List) responseBody.get("items")).get(0)).get("create")).get("_index"); + } + } + } + + private String bulkIndexRequestBody(int numDocs, Instant startTime) { + StringBuilder requestBody = new StringBuilder(); + + for (int j = 0; j < numDocs; j++) { + String hostName = "host" + j % 50; // Not realistic, but makes asserting search / query response easier. + String methodName = "method" + j % 5; + String ip = NetworkAddress.format(randomIp(true)); + String message = randomAlphasDelimitedBySpace(10, 1, 15); + recordSmallestMessage(message); + long length = randomLong(); + double factor = randomDouble(); + + requestBody.append( + BULK_ITEM_TEMPLATE.replace("$now", formatInstant(startTime)) + .replace("$host", hostName) + .replace("$method", methodName) + .replace("$ip", ip) + .replace("$message", message) + .replace("$length", Long.toString(length)) + .replace("$factor", Double.toString(factor)) + ); + requestBody.append('\n'); + + startTime = startTime.plusMillis(1); + } + + return requestBody.toString(); + } + + private void recordSmallestMessage(final String message) { + if (smallestMessage == null || message.compareTo(smallestMessage) < 0) { + smallestMessage = message; + } + } + + void search(String dataStreamName) throws Exception { + var searchRequest = new Request("POST", "/" + dataStreamName + "/_search"); + searchRequest.addParameter("pretty", "true"); + searchRequest.setJsonEntity(""" + { + "size": 500 + } + """); + var response = client().performRequest(searchRequest); + assertOK(response); + var responseBody = entityAsMap(response); + logger.info("{}", responseBody); + + Integer totalCount = ObjectPath.evaluate(responseBody, "hits.total.value"); + assertThat(totalCount, greaterThanOrEqualTo(NUM_REQUESTS * NUM_DOCS_PER_REQUEST)); + } + + private void query(String dataStreamName) throws Exception { + var queryRequest = new Request("POST", "/_query"); + queryRequest.addParameter("pretty", "true"); + queryRequest.setJsonEntity(""" + { + "query": "FROM $ds | STATS max(length), max(factor) BY message | SORT message | LIMIT 5" + } + """.replace("$ds", dataStreamName)); + var response = client().performRequest(queryRequest); + assertOK(response); + var responseBody = entityAsMap(response); + logger.info("{}", responseBody); + + String column1 = ObjectPath.evaluate(responseBody, "columns.0.name"); + assertThat(column1, equalTo("max(length)")); + String column2 = ObjectPath.evaluate(responseBody, "columns.1.name"); + assertThat(column2, equalTo("max(factor)")); + String column3 = ObjectPath.evaluate(responseBody, "columns.2.name"); + assertThat(column3, equalTo("message")); + + Long maxRx = ObjectPath.evaluate(responseBody, "values.0.0"); + assertThat(maxRx, notNullValue()); + Double maxTx = ObjectPath.evaluate(responseBody, "values.0.1"); + assertThat(maxTx, notNullValue()); + String key = ObjectPath.evaluate(responseBody, "values.0.2"); + assertThat(key, equalTo(smallestMessage)); + } + + protected static void startTrial() throws IOException { + Request startTrial = new Request("POST", "/_license/start_trial"); + startTrial.addParameter("acknowledge", "true"); + try { + assertOK(client().performRequest(startTrial)); + } catch (ResponseException e) { + var responseBody = entityAsMap(e.getResponse()); + String error = ObjectPath.evaluate(responseBody, "error_message"); + assertThat(error, containsString("Trial was already activated.")); + } + } + + static Map getIndexSettingsWithDefaults(String index) throws IOException { + Request request = new Request("GET", "/" + index + "/_settings"); + request.addParameter("flat_settings", "true"); + request.addParameter("include_defaults", "true"); + Response response = client().performRequest(request); + try (InputStream is = response.getEntity().getContent()) { + return XContentHelper.convertToMap( + XContentType.fromMediaType(response.getEntity().getContentType().getValue()).xContent(), + is, + true + ); + } + } + + static String formatInstant(Instant instant) { + return DateFormatter.forPattern(FormatNames.STRICT_DATE_OPTIONAL_TIME.getName()).format(instant); + } + +} diff --git a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java index ff89074c3f063..c5811e46afd72 100644 --- a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java +++ b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java @@ -51,6 +51,7 @@ import org.elasticsearch.common.time.DateMathParser; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.core.Nullable; +import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.analysis.AnalyzerScope; @@ -91,8 +92,6 @@ import java.util.Set; import java.util.TreeSet; -import static org.elasticsearch.index.IndexSettings.IGNORE_ABOVE_SETTING; - /** * A {@link FieldMapper} for indexing fields with ngrams for efficient wildcard matching */ @@ -211,23 +210,18 @@ public static class Builder extends FieldMapper.Builder { final Parameter> meta = Parameter.metaParam(); + final IndexMode indexMode; final IndexVersion indexVersionCreated; - final int ignoreAboveDefault; - public Builder(final String name, IndexVersion indexVersionCreated) { - this(name, Integer.MAX_VALUE, indexVersionCreated); + this(name, IndexMode.STANDARD, indexVersionCreated); } - private Builder(String name, int ignoreAboveDefault, IndexVersion indexVersionCreated) { + private Builder(String name, IndexMode indexMode, IndexVersion indexVersionCreated) { super(name); this.indexVersionCreated = indexVersionCreated; - this.ignoreAboveDefault = ignoreAboveDefault; - this.ignoreAbove = Parameter.intParam("ignore_above", true, m -> toType(m).ignoreAbove, ignoreAboveDefault).addValidator(v -> { - if (v < 0) { - throw new IllegalArgumentException("[ignore_above] must be positive, got [" + v + "]"); - } - }); + this.indexMode = indexMode; + this.ignoreAbove = Parameter.ignoreAboveParam(m -> toType(m).ignoreAbove, indexMode, indexVersionCreated); } @Override @@ -259,7 +253,7 @@ public WildcardFieldMapper build(MapperBuilderContext context) { } public static final TypeParser PARSER = new TypeParser( - (n, c) -> new Builder(n, IGNORE_ABOVE_SETTING.get(c.getSettings()), c.indexVersionCreated()) + (n, c) -> new Builder(n, c.getIndexSettings().getMode(), c.indexVersionCreated()) ); public static final char TOKEN_START_OR_END_CHAR = 0; @@ -1000,10 +994,10 @@ protected String parseSourceValue(Object value) { assert NGRAM_FIELD_TYPE.indexOptions() == IndexOptions.DOCS; } private final String nullValue; + private final IndexMode indexMode; private final IndexVersion indexVersionCreated; private final int ignoreAbove; - private final int ignoreAboveDefault; private final boolean storeIgnored; private final String originalName; @@ -1018,9 +1012,9 @@ private WildcardFieldMapper( super(simpleName, mappedFieldType, builderParams); this.nullValue = builder.nullValue.getValue(); this.storeIgnored = storeIgnored; + this.indexMode = builder.indexMode; this.indexVersionCreated = indexVersionCreated; this.ignoreAbove = builder.ignoreAbove.getValue(); - this.ignoreAboveDefault = builder.ignoreAboveDefault; this.originalName = storeIgnored ? fullPath() + "._original" : null; } @@ -1096,7 +1090,7 @@ protected String contentType() { @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(leafName(), ignoreAboveDefault, indexVersionCreated).init(this); + return new Builder(leafName(), indexMode, indexVersionCreated).init(this); } @Override From 5bc89e82963278047bd2a83c70786b5502771051 Mon Sep 17 00:00:00 2001 From: Dmitry Kubikov Date: Fri, 5 Sep 2025 14:05:48 -0700 Subject: [PATCH 03/23] Added unit tests --- .../elasticsearch/index/IndexSettings.java | 4 +- .../index/mapper/KeywordFieldMapper.java | 38 +++-- .../index/mapper/KeywordFieldTypeTests.java | 148 +++++++++++++++++- .../index/mapper/MultiFieldsTests.java | 1 - .../index/mapper/ParameterTests.java | 62 ++++++++ .../index/mapper/TextFieldTypeTests.java | 90 +++++++++++ 6 files changed, 325 insertions(+), 18 deletions(-) create mode 100644 server/src/test/java/org/elasticsearch/index/mapper/ParameterTests.java diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index 8f1f6d4ef6b2f..3057449b79a52 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -803,8 +803,8 @@ public Iterator> settings() { * occupy at most 4 bytes. */ - private static final int IGNORE_ABOVE_DEFAULT = Integer.MAX_VALUE; - private static final int IGNORE_ABOVE_DEFAULT_LOGSDB = 8191; + public static final int IGNORE_ABOVE_DEFAULT = Integer.MAX_VALUE; + public static final int IGNORE_ABOVE_DEFAULT_LOGSDB = 8191; public static final Setting IGNORE_ABOVE_SETTING = Setting.intSetting( "index.mapping.ignore_above", diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index d6702ffe5c154..6a2bc2dc263b8 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -90,6 +90,7 @@ import static org.apache.lucene.index.IndexWriter.MAX_TERM_LENGTH; import static org.elasticsearch.core.Strings.format; import static org.elasticsearch.index.IndexSettings.USE_DOC_VALUES_SKIPPER; +import static org.elasticsearch.index.IndexSettings.getIgnoreAboveDefaultValue; import static org.elasticsearch.index.mapper.FieldArrayContext.getOffsetsFieldName; /** @@ -514,6 +515,8 @@ private static boolean indexSortConfigByHostName(final IndexSortConfig indexSort public static final class KeywordFieldType extends StringFieldType { + private static final IndexMode DEFAULT_INDEX_MODE = IndexMode.STANDARD; + private final int ignoreAbove; private final int ignoreAboveDefaultValue; private final String nullValue; @@ -558,26 +561,29 @@ public KeywordFieldType( this.originalName = isSyntheticSource ? name + "._original" : null; } + public KeywordFieldType(String name) { + this(name, true, true, Collections.emptyMap()); + } + public KeywordFieldType(String name, boolean isIndexed, boolean hasDocValues, Map meta) { super(name, isIndexed, false, hasDocValues, TextSearchInfo.SIMPLE_MATCH_ONLY, meta); + + final int ignoreAboveDefault = getIgnoreAboveDefaultValue(DEFAULT_INDEX_MODE, IndexVersion.current()); + this.normalizer = Lucene.KEYWORD_ANALYZER; - this.ignoreAbove = Integer.MAX_VALUE; - this.ignoreAboveDefaultValue = Integer.MAX_VALUE; + this.ignoreAbove = ignoreAboveDefault; + this.ignoreAboveDefaultValue = ignoreAboveDefault; this.nullValue = null; this.eagerGlobalOrdinals = false; this.scriptValues = null; this.isDimension = false; this.isSyntheticSource = false; - this.indexMode = IndexMode.STANDARD; + this.indexMode = DEFAULT_INDEX_MODE; this.indexSortConfig = null; this.hasDocValuesSkipper = false; this.originalName = null; } - public KeywordFieldType(String name) { - this(name, true, true, Collections.emptyMap()); - } - public KeywordFieldType(String name, FieldType fieldType) { super( name, @@ -587,15 +593,18 @@ public KeywordFieldType(String name, FieldType fieldType) { textSearchInfo(fieldType, null, Lucene.KEYWORD_ANALYZER, Lucene.KEYWORD_ANALYZER), Collections.emptyMap() ); + + final int ignoreAboveDefault = getIgnoreAboveDefaultValue(DEFAULT_INDEX_MODE, IndexVersion.current()); + this.normalizer = Lucene.KEYWORD_ANALYZER; - this.ignoreAbove = Integer.MAX_VALUE; - this.ignoreAboveDefaultValue = Integer.MAX_VALUE; + this.ignoreAbove = ignoreAboveDefault; + this.ignoreAboveDefaultValue = ignoreAboveDefault; this.nullValue = null; this.eagerGlobalOrdinals = false; this.scriptValues = null; this.isDimension = false; this.isSyntheticSource = false; - this.indexMode = IndexMode.STANDARD; + this.indexMode = DEFAULT_INDEX_MODE; this.indexSortConfig = null; this.hasDocValuesSkipper = DocValuesSkipIndexType.NONE.equals(fieldType.docValuesSkipIndexType()) == false; this.originalName = null; @@ -603,15 +612,18 @@ public KeywordFieldType(String name, FieldType fieldType) { public KeywordFieldType(String name, NamedAnalyzer analyzer) { super(name, true, false, true, textSearchInfo(Defaults.FIELD_TYPE, null, analyzer, analyzer), Collections.emptyMap()); + + final int ignoreAboveDefault = getIgnoreAboveDefaultValue(DEFAULT_INDEX_MODE, IndexVersion.current()); + this.normalizer = Lucene.KEYWORD_ANALYZER; - this.ignoreAbove = Integer.MAX_VALUE; - this.ignoreAboveDefaultValue = Integer.MAX_VALUE; + this.ignoreAbove = ignoreAboveDefault; + this.ignoreAboveDefaultValue = ignoreAboveDefault; this.nullValue = null; this.eagerGlobalOrdinals = false; this.scriptValues = null; this.isDimension = false; this.isSyntheticSource = false; - this.indexMode = IndexMode.STANDARD; + this.indexMode = DEFAULT_INDEX_MODE; this.indexSortConfig = null; this.hasDocValuesSkipper = false; this.originalName = null; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java index 05a6c24a2b743..6a1b105786b7a 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java @@ -9,7 +9,6 @@ package org.elasticsearch.index.mapper; import com.carrotsearch.randomizedtesting.generators.RandomStrings; - import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.LowerCaseFilter; import org.apache.lucene.analysis.TokenFilter; @@ -36,9 +35,13 @@ import org.apache.lucene.tests.index.RandomIndexWriter; import org.apache.lucene.util.BytesRef; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.common.lucene.Lucene; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.Fuzziness; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.CharFilterFactory; @@ -58,6 +61,11 @@ import java.util.List; import java.util.Map; +import static org.elasticsearch.index.IndexSettings.IGNORE_ABOVE_DEFAULT; +import static org.elasticsearch.index.IndexSettings.IGNORE_ABOVE_DEFAULT_LOGSDB; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; + public class KeywordFieldTypeTests extends FieldTypeTestCase { public void testIsFieldWithinQuery() throws IOException { @@ -245,7 +253,6 @@ public void testFetchSourceValue() throws IOException { "field", createIndexAnalyzers(), ScriptCompiler.NONE, - Integer.MAX_VALUE, IndexVersion.current(), randomFrom(Mapper.SourceKeepMode.values()) ).normalizer("lowercase").build(MapperBuilderContext.root(false, false)).fieldType(); @@ -291,6 +298,143 @@ public void testGetTerms() throws IOException { } } + public void test_isIgnoreAboveSet_returns_true_when_ignore_above_is_given() { + // given + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) + .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + .build(); + IndexSettings indexSettings = new IndexSettings(IndexMetadata.builder("index").settings(settings).build(), settings); + MappingParserContext mappingParserContext = mock(MappingParserContext.class); + doReturn(settings).when(mappingParserContext).getSettings(); + doReturn(indexSettings).when(mappingParserContext).getIndexSettings(); + doReturn(mock(ScriptCompiler.class)).when(mappingParserContext).scriptCompiler(); + + KeywordFieldMapper.Builder builder = new KeywordFieldMapper.Builder("child", mappingParserContext); + builder.ignoreAbove(123); + + KeywordFieldMapper.KeywordFieldType fieldType = new KeywordFieldMapper.KeywordFieldType( + "child", + mock(FieldType.class), + mock(NamedAnalyzer.class), + mock(NamedAnalyzer.class), + mock(NamedAnalyzer.class), + builder, + true + ); + + // when/then + assertTrue(fieldType.isIgnoreAboveSet()); + } + + public void test_isIgnoreAboveSet_returns_false_when_ignore_above_is_not_given() { + // given + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) + .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + .build(); + IndexSettings indexSettings = new IndexSettings(IndexMetadata.builder("index").settings(settings).build(), settings); + MappingParserContext mappingParserContext = mock(MappingParserContext.class); + doReturn(settings).when(mappingParserContext).getSettings(); + doReturn(indexSettings).when(mappingParserContext).getIndexSettings(); + doReturn(mock(ScriptCompiler.class)).when(mappingParserContext).scriptCompiler(); + + KeywordFieldMapper.Builder builder = new KeywordFieldMapper.Builder("child", mappingParserContext); + + KeywordFieldMapper.KeywordFieldType fieldType = new KeywordFieldMapper.KeywordFieldType( + "child", + mock(FieldType.class), + mock(NamedAnalyzer.class), + mock(NamedAnalyzer.class), + mock(NamedAnalyzer.class), + builder, + true + ); + + // when/then + assertFalse(fieldType.isIgnoreAboveSet()); + } + + public void test_isIgnoreAboveSet_returns_false_when_ignore_above_is_given_but_its_the_same_as_default() { + // given + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) + .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + .build(); + IndexSettings indexSettings = new IndexSettings(IndexMetadata.builder("index").settings(settings).build(), settings); + MappingParserContext mappingParserContext = mock(MappingParserContext.class); + doReturn(settings).when(mappingParserContext).getSettings(); + doReturn(indexSettings).when(mappingParserContext).getIndexSettings(); + doReturn(mock(ScriptCompiler.class)).when(mappingParserContext).scriptCompiler(); + + KeywordFieldMapper.Builder builder = new KeywordFieldMapper.Builder("child", mappingParserContext); + builder.ignoreAbove(IGNORE_ABOVE_DEFAULT); + + KeywordFieldMapper.KeywordFieldType fieldType = new KeywordFieldMapper.KeywordFieldType( + "child", + mock(FieldType.class), + mock(NamedAnalyzer.class), + mock(NamedAnalyzer.class), + mock(NamedAnalyzer.class), + builder, + true + ); + + // when/then + assertFalse(fieldType.isIgnoreAboveSet()); + } + + public void test_isIgnoreAboveSet_returns_false_when_ignore_above_is_given_but_its_the_same_as_default_for_logsdb_indices() { + // given + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) + .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + .build(); + IndexSettings indexSettings = new IndexSettings(IndexMetadata.builder("index").settings(settings).build(), settings); + MappingParserContext mappingParserContext = mock(MappingParserContext.class); + doReturn(settings).when(mappingParserContext).getSettings(); + doReturn(indexSettings).when(mappingParserContext).getIndexSettings(); + doReturn(mock(ScriptCompiler.class)).when(mappingParserContext).scriptCompiler(); + + KeywordFieldMapper.Builder builder = new KeywordFieldMapper.Builder("child", mappingParserContext); + builder.ignoreAbove(IGNORE_ABOVE_DEFAULT_LOGSDB); + + KeywordFieldMapper.KeywordFieldType fieldType = new KeywordFieldMapper.KeywordFieldType( + "child", + mock(FieldType.class), + mock(NamedAnalyzer.class), + mock(NamedAnalyzer.class), + mock(NamedAnalyzer.class), + builder, + true + ); + + // when/then + assertFalse(fieldType.isIgnoreAboveSet()); + } + + public void test_isIgnoreAboveSet_returns_false_for_non_primary_constructor() { + // given + KeywordFieldType fieldType1 = new KeywordFieldType("field"); + KeywordFieldType fieldType2 = new KeywordFieldType("field", mock(FieldType.class)); + KeywordFieldType fieldType3 = new KeywordFieldType("field", true, true, Collections.emptyMap()); + KeywordFieldType fieldType4 = new KeywordFieldType("field", mock(NamedAnalyzer.class)); + + // when/then + assertFalse(fieldType1.isIgnoreAboveSet()); + assertFalse(fieldType2.isIgnoreAboveSet()); + assertFalse(fieldType3.isIgnoreAboveSet()); + assertFalse(fieldType4.isIgnoreAboveSet()); + } + private static IndexAnalyzers createIndexAnalyzers() { return IndexAnalyzers.of( Map.of("default", new NamedAnalyzer("default", AnalyzerScope.INDEX, new StandardAnalyzer())), diff --git a/server/src/test/java/org/elasticsearch/index/mapper/MultiFieldsTests.java b/server/src/test/java/org/elasticsearch/index/mapper/MultiFieldsTests.java index 4c5bfeb66b075..e318fe527850a 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/MultiFieldsTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/MultiFieldsTests.java @@ -63,7 +63,6 @@ private KeywordFieldMapper.Builder getKeywordFieldMapperBuilder(boolean isStored "field", IndexAnalyzers.of(Map.of(), Map.of("normalizer", Lucene.STANDARD_ANALYZER), Map.of()), ScriptCompiler.NONE, - Integer.MAX_VALUE, IndexVersion.current(), Mapper.SourceKeepMode.NONE ); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ParameterTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ParameterTests.java new file mode 100644 index 0000000000000..8705f852d327c --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/ParameterTests.java @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.mapper; + +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.test.ESTestCase; + +import static org.elasticsearch.index.IndexSettings.IGNORE_ABOVE_DEFAULT; +import static org.elasticsearch.index.IndexSettings.IGNORE_ABOVE_DEFAULT_LOGSDB; + +public class ParameterTests extends ESTestCase { + + public void test_ignore_above_param_default() { + // when + FieldMapper.Parameter ignoreAbove = FieldMapper.Parameter.ignoreAboveParam((FieldMapper fm) -> 123); + + // then + assertEquals(IGNORE_ABOVE_DEFAULT, ignoreAbove.getValue().intValue()); + } + + public void test_ignore_above_param_default_for_standard_indices() { + // when + FieldMapper.Parameter ignoreAbove = FieldMapper.Parameter.ignoreAboveParam( + (FieldMapper fm) -> 123, + IndexMode.STANDARD, + IndexVersion.current() + ); + + // then + assertEquals(IGNORE_ABOVE_DEFAULT, ignoreAbove.getValue().intValue()); + } + + public void test_ignore_above_param_default_for_logsdb_indices() { + // when + FieldMapper.Parameter ignoreAbove = FieldMapper.Parameter.ignoreAboveParam( + (FieldMapper fm) -> 123, + IndexMode.LOGSDB, + IndexVersion.current() + ); + + // then + assertEquals(IGNORE_ABOVE_DEFAULT_LOGSDB, ignoreAbove.getValue().intValue()); + } + + public void test_ignore_above_param_invalid_value() { + // when + FieldMapper.Parameter ignoreAbove = FieldMapper.Parameter.ignoreAboveParam((FieldMapper fm) -> -1); + ignoreAbove.setValue(-1); + + // then + assertThrows(IllegalArgumentException.class, ignoreAbove::validate); + } + +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java index 4d246d3c557a6..0c7013e198af3 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java @@ -8,6 +8,7 @@ */ package org.elasticsearch.index.mapper; +import org.apache.lucene.document.FieldType; import org.apache.lucene.index.Term; import org.apache.lucene.queries.intervals.Intervals; import org.apache.lucene.queries.intervals.IntervalsSource; @@ -31,10 +32,18 @@ import org.apache.lucene.util.automaton.CharacterRunAutomaton; import org.apache.lucene.util.automaton.Operations; import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.lucene.BytesRefs; +import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.lucene.search.AutomatonQueries; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.Fuzziness; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.mapper.TextFieldMapper.TextFieldType; +import org.elasticsearch.script.ScriptCompiler; import java.io.IOException; import java.util.ArrayList; @@ -44,6 +53,8 @@ import static org.apache.lucene.search.MultiTermQuery.CONSTANT_SCORE_BLENDED_REWRITE; import static org.hamcrest.Matchers.equalTo; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; public class TextFieldTypeTests extends FieldTypeTestCase { @@ -289,4 +300,83 @@ public void testRangeIntervals() { rangeIntervals ); } + + public void test_block_loader_uses_synthetic_source_delegate_when_ignore_above_is_not_set() { + // given + KeywordFieldMapper.KeywordFieldType syntheticSourceDelegate = new KeywordFieldMapper.KeywordFieldType( + "child", + true, + true, + Collections.emptyMap() + ); + + TextFieldType ft = new TextFieldType( + "parent", + true, + false, + new TextSearchInfo(TextFieldMapper.Defaults.FIELD_TYPE, null, Lucene.STANDARD_ANALYZER, Lucene.STANDARD_ANALYZER), + true, + syntheticSourceDelegate, + Collections.singletonMap("potato", "tomato"), + false, + false + ); + + // when + ft.blockLoader(mock(MappedFieldType.BlockLoaderContext.class)); + BlockLoader blockLoader = ft.blockLoader(mock(MappedFieldType.BlockLoaderContext.class)); + + // then + // verify that we delegate block loading to the synthetic source delegate + assertTrue(blockLoader instanceof BlockLoader.Delegating); + assertThat(((BlockLoader.Delegating) blockLoader).delegatingTo(), equalTo("child")); + } + + public void test_block_loader_does_not_use_synthetic_source_delegate_when_ignore_above_is_set() { + // given + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) + .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + .build(); + IndexSettings indexSettings = new IndexSettings(IndexMetadata.builder("index").settings(settings).build(), settings); + MappingParserContext mappingParserContext = mock(MappingParserContext.class); + doReturn(settings).when(mappingParserContext).getSettings(); + doReturn(indexSettings).when(mappingParserContext).getIndexSettings(); + doReturn(mock(ScriptCompiler.class)).when(mappingParserContext).scriptCompiler(); + + KeywordFieldMapper.Builder builder = new KeywordFieldMapper.Builder("child", mappingParserContext); + builder.ignoreAbove(123); + + KeywordFieldMapper.KeywordFieldType syntheticSourceDelegate = new KeywordFieldMapper.KeywordFieldType( + "child", + mock(FieldType.class), + mock(NamedAnalyzer.class), + mock(NamedAnalyzer.class), + mock(NamedAnalyzer.class), + builder, + true + ); + + TextFieldType ft = new TextFieldType( + "parent", + true, + false, + new TextSearchInfo(TextFieldMapper.Defaults.FIELD_TYPE, null, Lucene.STANDARD_ANALYZER, Lucene.STANDARD_ANALYZER), + true, + syntheticSourceDelegate, + Collections.singletonMap("potato", "tomato"), + false, + false + ); + + // when + ft.blockLoader(mock(MappedFieldType.BlockLoaderContext.class)); + BlockLoader blockLoader = ft.blockLoader(mock(MappedFieldType.BlockLoaderContext.class)); + + // then + // verify that we don't delegate anything + assertFalse(blockLoader instanceof BlockLoader.Delegating); + } } From 78ba81ce470553d94506441c7a9353d9fabe7aaa Mon Sep 17 00:00:00 2001 From: Dmitry Kubikov Date: Fri, 5 Sep 2025 14:09:38 -0700 Subject: [PATCH 04/23] Undo changed to match only text bwc test --- .../MatchOnlyTextRollingUpgradeIT.java | 164 +++++++----------- 1 file changed, 66 insertions(+), 98 deletions(-) diff --git a/x-pack/plugin/logsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/MatchOnlyTextRollingUpgradeIT.java b/x-pack/plugin/logsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/MatchOnlyTextRollingUpgradeIT.java index 0f054fc816754..df4ef4db1b29d 100644 --- a/x-pack/plugin/logsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/MatchOnlyTextRollingUpgradeIT.java +++ b/x-pack/plugin/logsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/MatchOnlyTextRollingUpgradeIT.java @@ -10,6 +10,7 @@ package org.elasticsearch.upgrades; import com.carrotsearch.randomizedtesting.annotations.Name; + import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; @@ -36,19 +37,11 @@ public class MatchOnlyTextRollingUpgradeIT extends AbstractRollingUpgradeWithSecurityTestCase { - private static final String DATA_STREAM = "logs-bwc-test"; - - private static final int IGNORE_ABOVE_MAX = 256; - private static final int NUM_REQUESTS = 4; - private static final int NUM_DOCS_PER_REQUEST = 1024; - static String BULK_ITEM_TEMPLATE = """ - { "create": {} } {"@timestamp": "$now", "host.name": "$host", "method": "$method", "ip": "$ip", "message": "$message", "length": $length, "factor": $factor} """; - // "ignore_above": "$IGNORE_ABOVE" private static final String TEMPLATE = """ { "mappings": { @@ -60,13 +53,7 @@ public class MatchOnlyTextRollingUpgradeIT extends AbstractRollingUpgradeWithSec "type": "keyword" }, "message": { - "type": "match_only_text", - "fields": { - "keyword": { - "type": "keyword", - "store": true - } - } + "type": "match_only_text" }, "ip": { "type": "ip" @@ -86,67 +73,50 @@ public MatchOnlyTextRollingUpgradeIT(@Name("upgradedNodes") int upgradedNodes) { } public void testIndexing() throws Exception { - + String dataStreamName = "logs-bwc-test"; if (isOldCluster()) { - // given - enable logsdb and create a template startTrial(); enableLogsdbByDefault(); - String templateId = getClass().getSimpleName().toLowerCase(Locale.ROOT); - createTemplate(DATA_STREAM, templateId, prepareTemplate()); + createTemplate(dataStreamName, getClass().getSimpleName().toLowerCase(Locale.ROOT), TEMPLATE); - // when - index some documents - bulkIndex(NUM_REQUESTS, NUM_DOCS_PER_REQUEST); + Instant startTime = Instant.now().minusSeconds(60 * 60); + bulkIndex(dataStreamName, 4, 1024, startTime); - // then - verify that logsdb and synthetic source are both enabled - String firstBackingIndex = getWriteBackingIndex(client(), DATA_STREAM, 0); + String firstBackingIndex = getWriteBackingIndex(client(), dataStreamName, 0); var settings = (Map) getIndexSettingsWithDefaults(firstBackingIndex).get(firstBackingIndex); assertThat(((Map) settings.get("settings")).get("index.mode"), equalTo("logsdb")); assertThat(((Map) settings.get("defaults")).get("index.mapping.source.mode"), equalTo("SYNTHETIC")); - // then continued - verify that the created data stream using the created template - LogsdbIndexingRollingUpgradeIT.assertDataStream(DATA_STREAM, templateId); - - // when/then - run some queries and verify results - ensureGreen(DATA_STREAM); - search(DATA_STREAM); - query(DATA_STREAM); - + ensureGreen(dataStreamName); + search(dataStreamName); + query(dataStreamName); } else if (isMixedCluster()) { - // when - bulkIndex(NUM_REQUESTS, NUM_DOCS_PER_REQUEST); - - // when/then - ensureGreen(DATA_STREAM); - search(DATA_STREAM); - query(DATA_STREAM); + Instant startTime = Instant.now().minusSeconds(60 * 30); + bulkIndex(dataStreamName, 4, 1024, startTime); + ensureGreen(dataStreamName); + search(dataStreamName); + query(dataStreamName); } else if (isUpgradedCluster()) { - // when/then - ensureGreen(DATA_STREAM); - bulkIndex(NUM_REQUESTS, NUM_DOCS_PER_REQUEST); - search(DATA_STREAM); - query(DATA_STREAM); + ensureGreen(dataStreamName); + Instant startTime = Instant.now(); + bulkIndex(dataStreamName, 4, 1024, startTime); + search(dataStreamName); + query(dataStreamName); - // when/then continued - force merge all shard segments into one - var forceMergeRequest = new Request("POST", "/" + DATA_STREAM + "/_forcemerge"); + var forceMergeRequest = new Request("POST", "/" + dataStreamName + "/_forcemerge"); forceMergeRequest.addParameter("max_num_segments", "1"); assertOK(client().performRequest(forceMergeRequest)); - // then continued - ensureGreen(DATA_STREAM); - search(DATA_STREAM); - query(DATA_STREAM); + ensureGreen(dataStreamName); + search(dataStreamName); + query(dataStreamName); } } - private String prepareTemplate() { - return TEMPLATE.replace("$IGNORE_ABOVE", String.valueOf(randomInt(IGNORE_ABOVE_MAX))); - } - static void createTemplate(String dataStreamName, String id, String template) throws IOException { final String INDEX_TEMPLATE = """ { - "priority": 500, "index_patterns": ["$DATASTREAM"], "template": $TEMPLATE, "data_stream": { @@ -157,52 +127,46 @@ static void createTemplate(String dataStreamName, String id, String template) th assertOK(client().performRequest(putIndexTemplateRequest)); } - private void bulkIndex(int numRequest, int numDocs) throws Exception { + static String bulkIndex(String dataStreamName, int numRequest, int numDocs, Instant startTime) throws Exception { String firstIndex = null; - Instant startTime = Instant.now().minusSeconds(60 * 60); - for (int i = 0; i < numRequest; i++) { - var bulkRequest = new Request("POST", "/" + DATA_STREAM + "/_bulk"); - bulkRequest.setJsonEntity(bulkIndexRequestBody(numDocs, startTime)); + var bulkRequest = new Request("POST", "/" + dataStreamName + "/_bulk"); + StringBuilder requestBody = new StringBuilder(); + for (int j = 0; j < numDocs; j++) { + String hostName = "host" + j % 50; // Not realistic, but makes asserting search / query response easier. + String methodName = "method" + j % 5; + String ip = NetworkAddress.format(randomIp(true)); + String param = "chicken" + randomInt(5); + String message = "the quick brown fox jumps over the " + param; + long length = randomLong(); + double factor = randomDouble(); + + requestBody.append("{\"create\": {}}"); + requestBody.append('\n'); + requestBody.append( + BULK_ITEM_TEMPLATE.replace("$now", formatInstant(startTime)) + .replace("$host", hostName) + .replace("$method", methodName) + .replace("$ip", ip) + .replace("$message", message) + .replace("$length", Long.toString(length)) + .replace("$factor", Double.toString(factor)) + ); + requestBody.append('\n'); + + startTime = startTime.plusMillis(1); + } + bulkRequest.setJsonEntity(requestBody.toString()); bulkRequest.addParameter("refresh", "true"); - var response = client().performRequest(bulkRequest); - var responseBody = entityAsMap(response); - assertOK(response); + var responseBody = entityAsMap(response); assertThat("errors in response:\n " + responseBody, responseBody.get("errors"), equalTo(false)); if (firstIndex == null) { firstIndex = (String) ((Map) ((Map) ((List) responseBody.get("items")).get(0)).get("create")).get("_index"); } } - } - - private String bulkIndexRequestBody(int numDocs, Instant startTime) { - StringBuilder requestBody = new StringBuilder(); - - for (int j = 0; j < numDocs; j++) { - String hostName = "host" + j % 50; // Not realistic, but makes asserting search / query response easier. - String methodName = "method" + j % 5; - String ip = NetworkAddress.format(randomIp(true)); - String message = randomAlphasDelimitedBySpace(10, 1, 15); - long length = randomLong(); - double factor = randomDouble(); - - requestBody.append( - BULK_ITEM_TEMPLATE.replace("$now", formatInstant(startTime)) - .replace("$host", hostName) - .replace("$method", methodName) - .replace("$ip", ip) - .replace("$message", message) - .replace("$length", Long.toString(length)) - .replace("$factor", Double.toString(factor)) - ); - requestBody.append('\n'); - - startTime = startTime.plusMillis(1); - } - - return requestBody.toString(); + return firstIndex; } void search(String dataStreamName) throws Exception { @@ -210,19 +174,24 @@ void search(String dataStreamName) throws Exception { searchRequest.addParameter("pretty", "true"); searchRequest.setJsonEntity(""" { - "size": 500 + "size": 500, + "query": { + "match_phrase": { + "message": "chicken" + } + } } - """); + """.replace("chicken", "chicken" + randomInt(5))); var response = client().performRequest(searchRequest); assertOK(response); var responseBody = entityAsMap(response); logger.info("{}", responseBody); Integer totalCount = ObjectPath.evaluate(responseBody, "hits.total.value"); - assertThat(totalCount, greaterThanOrEqualTo(NUM_REQUESTS * NUM_DOCS_PER_REQUEST)); + assertThat(totalCount, greaterThanOrEqualTo(512)); } - private void query(String dataStreamName) throws Exception { + void query(String dataStreamName) throws Exception { var queryRequest = new Request("POST", "/_query"); queryRequest.addParameter("pretty", "true"); queryRequest.setJsonEntity(""" @@ -233,7 +202,6 @@ private void query(String dataStreamName) throws Exception { var response = client().performRequest(queryRequest); assertOK(response); var responseBody = entityAsMap(response); - logger.info("potato response body"); logger.info("{}", responseBody); String column1 = ObjectPath.evaluate(responseBody, "columns.0.name"); @@ -243,12 +211,12 @@ private void query(String dataStreamName) throws Exception { assertThat(column2, equalTo("max(factor)")); assertThat(column3, equalTo("message")); + String key = ObjectPath.evaluate(responseBody, "values.0.2"); + assertThat(key, equalTo("the quick brown fox jumps over the chicken0")); Long maxRx = ObjectPath.evaluate(responseBody, "values.0.0"); + assertThat(maxRx, notNullValue()); Double maxTx = ObjectPath.evaluate(responseBody, "values.0.1"); - String key = ObjectPath.evaluate(responseBody, "values.0.2"); assertThat(maxTx, notNullValue()); - assertThat(maxRx, notNullValue()); - assertThat(key, equalTo("banana boy")); } protected static void startTrial() throws IOException { From 5b458f803bc4c132cd89cd847594e3af52e3b7e8 Mon Sep 17 00:00:00 2001 From: Dmitry Kubikov Date: Fri, 5 Sep 2025 14:15:05 -0700 Subject: [PATCH 05/23] formatting --- .../icu/ICUCollationKeywordFieldMapper.java | 1 + .../flattened/FlattenedFieldMapper.java | 7 +- .../index/mapper/KeywordFieldTypeTests.java | 97 ++++++++++--------- .../index/mapper/ParameterTests.java | 6 +- .../org/elasticsearch/test/ESTestCase.java | 1 + .../upgrades/TextRollingUpgradeIT.java | 29 +++--- 6 files changed, 70 insertions(+), 71 deletions(-) diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapper.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapper.java index cd80164ff2c03..8b0183ede9ff0 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapper.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapper.java @@ -13,6 +13,7 @@ import com.ibm.icu.text.RawCollationKey; import com.ibm.icu.text.RuleBasedCollator; import com.ibm.icu.util.ULocale; + import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.SortedSetDocValuesField; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java index 8d2a586335cec..05097c449dbcf 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java @@ -827,12 +827,7 @@ public void validateMatchedRoutingPath(final String routingPath) { private final FlattenedFieldParser fieldParser; private final Builder builder; - private FlattenedFieldMapper( - String leafName, - MappedFieldType mappedFieldType, - BuilderParams builderParams, - Builder builder - ) { + private FlattenedFieldMapper(String leafName, MappedFieldType mappedFieldType, BuilderParams builderParams, Builder builder) { super(leafName, mappedFieldType, builderParams); this.builder = builder; this.ignoreAbove = builder.ignoreAbove.get(); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java index 6a1b105786b7a..889f40beca5c2 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java @@ -9,6 +9,7 @@ package org.elasticsearch.index.mapper; import com.carrotsearch.randomizedtesting.generators.RandomStrings; + import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.LowerCaseFilter; import org.apache.lucene.analysis.TokenFilter; @@ -301,11 +302,11 @@ public void testGetTerms() throws IOException { public void test_isIgnoreAboveSet_returns_true_when_ignore_above_is_given() { // given Settings settings = Settings.builder() - .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) - .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD) - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) - .build(); + .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) + .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + .build(); IndexSettings indexSettings = new IndexSettings(IndexMetadata.builder("index").settings(settings).build(), settings); MappingParserContext mappingParserContext = mock(MappingParserContext.class); doReturn(settings).when(mappingParserContext).getSettings(); @@ -316,13 +317,13 @@ public void test_isIgnoreAboveSet_returns_true_when_ignore_above_is_given() { builder.ignoreAbove(123); KeywordFieldMapper.KeywordFieldType fieldType = new KeywordFieldMapper.KeywordFieldType( - "child", - mock(FieldType.class), - mock(NamedAnalyzer.class), - mock(NamedAnalyzer.class), - mock(NamedAnalyzer.class), - builder, - true + "child", + mock(FieldType.class), + mock(NamedAnalyzer.class), + mock(NamedAnalyzer.class), + mock(NamedAnalyzer.class), + builder, + true ); // when/then @@ -332,11 +333,11 @@ public void test_isIgnoreAboveSet_returns_true_when_ignore_above_is_given() { public void test_isIgnoreAboveSet_returns_false_when_ignore_above_is_not_given() { // given Settings settings = Settings.builder() - .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) - .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD) - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) - .build(); + .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) + .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + .build(); IndexSettings indexSettings = new IndexSettings(IndexMetadata.builder("index").settings(settings).build(), settings); MappingParserContext mappingParserContext = mock(MappingParserContext.class); doReturn(settings).when(mappingParserContext).getSettings(); @@ -346,13 +347,13 @@ public void test_isIgnoreAboveSet_returns_false_when_ignore_above_is_not_given() KeywordFieldMapper.Builder builder = new KeywordFieldMapper.Builder("child", mappingParserContext); KeywordFieldMapper.KeywordFieldType fieldType = new KeywordFieldMapper.KeywordFieldType( - "child", - mock(FieldType.class), - mock(NamedAnalyzer.class), - mock(NamedAnalyzer.class), - mock(NamedAnalyzer.class), - builder, - true + "child", + mock(FieldType.class), + mock(NamedAnalyzer.class), + mock(NamedAnalyzer.class), + mock(NamedAnalyzer.class), + builder, + true ); // when/then @@ -362,11 +363,11 @@ public void test_isIgnoreAboveSet_returns_false_when_ignore_above_is_not_given() public void test_isIgnoreAboveSet_returns_false_when_ignore_above_is_given_but_its_the_same_as_default() { // given Settings settings = Settings.builder() - .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) - .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD) - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) - .build(); + .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) + .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + .build(); IndexSettings indexSettings = new IndexSettings(IndexMetadata.builder("index").settings(settings).build(), settings); MappingParserContext mappingParserContext = mock(MappingParserContext.class); doReturn(settings).when(mappingParserContext).getSettings(); @@ -377,13 +378,13 @@ public void test_isIgnoreAboveSet_returns_false_when_ignore_above_is_given_but_i builder.ignoreAbove(IGNORE_ABOVE_DEFAULT); KeywordFieldMapper.KeywordFieldType fieldType = new KeywordFieldMapper.KeywordFieldType( - "child", - mock(FieldType.class), - mock(NamedAnalyzer.class), - mock(NamedAnalyzer.class), - mock(NamedAnalyzer.class), - builder, - true + "child", + mock(FieldType.class), + mock(NamedAnalyzer.class), + mock(NamedAnalyzer.class), + mock(NamedAnalyzer.class), + builder, + true ); // when/then @@ -393,11 +394,11 @@ public void test_isIgnoreAboveSet_returns_false_when_ignore_above_is_given_but_i public void test_isIgnoreAboveSet_returns_false_when_ignore_above_is_given_but_its_the_same_as_default_for_logsdb_indices() { // given Settings settings = Settings.builder() - .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) - .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB) - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) - .build(); + .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) + .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + .build(); IndexSettings indexSettings = new IndexSettings(IndexMetadata.builder("index").settings(settings).build(), settings); MappingParserContext mappingParserContext = mock(MappingParserContext.class); doReturn(settings).when(mappingParserContext).getSettings(); @@ -408,13 +409,13 @@ public void test_isIgnoreAboveSet_returns_false_when_ignore_above_is_given_but_i builder.ignoreAbove(IGNORE_ABOVE_DEFAULT_LOGSDB); KeywordFieldMapper.KeywordFieldType fieldType = new KeywordFieldMapper.KeywordFieldType( - "child", - mock(FieldType.class), - mock(NamedAnalyzer.class), - mock(NamedAnalyzer.class), - mock(NamedAnalyzer.class), - builder, - true + "child", + mock(FieldType.class), + mock(NamedAnalyzer.class), + mock(NamedAnalyzer.class), + mock(NamedAnalyzer.class), + builder, + true ); // when/then diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ParameterTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ParameterTests.java index 8705f852d327c..d33e6eb12b60c 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ParameterTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ParameterTests.java @@ -29,9 +29,9 @@ public void test_ignore_above_param_default() { public void test_ignore_above_param_default_for_standard_indices() { // when FieldMapper.Parameter ignoreAbove = FieldMapper.Parameter.ignoreAboveParam( - (FieldMapper fm) -> 123, - IndexMode.STANDARD, - IndexVersion.current() + (FieldMapper fm) -> 123, + IndexMode.STANDARD, + IndexVersion.current() ); // then diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java index 0fd0cbf494013..5b35d736ed314 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java @@ -20,6 +20,7 @@ import com.carrotsearch.randomizedtesting.generators.RandomPicks; import com.carrotsearch.randomizedtesting.generators.RandomStrings; import com.carrotsearch.randomizedtesting.rules.TestRuleAdapter; + import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; diff --git a/x-pack/plugin/logsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/TextRollingUpgradeIT.java b/x-pack/plugin/logsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/TextRollingUpgradeIT.java index 73ad0fd128800..9a7fa40c38c5e 100644 --- a/x-pack/plugin/logsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/TextRollingUpgradeIT.java +++ b/x-pack/plugin/logsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/TextRollingUpgradeIT.java @@ -8,6 +8,7 @@ package org.elasticsearch.upgrades; import com.carrotsearch.randomizedtesting.annotations.Name; + import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.client.ResponseException; @@ -41,10 +42,10 @@ public class TextRollingUpgradeIT extends AbstractRollingUpgradeWithSecurityTest private static final int NUM_DOCS_PER_REQUEST = 1024; static String BULK_ITEM_TEMPLATE = - """ - { "create": {} } - {"@timestamp": "$now", "host.name": "$host", "method": "$method", "ip": "$ip", "message": "$message", "length": $length, "factor": $factor} - """; + """ + { "create": {} } + {"@timestamp": "$now", "host.name": "$host", "method": "$method", "ip": "$ip", "message": "$message", "length": $length, "factor": $factor} + """; private static final String TEMPLATE = """ { @@ -196,13 +197,13 @@ private String bulkIndexRequestBody(int numDocs, Instant startTime) { double factor = randomDouble(); requestBody.append( - BULK_ITEM_TEMPLATE.replace("$now", formatInstant(startTime)) - .replace("$host", hostName) - .replace("$method", methodName) - .replace("$ip", ip) - .replace("$message", message) - .replace("$length", Long.toString(length)) - .replace("$factor", Double.toString(factor)) + BULK_ITEM_TEMPLATE.replace("$now", formatInstant(startTime)) + .replace("$host", hostName) + .replace("$method", methodName) + .replace("$ip", ip) + .replace("$message", message) + .replace("$length", Long.toString(length)) + .replace("$factor", Double.toString(factor)) ); requestBody.append('\n'); @@ -282,9 +283,9 @@ static Map getIndexSettingsWithDefaults(String index) throws IOE Response response = client().performRequest(request); try (InputStream is = response.getEntity().getContent()) { return XContentHelper.convertToMap( - XContentType.fromMediaType(response.getEntity().getContentType().getValue()).xContent(), - is, - true + XContentType.fromMediaType(response.getEntity().getContentType().getValue()).xContent(), + is, + true ); } } From 83bab473e8ee4a1488572011ed73cd9569af5a46 Mon Sep 17 00:00:00 2001 From: Dmitry Kubikov Date: Fri, 5 Sep 2025 16:21:45 -0700 Subject: [PATCH 06/23] Removed indexMode from field type --- .../index/mapper/KeywordFieldMapper.java | 29 ++----------------- 1 file changed, 3 insertions(+), 26 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 6a2bc2dc263b8..53c5af420e97f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -515,8 +515,6 @@ private static boolean indexSortConfigByHostName(final IndexSortConfig indexSort public static final class KeywordFieldType extends StringFieldType { - private static final IndexMode DEFAULT_INDEX_MODE = IndexMode.STANDARD; - private final int ignoreAbove; private final int ignoreAboveDefaultValue; private final String nullValue; @@ -525,7 +523,6 @@ public static final class KeywordFieldType extends StringFieldType { private final FieldValues scriptValues; private final boolean isDimension; private final boolean isSyntheticSource; - private final IndexMode indexMode; private final IndexSortConfig indexSortConfig; private final boolean hasDocValuesSkipper; private final String originalName; @@ -555,7 +552,6 @@ public KeywordFieldType( this.scriptValues = builder.scriptValues(); this.isDimension = builder.dimension.getValue(); this.isSyntheticSource = isSyntheticSource; - this.indexMode = builder.indexMode; this.indexSortConfig = builder.indexSortConfig; this.hasDocValuesSkipper = DocValuesSkipIndexType.NONE.equals(fieldType.docValuesSkipIndexType()) == false; this.originalName = isSyntheticSource ? name + "._original" : null; @@ -567,18 +563,13 @@ public KeywordFieldType(String name) { public KeywordFieldType(String name, boolean isIndexed, boolean hasDocValues, Map meta) { super(name, isIndexed, false, hasDocValues, TextSearchInfo.SIMPLE_MATCH_ONLY, meta); - - final int ignoreAboveDefault = getIgnoreAboveDefaultValue(DEFAULT_INDEX_MODE, IndexVersion.current()); - this.normalizer = Lucene.KEYWORD_ANALYZER; - this.ignoreAbove = ignoreAboveDefault; - this.ignoreAboveDefaultValue = ignoreAboveDefault; + this.ignoreAbove = this.ignoreAboveDefaultValue = getIgnoreAboveDefaultValue(IndexMode.STANDARD, IndexVersion.current()); this.nullValue = null; this.eagerGlobalOrdinals = false; this.scriptValues = null; this.isDimension = false; this.isSyntheticSource = false; - this.indexMode = DEFAULT_INDEX_MODE; this.indexSortConfig = null; this.hasDocValuesSkipper = false; this.originalName = null; @@ -593,18 +584,13 @@ public KeywordFieldType(String name, FieldType fieldType) { textSearchInfo(fieldType, null, Lucene.KEYWORD_ANALYZER, Lucene.KEYWORD_ANALYZER), Collections.emptyMap() ); - - final int ignoreAboveDefault = getIgnoreAboveDefaultValue(DEFAULT_INDEX_MODE, IndexVersion.current()); - this.normalizer = Lucene.KEYWORD_ANALYZER; - this.ignoreAbove = ignoreAboveDefault; - this.ignoreAboveDefaultValue = ignoreAboveDefault; + this.ignoreAbove = this.ignoreAboveDefaultValue = getIgnoreAboveDefaultValue(IndexMode.STANDARD, IndexVersion.current()); this.nullValue = null; this.eagerGlobalOrdinals = false; this.scriptValues = null; this.isDimension = false; this.isSyntheticSource = false; - this.indexMode = DEFAULT_INDEX_MODE; this.indexSortConfig = null; this.hasDocValuesSkipper = DocValuesSkipIndexType.NONE.equals(fieldType.docValuesSkipIndexType()) == false; this.originalName = null; @@ -612,18 +598,13 @@ public KeywordFieldType(String name, FieldType fieldType) { public KeywordFieldType(String name, NamedAnalyzer analyzer) { super(name, true, false, true, textSearchInfo(Defaults.FIELD_TYPE, null, analyzer, analyzer), Collections.emptyMap()); - - final int ignoreAboveDefault = getIgnoreAboveDefaultValue(DEFAULT_INDEX_MODE, IndexVersion.current()); - this.normalizer = Lucene.KEYWORD_ANALYZER; - this.ignoreAbove = ignoreAboveDefault; - this.ignoreAboveDefaultValue = ignoreAboveDefault; + this.ignoreAbove = this.ignoreAboveDefaultValue = getIgnoreAboveDefaultValue(IndexMode.STANDARD, IndexVersion.current()); this.nullValue = null; this.eagerGlobalOrdinals = false; this.scriptValues = null; this.isDimension = false; this.isSyntheticSource = false; - this.indexMode = DEFAULT_INDEX_MODE; this.indexSortConfig = null; this.hasDocValuesSkipper = false; this.originalName = null; @@ -1076,10 +1057,6 @@ public boolean hasNormalizer() { return normalizer != Lucene.KEYWORD_ANALYZER; } - public IndexMode getIndexMode() { - return indexMode; - } - public IndexSortConfig getIndexSortConfig() { return indexSortConfig; } From bee6473f74f952c277418a0d910c395282034c0d Mon Sep 17 00:00:00 2001 From: Dmitry Kubikov Date: Fri, 5 Sep 2025 16:30:57 -0700 Subject: [PATCH 07/23] Added another test case --- .../index/mapper/KeywordFieldTypeTests.java | 48 +++++++++++++++---- 1 file changed, 39 insertions(+), 9 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java index 889f40beca5c2..e9e91103f53fc 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java @@ -9,7 +9,6 @@ package org.elasticsearch.index.mapper; import com.carrotsearch.randomizedtesting.generators.RandomStrings; - import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.LowerCaseFilter; import org.apache.lucene.analysis.TokenFilter; @@ -313,11 +312,11 @@ public void test_isIgnoreAboveSet_returns_true_when_ignore_above_is_given() { doReturn(indexSettings).when(mappingParserContext).getIndexSettings(); doReturn(mock(ScriptCompiler.class)).when(mappingParserContext).scriptCompiler(); - KeywordFieldMapper.Builder builder = new KeywordFieldMapper.Builder("child", mappingParserContext); + KeywordFieldMapper.Builder builder = new KeywordFieldMapper.Builder("field", mappingParserContext); builder.ignoreAbove(123); KeywordFieldMapper.KeywordFieldType fieldType = new KeywordFieldMapper.KeywordFieldType( - "child", + "field", mock(FieldType.class), mock(NamedAnalyzer.class), mock(NamedAnalyzer.class), @@ -344,10 +343,10 @@ public void test_isIgnoreAboveSet_returns_false_when_ignore_above_is_not_given() doReturn(indexSettings).when(mappingParserContext).getIndexSettings(); doReturn(mock(ScriptCompiler.class)).when(mappingParserContext).scriptCompiler(); - KeywordFieldMapper.Builder builder = new KeywordFieldMapper.Builder("child", mappingParserContext); + KeywordFieldMapper.Builder builder = new KeywordFieldMapper.Builder("field", mappingParserContext); KeywordFieldMapper.KeywordFieldType fieldType = new KeywordFieldMapper.KeywordFieldType( - "child", + "field", mock(FieldType.class), mock(NamedAnalyzer.class), mock(NamedAnalyzer.class), @@ -374,11 +373,11 @@ public void test_isIgnoreAboveSet_returns_false_when_ignore_above_is_given_but_i doReturn(indexSettings).when(mappingParserContext).getIndexSettings(); doReturn(mock(ScriptCompiler.class)).when(mappingParserContext).scriptCompiler(); - KeywordFieldMapper.Builder builder = new KeywordFieldMapper.Builder("child", mappingParserContext); + KeywordFieldMapper.Builder builder = new KeywordFieldMapper.Builder("field", mappingParserContext); builder.ignoreAbove(IGNORE_ABOVE_DEFAULT); KeywordFieldMapper.KeywordFieldType fieldType = new KeywordFieldMapper.KeywordFieldType( - "child", + "field", mock(FieldType.class), mock(NamedAnalyzer.class), mock(NamedAnalyzer.class), @@ -405,11 +404,11 @@ public void test_isIgnoreAboveSet_returns_false_when_ignore_above_is_given_but_i doReturn(indexSettings).when(mappingParserContext).getIndexSettings(); doReturn(mock(ScriptCompiler.class)).when(mappingParserContext).scriptCompiler(); - KeywordFieldMapper.Builder builder = new KeywordFieldMapper.Builder("child", mappingParserContext); + KeywordFieldMapper.Builder builder = new KeywordFieldMapper.Builder("field", mappingParserContext); builder.ignoreAbove(IGNORE_ABOVE_DEFAULT_LOGSDB); KeywordFieldMapper.KeywordFieldType fieldType = new KeywordFieldMapper.KeywordFieldType( - "child", + "field", mock(FieldType.class), mock(NamedAnalyzer.class), mock(NamedAnalyzer.class), @@ -422,6 +421,37 @@ public void test_isIgnoreAboveSet_returns_false_when_ignore_above_is_given_but_i assertFalse(fieldType.isIgnoreAboveSet()); } + public void test_isIgnoreAboveSet_returns_true_when_ignore_above_is_given_as_logsdb_default_but_index_mod_is_not_logsdb() { + // given + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) + .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + .build(); + IndexSettings indexSettings = new IndexSettings(IndexMetadata.builder("index").settings(settings).build(), settings); + MappingParserContext mappingParserContext = mock(MappingParserContext.class); + doReturn(settings).when(mappingParserContext).getSettings(); + doReturn(indexSettings).when(mappingParserContext).getIndexSettings(); + doReturn(mock(ScriptCompiler.class)).when(mappingParserContext).scriptCompiler(); + + KeywordFieldMapper.Builder builder = new KeywordFieldMapper.Builder("field", mappingParserContext); + builder.ignoreAbove(IGNORE_ABOVE_DEFAULT_LOGSDB); + + KeywordFieldMapper.KeywordFieldType fieldType = new KeywordFieldMapper.KeywordFieldType( + "field", + mock(FieldType.class), + mock(NamedAnalyzer.class), + mock(NamedAnalyzer.class), + mock(NamedAnalyzer.class), + builder, + true + ); + + // when/then + assertTrue(fieldType.isIgnoreAboveSet()); + } + public void test_isIgnoreAboveSet_returns_false_for_non_primary_constructor() { // given KeywordFieldType fieldType1 = new KeywordFieldType("field"); From 696bccf6726d609ef03574f5270c3401a84f5e47 Mon Sep 17 00:00:00 2001 From: Dmitry Kubikov Date: Fri, 5 Sep 2025 16:35:10 -0700 Subject: [PATCH 08/23] Fixed failing bwc tests --- .../elasticsearch/index/IndexSettings.java | 27 +++++----- .../index/mapper/FieldMapper.java | 9 +--- .../index/mapper/KeywordFieldMapper.java | 33 ++++++++++--- .../flattened/FlattenedFieldMapper.java | 6 ++- .../index/mapper/KeywordFieldTypeTests.java | 49 ++++++++++++++++--- .../index/mapper/ParameterTests.java | 31 +----------- .../upgrades/TextRollingUpgradeIT.java | 2 +- .../wildcard/mapper/WildcardFieldMapper.java | 6 ++- 8 files changed, 101 insertions(+), 62 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index 3057449b79a52..2bb44b26a1716 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -803,33 +803,38 @@ public Iterator> settings() { * occupy at most 4 bytes. */ - public static final int IGNORE_ABOVE_DEFAULT = Integer.MAX_VALUE; - public static final int IGNORE_ABOVE_DEFAULT_LOGSDB = 8191; + public static final int IGNORE_ABOVE_DEFAULT_STANDARD_INDICES = Integer.MAX_VALUE; + public static final int IGNORE_ABOVE_DEFAULT_LOGSDB_INDICES = 8191; public static final Setting IGNORE_ABOVE_SETTING = Setting.intSetting( "index.mapping.ignore_above", - IndexSettings::getIgnoreAboveDefaultValue, + settings -> String.valueOf(getIgnoreAboveDefaultValue(settings)), 0, Integer.MAX_VALUE, Property.IndexScope, Property.ServerlessPublic ); - private static String getIgnoreAboveDefaultValue(final Settings settings) { - return String.valueOf( - getIgnoreAboveDefaultValue(IndexSettings.MODE.get(settings), IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings)) - ); + private static int getIgnoreAboveDefaultValue(final Settings settings) { + if (settings == null) { + return IGNORE_ABOVE_DEFAULT_STANDARD_INDICES; + } + return getIgnoreAboveDefaultValue(IndexSettings.MODE.get(settings), IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings)); } public static int getIgnoreAboveDefaultValue(final IndexMode indexMode, final IndexVersion indexCreatedVersion) { - if (indexMode == IndexMode.LOGSDB - && (indexCreatedVersion != null && indexCreatedVersion.onOrAfter(IndexVersions.ENABLE_IGNORE_ABOVE_LOGSDB))) { - return IGNORE_ABOVE_DEFAULT_LOGSDB; + if (diffIgnoreAboveDefaultForLogs(indexMode, indexCreatedVersion)) { + return IGNORE_ABOVE_DEFAULT_LOGSDB_INDICES; } else { - return IGNORE_ABOVE_DEFAULT; + return IGNORE_ABOVE_DEFAULT_STANDARD_INDICES; } } + private static boolean diffIgnoreAboveDefaultForLogs(final IndexMode indexMode, final IndexVersion indexCreatedVersion) { + return indexMode == IndexMode.LOGSDB + && (indexCreatedVersion != null && indexCreatedVersion.onOrAfter(IndexVersions.ENABLE_IGNORE_ABOVE_LOGSDB)); + } + public static final Setting SEQ_NO_INDEX_OPTIONS_SETTING = Setting.enumSetting( SeqNoFieldMapper.SeqNoIndexOptions.class, settings -> { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index 048f5a2254cbe..a5f2d22360d6a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -1338,15 +1338,10 @@ public static Parameter normsParam(Function initi } public static Parameter ignoreAboveParam(Function initializer) { - return ignoreAboveParam(initializer, null, null); + return ignoreAboveParam(initializer, IndexSettings.IGNORE_ABOVE_DEFAULT_STANDARD_INDICES); } - public static Parameter ignoreAboveParam( - Function initializer, - final IndexMode indexMode, - final IndexVersion indexCreatedVersion - ) { - final int defaultValue = IndexSettings.getIgnoreAboveDefaultValue(indexMode, indexCreatedVersion); + public static Parameter ignoreAboveParam(Function initializer, final int defaultValue) { return Parameter.intParam("ignore_above", true, initializer, defaultValue).addValidator(v -> { if (v < 0) { throw new IllegalArgumentException("[ignore_above] must be positive, got [" + v + "]"); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 53c5af420e97f..9aee8a9f5ce14 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -42,6 +42,7 @@ import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.core.Nullable; import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexSortConfig; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; @@ -89,8 +90,9 @@ import static org.apache.lucene.index.IndexWriter.MAX_TERM_LENGTH; import static org.elasticsearch.core.Strings.format; +import static org.elasticsearch.index.IndexSettings.IGNORE_ABOVE_DEFAULT_STANDARD_INDICES; +import static org.elasticsearch.index.IndexSettings.IGNORE_ABOVE_SETTING; import static org.elasticsearch.index.IndexSettings.USE_DOC_VALUES_SKIPPER; -import static org.elasticsearch.index.IndexSettings.getIgnoreAboveDefaultValue; import static org.elasticsearch.index.mapper.FieldArrayContext.getOffsetsFieldName; /** @@ -179,6 +181,7 @@ public static final class Builder extends FieldMapper.DimensionBuilder { false ); private final Parameter ignoreAbove; + private final int ignoreAboveDefault; private final IndexSortConfig indexSortConfig; private final IndexMode indexMode; private final Parameter indexOptions = TextParams.keywordIndexOptions(m -> toType(m).indexOptions); @@ -217,6 +220,7 @@ public Builder(final String name, final MappingParserContext mappingParserContex name, mappingParserContext.getIndexAnalyzers(), mappingParserContext.scriptCompiler(), + IGNORE_ABOVE_SETTING.get(mappingParserContext.getSettings()), mappingParserContext.getIndexSettings().getIndexVersionCreated(), mappingParserContext.getIndexSettings().getMode(), mappingParserContext.getIndexSettings().getIndexSortConfig(), @@ -233,13 +237,25 @@ public Builder(final String name, final MappingParserContext mappingParserContex IndexVersion indexCreatedVersion, SourceKeepMode sourceKeepMode ) { - this(name, indexAnalyzers, scriptCompiler, indexCreatedVersion, IndexMode.STANDARD, null, false, false, sourceKeepMode); + this( + name, + indexAnalyzers, + scriptCompiler, + IndexSettings.getIgnoreAboveDefaultValue(IndexMode.STANDARD, indexCreatedVersion), + indexCreatedVersion, + IndexMode.STANDARD, + null, + false, + false, + sourceKeepMode + ); } private Builder( String name, IndexAnalyzers indexAnalyzers, ScriptCompiler scriptCompiler, + int ignoreAboveDefault, IndexVersion indexCreatedVersion, IndexMode indexMode, IndexSortConfig indexSortConfig, @@ -273,7 +289,8 @@ private Builder( ); } }).precludesParameters(normalizer); - this.ignoreAbove = Parameter.ignoreAboveParam(m -> toType(m).fieldType().ignoreAbove(), indexMode, indexCreatedVersion); + this.ignoreAboveDefault = ignoreAboveDefault; + this.ignoreAbove = Parameter.ignoreAboveParam(m -> toType(m).fieldType().ignoreAbove(), ignoreAboveDefault); this.indexSortConfig = indexSortConfig; this.indexMode = indexMode; this.enableDocValuesSkipper = enableDocValuesSkipper; @@ -295,6 +312,7 @@ public static Builder buildWithDocValuesSkipper( name, null, ScriptCompiler.NONE, + IndexSettings.getIgnoreAboveDefaultValue(indexMode, indexCreatedVersion), indexCreatedVersion, indexMode, // Sort config is used to decide if DocValueSkippers can be used. Since skippers are forced, a sort config is not needed. @@ -564,7 +582,7 @@ public KeywordFieldType(String name) { public KeywordFieldType(String name, boolean isIndexed, boolean hasDocValues, Map meta) { super(name, isIndexed, false, hasDocValues, TextSearchInfo.SIMPLE_MATCH_ONLY, meta); this.normalizer = Lucene.KEYWORD_ANALYZER; - this.ignoreAbove = this.ignoreAboveDefaultValue = getIgnoreAboveDefaultValue(IndexMode.STANDARD, IndexVersion.current()); + this.ignoreAbove = this.ignoreAboveDefaultValue = IGNORE_ABOVE_DEFAULT_STANDARD_INDICES; this.nullValue = null; this.eagerGlobalOrdinals = false; this.scriptValues = null; @@ -585,7 +603,7 @@ public KeywordFieldType(String name, FieldType fieldType) { Collections.emptyMap() ); this.normalizer = Lucene.KEYWORD_ANALYZER; - this.ignoreAbove = this.ignoreAboveDefaultValue = getIgnoreAboveDefaultValue(IndexMode.STANDARD, IndexVersion.current()); + this.ignoreAbove = this.ignoreAboveDefaultValue = IGNORE_ABOVE_DEFAULT_STANDARD_INDICES; this.nullValue = null; this.eagerGlobalOrdinals = false; this.scriptValues = null; @@ -599,7 +617,7 @@ public KeywordFieldType(String name, FieldType fieldType) { public KeywordFieldType(String name, NamedAnalyzer analyzer) { super(name, true, false, true, textSearchInfo(Defaults.FIELD_TYPE, null, analyzer, analyzer), Collections.emptyMap()); this.normalizer = Lucene.KEYWORD_ANALYZER; - this.ignoreAbove = this.ignoreAboveDefaultValue = getIgnoreAboveDefaultValue(IndexMode.STANDARD, IndexVersion.current()); + this.ignoreAbove = this.ignoreAboveDefaultValue = IGNORE_ABOVE_DEFAULT_STANDARD_INDICES; this.nullValue = null; this.eagerGlobalOrdinals = false; this.scriptValues = null; @@ -1098,6 +1116,7 @@ public String originalName() { private final boolean isSyntheticSource; private final IndexAnalyzers indexAnalyzers; + private final int ignoreAboveDefault; private final IndexMode indexMode; private final IndexSortConfig indexSortConfig; private final boolean enableDocValuesSkipper; @@ -1129,6 +1148,7 @@ private KeywordFieldMapper( this.scriptCompiler = builder.scriptCompiler; this.indexCreatedVersion = builder.indexCreatedVersion; this.isSyntheticSource = isSyntheticSource; + this.ignoreAboveDefault = builder.ignoreAboveDefault; this.indexMode = builder.indexMode; this.indexSortConfig = builder.indexSortConfig; this.enableDocValuesSkipper = builder.enableDocValuesSkipper; @@ -1284,6 +1304,7 @@ public FieldMapper.Builder getMergeBuilder() { leafName(), indexAnalyzers, scriptCompiler, + ignoreAboveDefault, indexCreatedVersion, indexMode, indexSortConfig, diff --git a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java index 05097c449dbcf..ff66a414f4238 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java @@ -39,6 +39,7 @@ import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.core.Nullable; import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.fielddata.FieldData; @@ -193,7 +194,10 @@ private Builder(String name, IndexMode indexMode, IndexVersion indexCreatedVersi super(name); this.indexMode = indexMode; this.indexCreatedVersion = indexCreatedVersion; - this.ignoreAbove = Parameter.ignoreAboveParam(m -> builder(m).ignoreAbove.get(), indexMode, indexCreatedVersion); + this.ignoreAbove = Parameter.ignoreAboveParam( + m -> builder(m).ignoreAbove.get(), + IndexSettings.getIgnoreAboveDefaultValue(indexMode, indexCreatedVersion) + ); this.dimensions.precludesParameters(ignoreAbove); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java index e9e91103f53fc..799d4d5eaa8d3 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java @@ -61,8 +61,8 @@ import java.util.List; import java.util.Map; -import static org.elasticsearch.index.IndexSettings.IGNORE_ABOVE_DEFAULT; -import static org.elasticsearch.index.IndexSettings.IGNORE_ABOVE_DEFAULT_LOGSDB; +import static org.elasticsearch.index.IndexSettings.IGNORE_ABOVE_DEFAULT_LOGSDB_INDICES; +import static org.elasticsearch.index.IndexSettings.IGNORE_ABOVE_DEFAULT_STANDARD_INDICES; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; @@ -327,6 +327,7 @@ public void test_isIgnoreAboveSet_returns_true_when_ignore_above_is_given() { // when/then assertTrue(fieldType.isIgnoreAboveSet()); + assertEquals(123, fieldType.ignoreAbove()); } public void test_isIgnoreAboveSet_returns_false_when_ignore_above_is_not_given() { @@ -357,6 +358,7 @@ public void test_isIgnoreAboveSet_returns_false_when_ignore_above_is_not_given() // when/then assertFalse(fieldType.isIgnoreAboveSet()); + assertEquals(IGNORE_ABOVE_DEFAULT_STANDARD_INDICES, fieldType.ignoreAbove()); } public void test_isIgnoreAboveSet_returns_false_when_ignore_above_is_given_but_its_the_same_as_default() { @@ -374,7 +376,7 @@ public void test_isIgnoreAboveSet_returns_false_when_ignore_above_is_given_but_i doReturn(mock(ScriptCompiler.class)).when(mappingParserContext).scriptCompiler(); KeywordFieldMapper.Builder builder = new KeywordFieldMapper.Builder("field", mappingParserContext); - builder.ignoreAbove(IGNORE_ABOVE_DEFAULT); + builder.ignoreAbove(IGNORE_ABOVE_DEFAULT_STANDARD_INDICES); KeywordFieldMapper.KeywordFieldType fieldType = new KeywordFieldMapper.KeywordFieldType( "field", @@ -388,6 +390,7 @@ public void test_isIgnoreAboveSet_returns_false_when_ignore_above_is_given_but_i // when/then assertFalse(fieldType.isIgnoreAboveSet()); + assertEquals(IGNORE_ABOVE_DEFAULT_STANDARD_INDICES, fieldType.ignoreAbove()); } public void test_isIgnoreAboveSet_returns_false_when_ignore_above_is_given_but_its_the_same_as_default_for_logsdb_indices() { @@ -405,7 +408,7 @@ public void test_isIgnoreAboveSet_returns_false_when_ignore_above_is_given_but_i doReturn(mock(ScriptCompiler.class)).when(mappingParserContext).scriptCompiler(); KeywordFieldMapper.Builder builder = new KeywordFieldMapper.Builder("field", mappingParserContext); - builder.ignoreAbove(IGNORE_ABOVE_DEFAULT_LOGSDB); + builder.ignoreAbove(IGNORE_ABOVE_DEFAULT_LOGSDB_INDICES); KeywordFieldMapper.KeywordFieldType fieldType = new KeywordFieldMapper.KeywordFieldType( "field", @@ -419,15 +422,49 @@ public void test_isIgnoreAboveSet_returns_false_when_ignore_above_is_given_but_i // when/then assertFalse(fieldType.isIgnoreAboveSet()); + assertEquals(IGNORE_ABOVE_DEFAULT_LOGSDB_INDICES, fieldType.ignoreAbove()); } public void test_isIgnoreAboveSet_returns_true_when_ignore_above_is_given_as_logsdb_default_but_index_mod_is_not_logsdb() { + // given + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) + .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + .build(); + IndexSettings indexSettings = new IndexSettings(IndexMetadata.builder("index").settings(settings).build(), settings); + MappingParserContext mappingParserContext = mock(MappingParserContext.class); + doReturn(settings).when(mappingParserContext).getSettings(); + doReturn(indexSettings).when(mappingParserContext).getIndexSettings(); + doReturn(mock(ScriptCompiler.class)).when(mappingParserContext).scriptCompiler(); + + KeywordFieldMapper.Builder builder = new KeywordFieldMapper.Builder("field", mappingParserContext); + builder.ignoreAbove(IGNORE_ABOVE_DEFAULT_LOGSDB_INDICES); + + KeywordFieldMapper.KeywordFieldType fieldType = new KeywordFieldMapper.KeywordFieldType( + "field", + mock(FieldType.class), + mock(NamedAnalyzer.class), + mock(NamedAnalyzer.class), + mock(NamedAnalyzer.class), + builder, + true + ); + + // when/then + assertTrue(fieldType.isIgnoreAboveSet()); + assertEquals(IGNORE_ABOVE_DEFAULT_LOGSDB_INDICES, fieldType.ignoreAbove()); + } + + public void test_isIgnoreAboveSet_returns_false_when_ignore_above_is_configured_at_index_level() { // given Settings settings = Settings.builder() .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD) .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + .put(IndexSettings.IGNORE_ABOVE_SETTING.getKey(), 123) .build(); IndexSettings indexSettings = new IndexSettings(IndexMetadata.builder("index").settings(settings).build(), settings); MappingParserContext mappingParserContext = mock(MappingParserContext.class); @@ -436,7 +473,6 @@ public void test_isIgnoreAboveSet_returns_true_when_ignore_above_is_given_as_log doReturn(mock(ScriptCompiler.class)).when(mappingParserContext).scriptCompiler(); KeywordFieldMapper.Builder builder = new KeywordFieldMapper.Builder("field", mappingParserContext); - builder.ignoreAbove(IGNORE_ABOVE_DEFAULT_LOGSDB); KeywordFieldMapper.KeywordFieldType fieldType = new KeywordFieldMapper.KeywordFieldType( "field", @@ -449,7 +485,8 @@ public void test_isIgnoreAboveSet_returns_true_when_ignore_above_is_given_as_log ); // when/then - assertTrue(fieldType.isIgnoreAboveSet()); + assertFalse(fieldType.isIgnoreAboveSet()); + assertEquals(123, fieldType.ignoreAbove()); } public void test_isIgnoreAboveSet_returns_false_for_non_primary_constructor() { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ParameterTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ParameterTests.java index d33e6eb12b60c..23ef333fa1e89 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ParameterTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ParameterTests.java @@ -9,12 +9,9 @@ package org.elasticsearch.index.mapper; -import org.elasticsearch.index.IndexMode; -import org.elasticsearch.index.IndexVersion; import org.elasticsearch.test.ESTestCase; -import static org.elasticsearch.index.IndexSettings.IGNORE_ABOVE_DEFAULT; -import static org.elasticsearch.index.IndexSettings.IGNORE_ABOVE_DEFAULT_LOGSDB; +import static org.elasticsearch.index.IndexSettings.IGNORE_ABOVE_DEFAULT_STANDARD_INDICES; public class ParameterTests extends ESTestCase { @@ -23,31 +20,7 @@ public void test_ignore_above_param_default() { FieldMapper.Parameter ignoreAbove = FieldMapper.Parameter.ignoreAboveParam((FieldMapper fm) -> 123); // then - assertEquals(IGNORE_ABOVE_DEFAULT, ignoreAbove.getValue().intValue()); - } - - public void test_ignore_above_param_default_for_standard_indices() { - // when - FieldMapper.Parameter ignoreAbove = FieldMapper.Parameter.ignoreAboveParam( - (FieldMapper fm) -> 123, - IndexMode.STANDARD, - IndexVersion.current() - ); - - // then - assertEquals(IGNORE_ABOVE_DEFAULT, ignoreAbove.getValue().intValue()); - } - - public void test_ignore_above_param_default_for_logsdb_indices() { - // when - FieldMapper.Parameter ignoreAbove = FieldMapper.Parameter.ignoreAboveParam( - (FieldMapper fm) -> 123, - IndexMode.LOGSDB, - IndexVersion.current() - ); - - // then - assertEquals(IGNORE_ABOVE_DEFAULT_LOGSDB, ignoreAbove.getValue().intValue()); + assertEquals(IGNORE_ABOVE_DEFAULT_STANDARD_INDICES, ignoreAbove.getValue().intValue()); } public void test_ignore_above_param_invalid_value() { diff --git a/x-pack/plugin/logsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/TextRollingUpgradeIT.java b/x-pack/plugin/logsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/TextRollingUpgradeIT.java index 9a7fa40c38c5e..c7b2b99dc7632 100644 --- a/x-pack/plugin/logsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/TextRollingUpgradeIT.java +++ b/x-pack/plugin/logsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/TextRollingUpgradeIT.java @@ -79,7 +79,7 @@ public class TextRollingUpgradeIT extends AbstractRollingUpgradeWithSecurityTest } }"""; - // when sorted, this message will appear at the top + // when sorted, this message will appear at the top and hence can be used to validate query results private String smallestMessage; public TextRollingUpgradeIT(@Name("upgradedNodes") int upgradedNodes) { diff --git a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java index c5811e46afd72..c7ce503c7ad61 100644 --- a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java +++ b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java @@ -52,6 +52,7 @@ import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.core.Nullable; import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.analysis.AnalyzerScope; @@ -221,7 +222,10 @@ private Builder(String name, IndexMode indexMode, IndexVersion indexVersionCreat super(name); this.indexVersionCreated = indexVersionCreated; this.indexMode = indexMode; - this.ignoreAbove = Parameter.ignoreAboveParam(m -> toType(m).ignoreAbove, indexMode, indexVersionCreated); + this.ignoreAbove = Parameter.ignoreAboveParam( + m -> toType(m).ignoreAbove, + IndexSettings.getIgnoreAboveDefaultValue(indexMode, indexVersionCreated) + ); } @Override From ffb75550690bfed02983abe67db79bea4c0b46bd Mon Sep 17 00:00:00 2001 From: Dmitry Kubikov Date: Fri, 5 Sep 2025 16:35:10 -0700 Subject: [PATCH 09/23] Improved msg --- .../flattened/FlattenedFieldMapper.java | 26 ++++++--------- .../index/mapper/KeywordFieldTypeTests.java | 27 ++++++++-------- .../wildcard/mapper/WildcardFieldMapper.java | 32 +++++++++---------- 3 files changed, 38 insertions(+), 47 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java index ff66a414f4238..bc7ca0875e6a5 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java @@ -38,9 +38,7 @@ import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.core.Nullable; -import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; -import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.fielddata.FieldData; import org.elasticsearch.index.fielddata.FieldDataContext; @@ -85,6 +83,8 @@ import java.util.Set; import java.util.function.Function; +import static org.elasticsearch.index.IndexSettings.IGNORE_ABOVE_SETTING; + /** * A field mapper that accepts a JSON object and flattens it into a single field. This data type * can be a useful alternative to an 'object' mapping when the object has a large, unknown set @@ -152,6 +152,7 @@ public static class Builder extends FieldMapper.Builder { m -> builder(m).eagerGlobalOrdinals.get(), false ); + private final int ignoreAboveDefault; private final Parameter ignoreAbove; private final Parameter indexOptions = TextParams.keywordIndexOptions(m -> builder(m).indexOptions.get()); @@ -179,25 +180,18 @@ public static class Builder extends FieldMapper.Builder { private final Parameter> meta = Parameter.metaParam(); - private final IndexMode indexMode; - private final IndexVersion indexCreatedVersion; - public static FieldMapper.Parameter> dimensionsParam(Function> initializer) { return FieldMapper.Parameter.stringArrayParam(TIME_SERIES_DIMENSIONS_ARRAY_PARAM, false, initializer); } public Builder(final String name) { - this(name, IndexMode.STANDARD, IndexVersion.current()); + this(name, IndexSettings.IGNORE_ABOVE_DEFAULT_STANDARD_INDICES); } - private Builder(String name, IndexMode indexMode, IndexVersion indexCreatedVersion) { + private Builder(String name, int ignoreAboveDefault) { super(name); - this.indexMode = indexMode; - this.indexCreatedVersion = indexCreatedVersion; - this.ignoreAbove = Parameter.ignoreAboveParam( - m -> builder(m).ignoreAbove.get(), - IndexSettings.getIgnoreAboveDefaultValue(indexMode, indexCreatedVersion) - ); + this.ignoreAboveDefault = ignoreAboveDefault; + this.ignoreAbove = Parameter.ignoreAboveParam(m -> builder(m).ignoreAbove.get(), ignoreAboveDefault); this.dimensions.precludesParameters(ignoreAbove); } @@ -240,9 +234,7 @@ public FlattenedFieldMapper build(MapperBuilderContext context) { } } - public static final TypeParser PARSER = new TypeParser( - (n, c) -> new Builder(n, c.getIndexSettings().getMode(), c.indexVersionCreated()) - ); + public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n, IGNORE_ABOVE_SETTING.get(c.getSettings()))); /** * A field type that represents the values under a particular JSON key, used @@ -901,7 +893,7 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(leafName(), builder.indexMode, builder.indexCreatedVersion).init(this); + return new Builder(leafName(), builder.ignoreAboveDefault).init(this); } @Override diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java index 799d4d5eaa8d3..d85afcd85d891 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java @@ -9,6 +9,7 @@ package org.elasticsearch.index.mapper; import com.carrotsearch.randomizedtesting.generators.RandomStrings; + import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.LowerCaseFilter; import org.apache.lucene.analysis.TokenFilter; @@ -460,12 +461,12 @@ public void test_isIgnoreAboveSet_returns_true_when_ignore_above_is_given_as_log public void test_isIgnoreAboveSet_returns_false_when_ignore_above_is_configured_at_index_level() { // given Settings settings = Settings.builder() - .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) - .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD) - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) - .put(IndexSettings.IGNORE_ABOVE_SETTING.getKey(), 123) - .build(); + .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) + .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + .put(IndexSettings.IGNORE_ABOVE_SETTING.getKey(), 123) + .build(); IndexSettings indexSettings = new IndexSettings(IndexMetadata.builder("index").settings(settings).build(), settings); MappingParserContext mappingParserContext = mock(MappingParserContext.class); doReturn(settings).when(mappingParserContext).getSettings(); @@ -475,13 +476,13 @@ public void test_isIgnoreAboveSet_returns_false_when_ignore_above_is_configured_ KeywordFieldMapper.Builder builder = new KeywordFieldMapper.Builder("field", mappingParserContext); KeywordFieldMapper.KeywordFieldType fieldType = new KeywordFieldMapper.KeywordFieldType( - "field", - mock(FieldType.class), - mock(NamedAnalyzer.class), - mock(NamedAnalyzer.class), - mock(NamedAnalyzer.class), - builder, - true + "field", + mock(FieldType.class), + mock(NamedAnalyzer.class), + mock(NamedAnalyzer.class), + mock(NamedAnalyzer.class), + builder, + true ); // when/then diff --git a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java index c7ce503c7ad61..4e792ca2b079b 100644 --- a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java +++ b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java @@ -51,7 +51,6 @@ import org.elasticsearch.common.time.DateMathParser; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.core.Nullable; -import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; @@ -93,6 +92,8 @@ import java.util.Set; import java.util.TreeSet; +import static org.elasticsearch.index.IndexSettings.IGNORE_ABOVE_SETTING; + /** * A {@link FieldMapper} for indexing fields with ngrams for efficient wildcard matching */ @@ -211,21 +212,18 @@ public static class Builder extends FieldMapper.Builder { final Parameter> meta = Parameter.metaParam(); - final IndexMode indexMode; - final IndexVersion indexVersionCreated; + final IndexVersion indexCreatedVersion; + final int ignoreAboveDefault; public Builder(final String name, IndexVersion indexVersionCreated) { - this(name, IndexMode.STANDARD, indexVersionCreated); + this(name, IndexSettings.IGNORE_ABOVE_DEFAULT_STANDARD_INDICES, indexVersionCreated); } - private Builder(String name, IndexMode indexMode, IndexVersion indexVersionCreated) { + private Builder(String name, int ignoreAboveDefault, IndexVersion indexCreatedVersion) { super(name); - this.indexVersionCreated = indexVersionCreated; - this.indexMode = indexMode; - this.ignoreAbove = Parameter.ignoreAboveParam( - m -> toType(m).ignoreAbove, - IndexSettings.getIgnoreAboveDefaultValue(indexMode, indexVersionCreated) - ); + this.indexCreatedVersion = indexCreatedVersion; + this.ignoreAboveDefault = ignoreAboveDefault; + this.ignoreAbove = Parameter.ignoreAboveParam(m -> toType(m).ignoreAbove, ignoreAboveDefault); } @Override @@ -247,17 +245,17 @@ Builder nullValue(String nullValue) { public WildcardFieldMapper build(MapperBuilderContext context) { return new WildcardFieldMapper( leafName(), - new WildcardFieldType(context.buildFullName(leafName()), indexVersionCreated, meta.get(), this), + new WildcardFieldType(context.buildFullName(leafName()), indexCreatedVersion, meta.get(), this), context.isSourceSynthetic(), builderParams(this, context), - indexVersionCreated, + indexCreatedVersion, this ); } } public static final TypeParser PARSER = new TypeParser( - (n, c) -> new Builder(n, c.getIndexSettings().getMode(), c.indexVersionCreated()) + (n, c) -> new Builder(n, IGNORE_ABOVE_SETTING.get(c.getSettings()), c.indexVersionCreated()) ); public static final char TOKEN_START_OR_END_CHAR = 0; @@ -998,10 +996,10 @@ protected String parseSourceValue(Object value) { assert NGRAM_FIELD_TYPE.indexOptions() == IndexOptions.DOCS; } private final String nullValue; - private final IndexMode indexMode; private final IndexVersion indexVersionCreated; private final int ignoreAbove; + private final int ignoreAboveDefault; private final boolean storeIgnored; private final String originalName; @@ -1016,9 +1014,9 @@ private WildcardFieldMapper( super(simpleName, mappedFieldType, builderParams); this.nullValue = builder.nullValue.getValue(); this.storeIgnored = storeIgnored; - this.indexMode = builder.indexMode; this.indexVersionCreated = indexVersionCreated; this.ignoreAbove = builder.ignoreAbove.getValue(); + this.ignoreAboveDefault = builder.ignoreAboveDefault; this.originalName = storeIgnored ? fullPath() + "._original" : null; } @@ -1094,7 +1092,7 @@ protected String contentType() { @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(leafName(), indexMode, indexVersionCreated).init(this); + return new Builder(leafName(), ignoreAboveDefault, indexVersionCreated).init(this); } @Override From 000d944c77275fe29742b6327529bf3107cae2d3 Mon Sep 17 00:00:00 2001 From: Dmitry Kubikov Date: Mon, 8 Sep 2025 17:00:53 -0700 Subject: [PATCH 10/23] Added additional tests --- .../index/mapper/KeywordFieldMapper.java | 5 +- .../index/mapper/KeywordFieldMapperTests.java | 102 ++++++++++++++++++ .../index/mapper/KeywordFieldTypeTests.java | 33 +++++- 3 files changed, 138 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 9aee8a9f5ce14..7701abb76ff27 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -220,7 +220,7 @@ public Builder(final String name, final MappingParserContext mappingParserContex name, mappingParserContext.getIndexAnalyzers(), mappingParserContext.scriptCompiler(), - IGNORE_ABOVE_SETTING.get(mappingParserContext.getSettings()), + IGNORE_ABOVE_DEFAULT_STANDARD_INDICES, mappingParserContext.getIndexSettings().getIndexVersionCreated(), mappingParserContext.getIndexSettings().getMode(), mappingParserContext.getIndexSettings().getIndexSortConfig(), @@ -228,6 +228,9 @@ public Builder(final String name, final MappingParserContext mappingParserContex false, mappingParserContext.getIndexSettings().sourceKeepMode() ); + if (IGNORE_ABOVE_SETTING.exists(mappingParserContext.getSettings())) { + this.ignoreAbove.setValue(IGNORE_ABOVE_SETTING.get(mappingParserContext.getSettings())); + } } Builder( diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java index f6a7148c71091..cc2e0e61c2559 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java @@ -243,6 +243,108 @@ public void testIgnoreAbove() throws IOException { assertTrue(doc.rootDoc().getFields("_ignored").stream().anyMatch(field -> "field".equals(field.stringValue()))); } + public void test_ignore_above_index_level_setting() throws IOException { + // given + final MapperService mapperService = createMapperService( + Settings.builder() + .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.name()) + .put(IndexSettings.IGNORE_ABOVE_SETTING.getKey(), 123) + .build(), + mapping(b -> { + b.startObject("field"); + b.field("type", "keyword"); + b.endObject(); + }) + ); + + // when + final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("field"); + + // then + assertEquals(123, mapper.fieldType().ignoreAbove()); + assertTrue(mapper.fieldType().isIgnoreAboveSet()); + } + + public void test_ignore_above_index_level_setting_is_overridden_by_field_level_ignore_above_in_standard_indices() throws IOException { + // given + final MapperService mapperService = createMapperService( + Settings.builder() + .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD.name()) + .put(IndexSettings.IGNORE_ABOVE_SETTING.getKey(), 123) + .build(), + mapping(b -> { + b.startObject("potato"); + b.field("type", "keyword"); + b.field("ignore_above", 456); + b.endObject(); + + b.startObject("tomato"); + b.field("type", "keyword"); + b.field("ignore_above", 789); + b.endObject(); + + b.startObject("cheese"); + b.field("type", "keyword"); + b.endObject(); + }) + ); + + // when + final KeywordFieldMapper fieldMapper1 = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("potato"); + final KeywordFieldMapper fieldMapper2 = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("tomato"); + final KeywordFieldMapper fieldMapper3 = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("cheese"); + + // then + assertEquals(456, fieldMapper1.fieldType().ignoreAbove()); + assertTrue(fieldMapper1.fieldType().isIgnoreAboveSet()); + + assertEquals(789, fieldMapper2.fieldType().ignoreAbove()); + assertTrue(fieldMapper2.fieldType().isIgnoreAboveSet()); + + assertEquals(123, fieldMapper3.fieldType().ignoreAbove()); + assertTrue(fieldMapper3.fieldType().isIgnoreAboveSet()); + } + + public void test_ignore_above_index_level_setting_is_overridden_by_field_level_ignore_above_in_logsdb_indices() throws IOException { + // given + final MapperService mapperService = createMapperService( + Settings.builder() + .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB.name()) + .put(IndexSettings.IGNORE_ABOVE_SETTING.getKey(), 123) + .build(), + mapping(b -> { + b.startObject("potato"); + b.field("type", "keyword"); + b.field("ignore_above", 456); + b.endObject(); + + b.startObject("tomato"); + b.field("type", "keyword"); + b.field("ignore_above", 789); + b.endObject(); + + b.startObject("cheese"); + b.field("type", "keyword"); + b.endObject(); + }) + ); + + // when + final KeywordFieldMapper fieldMapper1 = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("potato"); + final KeywordFieldMapper fieldMapper2 = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("tomato"); + final KeywordFieldMapper fieldMapper3 = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("cheese"); + + // then + assertEquals(456, fieldMapper1.fieldType().ignoreAbove()); + assertTrue(fieldMapper1.fieldType().isIgnoreAboveSet()); + + assertEquals(789, fieldMapper2.fieldType().ignoreAbove()); + assertTrue(fieldMapper2.fieldType().isIgnoreAboveSet()); + + assertEquals(123, fieldMapper3.fieldType().ignoreAbove()); + assertTrue(fieldMapper3.fieldType().isIgnoreAboveSet()); + } + public void testNullValue() throws IOException { DocumentMapper mapper = createDocumentMapper(fieldMapping(this::minimalMapping)); ParsedDocument doc = mapper.parse(source(b -> b.nullField("field"))); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java index d85afcd85d891..539786006779d 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java @@ -9,7 +9,6 @@ package org.elasticsearch.index.mapper; import com.carrotsearch.randomizedtesting.generators.RandomStrings; - import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.LowerCaseFilter; import org.apache.lucene.analysis.TokenFilter; @@ -299,6 +298,38 @@ public void testGetTerms() throws IOException { } } + public void test_ignore_above_index_level_setting() { + // given + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) + .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + .put(IndexSettings.IGNORE_ABOVE_SETTING.getKey(), 123) + .build(); + IndexSettings indexSettings = new IndexSettings(IndexMetadata.builder("index").settings(settings).build(), settings); + MappingParserContext mappingParserContext = mock(MappingParserContext.class); + doReturn(settings).when(mappingParserContext).getSettings(); + doReturn(indexSettings).when(mappingParserContext).getIndexSettings(); + doReturn(mock(ScriptCompiler.class)).when(mappingParserContext).scriptCompiler(); + + KeywordFieldMapper.Builder builder = new KeywordFieldMapper.Builder("field", mappingParserContext); + + KeywordFieldMapper.KeywordFieldType fieldType = new KeywordFieldMapper.KeywordFieldType( + "field", + mock(FieldType.class), + mock(NamedAnalyzer.class), + mock(NamedAnalyzer.class), + mock(NamedAnalyzer.class), + builder, + true + ); + + // when/then + assertFalse(fieldType.isIgnoreAboveSet()); + assertEquals(123, fieldType.ignoreAbove()); + } + public void test_isIgnoreAboveSet_returns_true_when_ignore_above_is_given() { // given Settings settings = Settings.builder() From 1e9bfe42091531901eca4cc1d00b14f993e1548e Mon Sep 17 00:00:00 2001 From: Dmitry Kubikov Date: Mon, 8 Sep 2025 18:22:46 -0700 Subject: [PATCH 11/23] Added IgnoreAbove record, addressed index-level ignore above --- .../extras/MatchOnlyTextFieldMapper.java | 5 +- .../org/elasticsearch/index/IgnoreAbove.java | 74 +++++++++++++++++++ .../index/mapper/FieldMapper.java | 9 ++- .../index/mapper/KeywordFieldMapper.java | 58 +++++---------- .../index/mapper/TextFieldMapper.java | 6 +- .../flattened/FlattenedFieldMapper.java | 58 +++++++++------ .../index/mapper/KeywordFieldMapperTests.java | 32 ++++---- .../index/mapper/KeywordFieldTypeTests.java | 63 ++++++++-------- .../flattened/FlattenedFieldMapperTests.java | 5 +- .../RootFlattenedFieldTypeTests.java | 17 ++++- .../wildcard/mapper/WildcardFieldMapper.java | 64 +++++++++------- 11 files changed, 241 insertions(+), 150 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/index/IgnoreAbove.java diff --git a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java index faea13dac4e31..1d9fd9cb3a13b 100644 --- a/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java +++ b/modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java @@ -253,8 +253,7 @@ private IOFunction, IOExcepti String parentField = searchExecutionContext.parentPath(name()); var parent = searchExecutionContext.lookup().fieldType(parentField); - if (parent instanceof KeywordFieldMapper.KeywordFieldType keywordParent - && keywordParent.ignoreAbove() != Integer.MAX_VALUE) { + if (parent instanceof KeywordFieldMapper.KeywordFieldType keywordParent && keywordParent.ignoreAbove().isSet()) { if (parent.isStored()) { return storedFieldFetcher(parentField, keywordParent.originalName()); } else if (parent.hasDocValues()) { @@ -278,7 +277,7 @@ private IOFunction, IOExcepti if (kwd != null) { var fieldType = kwd.fieldType(); - if (fieldType.ignoreAbove() != Integer.MAX_VALUE) { + if (fieldType.ignoreAbove().isSet()) { if (fieldType.isStored()) { return storedFieldFetcher(fieldType.name(), fieldType.originalName()); } else if (fieldType.hasDocValues()) { diff --git a/server/src/main/java/org/elasticsearch/index/IgnoreAbove.java b/server/src/main/java/org/elasticsearch/index/IgnoreAbove.java new file mode 100644 index 0000000000000..44da20f2e5556 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/IgnoreAbove.java @@ -0,0 +1,74 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index; + +import org.elasticsearch.xcontent.XContentString; + +import static org.elasticsearch.index.IndexSettings.IGNORE_ABOVE_DEFAULT_LOGSDB_INDICES; +import static org.elasticsearch.index.IndexSettings.IGNORE_ABOVE_DEFAULT_STANDARD_INDICES; + +public record IgnoreAbove(Integer value, int defaultValue) { + + public static final IgnoreAbove IGNORE_ABOVE_STANDARD_INDICES = IgnoreAbove.builder() + .defaultValue(IGNORE_ABOVE_DEFAULT_STANDARD_INDICES) + .build(); + + public static final IgnoreAbove IGNORE_ABOVE_LOGSDB_INDICES = IgnoreAbove.builder() + .defaultValue(IGNORE_ABOVE_DEFAULT_LOGSDB_INDICES) + .build(); + + public int get() { + return value != null ? value : defaultValue; + } + + public boolean isSet() { + return value != defaultValue; + } + + public boolean isIgnored(final String s) { + if (s == null) return false; + return lengthExceedsIgnoreAbove(s.length()); + } + + public boolean isIgnored(final XContentString s) { + if (s == null) return false; + return lengthExceedsIgnoreAbove(s.stringLength()); + } + + private boolean lengthExceedsIgnoreAbove(int strLength) { + return strLength > get(); + } + + public static Builder builder() { + return new Builder(); + } + + public static final class Builder { + + private Integer value; + private int defaultValue; // cannot be null, hence int + + private Builder() {} + + public Builder value(Integer value) { + this.value = value; + return this; + } + + public Builder defaultValue(int defaultValue) { + this.defaultValue = defaultValue; + return this; + } + + public IgnoreAbove build() { + return new IgnoreAbove(value, defaultValue); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index a5f2d22360d6a..048f5a2254cbe 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -1338,10 +1338,15 @@ public static Parameter normsParam(Function initi } public static Parameter ignoreAboveParam(Function initializer) { - return ignoreAboveParam(initializer, IndexSettings.IGNORE_ABOVE_DEFAULT_STANDARD_INDICES); + return ignoreAboveParam(initializer, null, null); } - public static Parameter ignoreAboveParam(Function initializer, final int defaultValue) { + public static Parameter ignoreAboveParam( + Function initializer, + final IndexMode indexMode, + final IndexVersion indexCreatedVersion + ) { + final int defaultValue = IndexSettings.getIgnoreAboveDefaultValue(indexMode, indexCreatedVersion); return Parameter.intParam("ignore_above", true, initializer, defaultValue).addValidator(v -> { if (v < 0) { throw new IllegalArgumentException("[ignore_above] must be positive, got [" + v + "]"); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 7701abb76ff27..8966b478ba466 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -41,8 +41,8 @@ import org.elasticsearch.common.lucene.search.AutomatonQueries; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.core.Nullable; +import org.elasticsearch.index.IgnoreAbove; import org.elasticsearch.index.IndexMode; -import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexSortConfig; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; @@ -90,7 +90,6 @@ import static org.apache.lucene.index.IndexWriter.MAX_TERM_LENGTH; import static org.elasticsearch.core.Strings.format; -import static org.elasticsearch.index.IndexSettings.IGNORE_ABOVE_DEFAULT_STANDARD_INDICES; import static org.elasticsearch.index.IndexSettings.IGNORE_ABOVE_SETTING; import static org.elasticsearch.index.IndexSettings.USE_DOC_VALUES_SKIPPER; import static org.elasticsearch.index.mapper.FieldArrayContext.getOffsetsFieldName; @@ -181,7 +180,6 @@ public static final class Builder extends FieldMapper.DimensionBuilder { false ); private final Parameter ignoreAbove; - private final int ignoreAboveDefault; private final IndexSortConfig indexSortConfig; private final IndexMode indexMode; private final Parameter indexOptions = TextParams.keywordIndexOptions(m -> toType(m).indexOptions); @@ -220,7 +218,6 @@ public Builder(final String name, final MappingParserContext mappingParserContex name, mappingParserContext.getIndexAnalyzers(), mappingParserContext.scriptCompiler(), - IGNORE_ABOVE_DEFAULT_STANDARD_INDICES, mappingParserContext.getIndexSettings().getIndexVersionCreated(), mappingParserContext.getIndexSettings().getMode(), mappingParserContext.getIndexSettings().getIndexSortConfig(), @@ -228,6 +225,8 @@ public Builder(final String name, final MappingParserContext mappingParserContex false, mappingParserContext.getIndexSettings().sourceKeepMode() ); + + // if ignore_above is configured at index-level, then set it now if (IGNORE_ABOVE_SETTING.exists(mappingParserContext.getSettings())) { this.ignoreAbove.setValue(IGNORE_ABOVE_SETTING.get(mappingParserContext.getSettings())); } @@ -240,25 +239,13 @@ public Builder(final String name, final MappingParserContext mappingParserContex IndexVersion indexCreatedVersion, SourceKeepMode sourceKeepMode ) { - this( - name, - indexAnalyzers, - scriptCompiler, - IndexSettings.getIgnoreAboveDefaultValue(IndexMode.STANDARD, indexCreatedVersion), - indexCreatedVersion, - IndexMode.STANDARD, - null, - false, - false, - sourceKeepMode - ); + this(name, indexAnalyzers, scriptCompiler, indexCreatedVersion, IndexMode.STANDARD, null, false, false, sourceKeepMode); } private Builder( String name, IndexAnalyzers indexAnalyzers, ScriptCompiler scriptCompiler, - int ignoreAboveDefault, IndexVersion indexCreatedVersion, IndexMode indexMode, IndexSortConfig indexSortConfig, @@ -292,8 +279,7 @@ private Builder( ); } }).precludesParameters(normalizer); - this.ignoreAboveDefault = ignoreAboveDefault; - this.ignoreAbove = Parameter.ignoreAboveParam(m -> toType(m).fieldType().ignoreAbove(), ignoreAboveDefault); + this.ignoreAbove = Parameter.ignoreAboveParam(m -> toType(m).fieldType().ignoreAbove().get(), indexMode, indexCreatedVersion); this.indexSortConfig = indexSortConfig; this.indexMode = indexMode; this.enableDocValuesSkipper = enableDocValuesSkipper; @@ -315,7 +301,6 @@ public static Builder buildWithDocValuesSkipper( name, null, ScriptCompiler.NONE, - IndexSettings.getIgnoreAboveDefaultValue(indexMode, indexCreatedVersion), indexCreatedVersion, indexMode, // Sort config is used to decide if DocValueSkippers can be used. Since skippers are forced, a sort config is not needed. @@ -536,8 +521,7 @@ private static boolean indexSortConfigByHostName(final IndexSortConfig indexSort public static final class KeywordFieldType extends StringFieldType { - private final int ignoreAbove; - private final int ignoreAboveDefaultValue; + private final IgnoreAbove ignoreAbove; private final String nullValue; private final NamedAnalyzer normalizer; private final boolean eagerGlobalOrdinals; @@ -567,8 +551,10 @@ public KeywordFieldType( ); this.eagerGlobalOrdinals = builder.eagerGlobalOrdinals.getValue(); this.normalizer = normalizer; - this.ignoreAbove = builder.ignoreAbove.getValue(); - this.ignoreAboveDefaultValue = builder.ignoreAbove.getDefaultValue(); + this.ignoreAbove = IgnoreAbove.builder() + .value(builder.ignoreAbove.getValue()) + .defaultValue(builder.ignoreAbove.getDefaultValue()) + .build(); this.nullValue = builder.nullValue.getValue(); this.scriptValues = builder.scriptValues(); this.isDimension = builder.dimension.getValue(); @@ -585,7 +571,7 @@ public KeywordFieldType(String name) { public KeywordFieldType(String name, boolean isIndexed, boolean hasDocValues, Map meta) { super(name, isIndexed, false, hasDocValues, TextSearchInfo.SIMPLE_MATCH_ONLY, meta); this.normalizer = Lucene.KEYWORD_ANALYZER; - this.ignoreAbove = this.ignoreAboveDefaultValue = IGNORE_ABOVE_DEFAULT_STANDARD_INDICES; + this.ignoreAbove = IgnoreAbove.IGNORE_ABOVE_STANDARD_INDICES; this.nullValue = null; this.eagerGlobalOrdinals = false; this.scriptValues = null; @@ -606,7 +592,7 @@ public KeywordFieldType(String name, FieldType fieldType) { Collections.emptyMap() ); this.normalizer = Lucene.KEYWORD_ANALYZER; - this.ignoreAbove = this.ignoreAboveDefaultValue = IGNORE_ABOVE_DEFAULT_STANDARD_INDICES; + this.ignoreAbove = IgnoreAbove.IGNORE_ABOVE_STANDARD_INDICES; this.nullValue = null; this.eagerGlobalOrdinals = false; this.scriptValues = null; @@ -620,7 +606,7 @@ public KeywordFieldType(String name, FieldType fieldType) { public KeywordFieldType(String name, NamedAnalyzer analyzer) { super(name, true, false, true, textSearchInfo(Defaults.FIELD_TYPE, null, analyzer, analyzer), Collections.emptyMap()); this.normalizer = Lucene.KEYWORD_ANALYZER; - this.ignoreAbove = this.ignoreAboveDefaultValue = IGNORE_ABOVE_DEFAULT_STANDARD_INDICES; + this.ignoreAbove = IgnoreAbove.IGNORE_ABOVE_STANDARD_INDICES; this.nullValue = null; this.eagerGlobalOrdinals = false; this.scriptValues = null; @@ -934,10 +920,7 @@ protected String parseSourceValue(Object value) { } private String applyIgnoreAboveAndNormalizer(String value) { - if (value.length() > ignoreAbove) { - return null; - } - + if (ignoreAbove.isIgnored(value)) return null; return normalizeValue(normalizer(), name(), value); } @@ -1056,14 +1039,10 @@ public CollapseType collapseType() { /** Values that have more chars than the return value of this method will * be skipped at parsing time. */ - public int ignoreAbove() { + public IgnoreAbove ignoreAbove() { return ignoreAbove; } - public boolean isIgnoreAboveSet() { - return ignoreAbove != ignoreAboveDefaultValue; - } - @Override public boolean isDimension() { return isDimension; @@ -1119,7 +1098,6 @@ public String originalName() { private final boolean isSyntheticSource; private final IndexAnalyzers indexAnalyzers; - private final int ignoreAboveDefault; private final IndexMode indexMode; private final IndexSortConfig indexSortConfig; private final boolean enableDocValuesSkipper; @@ -1151,7 +1129,6 @@ private KeywordFieldMapper( this.scriptCompiler = builder.scriptCompiler; this.indexCreatedVersion = builder.indexCreatedVersion; this.isSyntheticSource = isSyntheticSource; - this.ignoreAboveDefault = builder.ignoreAboveDefault; this.indexMode = builder.indexMode; this.indexSortConfig = builder.indexSortConfig; this.enableDocValuesSkipper = builder.enableDocValuesSkipper; @@ -1212,7 +1189,7 @@ private boolean indexValue(DocumentParserContext context, XContentString value) return false; } - if (value.stringLength() > fieldType().ignoreAbove()) { + if (fieldType().ignoreAbove().isIgnored(value)) { context.addIgnoredField(fullPath()); if (isSyntheticSource) { // Save a copy of the field so synthetic source can load it @@ -1307,7 +1284,6 @@ public FieldMapper.Builder getMergeBuilder() { leafName(), indexAnalyzers, scriptCompiler, - ignoreAboveDefault, indexCreatedVersion, indexMode, indexSortConfig, @@ -1381,7 +1357,7 @@ protected BytesRef preserve(BytesRef value) { } } - if (fieldType().isIgnoreAboveSet()) { + if (fieldType().ignoreAbove.isSet()) { layers.add(new CompositeSyntheticFieldLoader.StoredFieldLayer(originalName) { @Override protected void writeValue(Object value, XContentBuilder b) throws IOException { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index 572e40135897d..1a232ecf09f72 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -1021,7 +1021,7 @@ public boolean isAggregatable() { * A delegate by definition must have doc_values or be stored so most of the time it can be used for loading. */ public boolean canUseSyntheticSourceDelegateForLoading() { - return syntheticSourceDelegate != null && syntheticSourceDelegate.isIgnoreAboveSet() == false; + return syntheticSourceDelegate != null && syntheticSourceDelegate.ignoreAbove().isSet() == false; } /** @@ -1029,7 +1029,7 @@ public boolean canUseSyntheticSourceDelegateForLoading() { */ public boolean canUseSyntheticSourceDelegateForQuerying() { return syntheticSourceDelegate != null - && syntheticSourceDelegate.isIgnoreAboveSet() == false + && syntheticSourceDelegate.ignoreAbove().isSet() == false && syntheticSourceDelegate.isIndexed(); } @@ -1045,7 +1045,7 @@ public boolean canUseSyntheticSourceDelegateForQueryingEquality(String str) { return false; } // Can't push equality if the field we're checking for is so big we'd ignore it. - return str.length() <= syntheticSourceDelegate.ignoreAbove(); + return syntheticSourceDelegate.ignoreAbove().isIgnored(str) == false; } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java index bc7ca0875e6a5..2835de6d7edab 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java @@ -38,7 +38,9 @@ import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.core.Nullable; -import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.IgnoreAbove; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.fielddata.FieldData; import org.elasticsearch.index.fielddata.FieldDataContext; @@ -56,6 +58,7 @@ import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.MapperBuilderContext; +import org.elasticsearch.index.mapper.MappingParserContext; import org.elasticsearch.index.mapper.SourceValueFetcher; import org.elasticsearch.index.mapper.StringFieldType; import org.elasticsearch.index.mapper.TextParams; @@ -125,8 +128,6 @@ private static Builder builder(Mapper in) { return ((FlattenedFieldMapper) in).builder; } - private final int ignoreAbove; - public static class Builder extends FieldMapper.Builder { final Parameter depthLimit = Parameter.intParam( @@ -152,7 +153,6 @@ public static class Builder extends FieldMapper.Builder { m -> builder(m).eagerGlobalOrdinals.get(), false ); - private final int ignoreAboveDefault; private final Parameter ignoreAbove; private final Parameter indexOptions = TextParams.keywordIndexOptions(m -> builder(m).indexOptions.get()); @@ -180,18 +180,31 @@ public static class Builder extends FieldMapper.Builder { private final Parameter> meta = Parameter.metaParam(); + private final IndexMode indexMode; + private final IndexVersion indexCreatedVersion; + public static FieldMapper.Parameter> dimensionsParam(Function> initializer) { return FieldMapper.Parameter.stringArrayParam(TIME_SERIES_DIMENSIONS_ARRAY_PARAM, false, initializer); } public Builder(final String name) { - this(name, IndexSettings.IGNORE_ABOVE_DEFAULT_STANDARD_INDICES); + this(name, IndexMode.STANDARD, IndexVersion.current()); + } + + private Builder(String name, MappingParserContext mappingParserContext) { + this(name, mappingParserContext.getIndexSettings().getMode(), mappingParserContext.indexVersionCreated()); + + // if ignore_above is configured at index-level, then set it now + if (IGNORE_ABOVE_SETTING.exists(mappingParserContext.getSettings())) { + this.ignoreAbove.setValue(IGNORE_ABOVE_SETTING.get(mappingParserContext.getSettings())); + } } - private Builder(String name, int ignoreAboveDefault) { + private Builder(String name, IndexMode indexMode, IndexVersion indexCreatedVersion) { super(name); - this.ignoreAboveDefault = ignoreAboveDefault; - this.ignoreAbove = Parameter.ignoreAboveParam(m -> builder(m).ignoreAbove.get(), ignoreAboveDefault); + this.indexMode = indexMode; + this.indexCreatedVersion = indexCreatedVersion; + this.ignoreAbove = Parameter.ignoreAboveParam(m -> builder(m).ignoreAbove.get(), indexMode, indexCreatedVersion); this.dimensions.precludesParameters(ignoreAbove); } @@ -228,13 +241,13 @@ public FlattenedFieldMapper build(MapperBuilderContext context) { splitQueriesOnWhitespace.get(), eagerGlobalOrdinals.get(), dimensions.get(), - ignoreAbove.getValue() + IgnoreAbove.builder().value(ignoreAbove.getValue()).defaultValue(ignoreAbove.getDefaultValue()).build() ); return new FlattenedFieldMapper(leafName(), ft, builderParams(this, context), this); } } - public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n, IGNORE_ABOVE_SETTING.get(c.getSettings()))); + public static final TypeParser PARSER = createTypeParserWithLegacySupport(FlattenedFieldMapper.Builder::new); /** * A field type that represents the values under a particular JSON key, used @@ -660,7 +673,7 @@ public static final class RootFlattenedFieldType extends StringFieldType impleme private final boolean eagerGlobalOrdinals; private final List dimensions; private final boolean isDimension; - private final int ignoreAbove; + private final IgnoreAbove ignoreAbove; RootFlattenedFieldType( String name, @@ -669,7 +682,7 @@ public static final class RootFlattenedFieldType extends StringFieldType impleme Map meta, boolean splitQueriesOnWhitespace, boolean eagerGlobalOrdinals, - int ignoreAbove + IgnoreAbove ignoreAbove ) { this(name, indexed, hasDocValues, meta, splitQueriesOnWhitespace, eagerGlobalOrdinals, Collections.emptyList(), ignoreAbove); } @@ -682,7 +695,7 @@ public static final class RootFlattenedFieldType extends StringFieldType impleme boolean splitQueriesOnWhitespace, boolean eagerGlobalOrdinals, List dimensions, - int ignoreAbove + IgnoreAbove ignoreAbove ) { super( name, @@ -736,6 +749,10 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) return sourceValueFetcher(context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet()); } + public IgnoreAbove ignoreAbove() { + return ignoreAbove; + } + private SourceValueFetcher sourceValueFetcher(Set sourcePaths) { return new SourceValueFetcher(sourcePaths, null) { @Override @@ -745,7 +762,7 @@ protected Object parseSourceValue(Object value) { final Map result = filterIgnoredValues((Map) valueAsMap); return result.isEmpty() ? null : result; } - if (value instanceof String valueAsString && valueAsString.length() <= ignoreAbove) { + if (value instanceof String valueAsString && ignoreAbove.isIgnored(valueAsString) == false) { return valueAsString; } return null; @@ -767,7 +784,7 @@ private Object filterIgnoredValues(final Object entryValue) { final List validValues = new ArrayList<>(); for (Object value : valueAsList) { if (value instanceof String valueAsString) { - if (valueAsString.length() <= ignoreAbove) { + if (ignoreAbove.isIgnored(valueAsString) == false) { validValues.add(valueAsString); } } else { @@ -783,7 +800,7 @@ private Object filterIgnoredValues(final Object entryValue) { } return validValues; } else if (entryValue instanceof String valueAsString) { - if (valueAsString.length() <= ignoreAbove) { + if (ignoreAbove.isIgnored(valueAsString) == false) { return valueAsString; } return null; @@ -826,7 +843,6 @@ public void validateMatchedRoutingPath(final String routingPath) { private FlattenedFieldMapper(String leafName, MappedFieldType mappedFieldType, BuilderParams builderParams, Builder builder) { super(leafName, mappedFieldType, builderParams); this.builder = builder; - this.ignoreAbove = builder.ignoreAbove.get(); this.fieldParser = new FlattenedFieldParser( mappedFieldType.name(), mappedFieldType.name() + KEYED_FIELD_SUFFIX, @@ -852,10 +868,6 @@ int depthLimit() { return builder.depthLimit.get(); } - public int ignoreAbove() { - return ignoreAbove; - } - @Override public RootFlattenedFieldType fieldType() { return (RootFlattenedFieldType) super.fieldType(); @@ -893,7 +905,7 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(leafName(), builder.ignoreAboveDefault).init(this); + return new Builder(leafName(), builder.indexMode, builder.indexCreatedVersion).init(this); } @Override @@ -903,7 +915,7 @@ protected SyntheticSourceSupport syntheticSourceSupport() { () -> new FlattenedSortedSetDocValuesSyntheticFieldLoader( fullPath(), fullPath() + KEYED_FIELD_SUFFIX, - ignoreAbove() < Integer.MAX_VALUE ? fullPath() + KEYED_IGNORED_VALUES_FIELD_SUFFIX : null, + fieldType().ignoreAbove.isSet() ? fullPath() + KEYED_IGNORED_VALUES_FIELD_SUFFIX : null, leafName() ) ); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java index cc2e0e61c2559..0449b4ff7dbf4 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldMapperTests.java @@ -179,7 +179,7 @@ protected void registerParameters(ParameterChecker checker) throws IOException { checker.registerUpdateCheck(b -> b.field("eager_global_ordinals", true), m -> assertTrue(m.fieldType().eagerGlobalOrdinals())); checker.registerUpdateCheck( b -> b.field("ignore_above", 256), - m -> assertEquals(256, ((KeywordFieldMapper) m).fieldType().ignoreAbove()) + m -> assertEquals(256, ((KeywordFieldMapper) m).fieldType().ignoreAbove().get()) ); checker.registerUpdateCheck( b -> b.field("split_queries_on_whitespace", true), @@ -261,8 +261,8 @@ public void test_ignore_above_index_level_setting() throws IOException { final KeywordFieldMapper mapper = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("field"); // then - assertEquals(123, mapper.fieldType().ignoreAbove()); - assertTrue(mapper.fieldType().isIgnoreAboveSet()); + assertEquals(123, mapper.fieldType().ignoreAbove().get()); + assertTrue(mapper.fieldType().ignoreAbove().isSet()); } public void test_ignore_above_index_level_setting_is_overridden_by_field_level_ignore_above_in_standard_indices() throws IOException { @@ -295,14 +295,14 @@ public void test_ignore_above_index_level_setting_is_overridden_by_field_level_i final KeywordFieldMapper fieldMapper3 = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("cheese"); // then - assertEquals(456, fieldMapper1.fieldType().ignoreAbove()); - assertTrue(fieldMapper1.fieldType().isIgnoreAboveSet()); + assertEquals(456, fieldMapper1.fieldType().ignoreAbove().get()); + assertTrue(fieldMapper1.fieldType().ignoreAbove().isSet()); - assertEquals(789, fieldMapper2.fieldType().ignoreAbove()); - assertTrue(fieldMapper2.fieldType().isIgnoreAboveSet()); + assertEquals(789, fieldMapper2.fieldType().ignoreAbove().get()); + assertTrue(fieldMapper2.fieldType().ignoreAbove().isSet()); - assertEquals(123, fieldMapper3.fieldType().ignoreAbove()); - assertTrue(fieldMapper3.fieldType().isIgnoreAboveSet()); + assertEquals(123, fieldMapper3.fieldType().ignoreAbove().get()); + assertTrue(fieldMapper3.fieldType().ignoreAbove().isSet()); } public void test_ignore_above_index_level_setting_is_overridden_by_field_level_ignore_above_in_logsdb_indices() throws IOException { @@ -335,14 +335,14 @@ public void test_ignore_above_index_level_setting_is_overridden_by_field_level_i final KeywordFieldMapper fieldMapper3 = (KeywordFieldMapper) mapperService.documentMapper().mappers().getMapper("cheese"); // then - assertEquals(456, fieldMapper1.fieldType().ignoreAbove()); - assertTrue(fieldMapper1.fieldType().isIgnoreAboveSet()); + assertEquals(456, fieldMapper1.fieldType().ignoreAbove().get()); + assertTrue(fieldMapper1.fieldType().ignoreAbove().isSet()); - assertEquals(789, fieldMapper2.fieldType().ignoreAbove()); - assertTrue(fieldMapper2.fieldType().isIgnoreAboveSet()); + assertEquals(789, fieldMapper2.fieldType().ignoreAbove().get()); + assertTrue(fieldMapper2.fieldType().ignoreAbove().isSet()); - assertEquals(123, fieldMapper3.fieldType().ignoreAbove()); - assertTrue(fieldMapper3.fieldType().isIgnoreAboveSet()); + assertEquals(123, fieldMapper3.fieldType().ignoreAbove().get()); + assertTrue(fieldMapper3.fieldType().ignoreAbove().isSet()); } public void testNullValue() throws IOException { @@ -433,7 +433,7 @@ public void testDimensionAndIgnoreAbove() throws IOException { b.field("time_series_dimension", true).field("ignore_above", 2048); })); KeywordFieldMapper field = (KeywordFieldMapper) documentMapper.mappers().getMapper("field"); - assertEquals(2048, field.fieldType().ignoreAbove()); + assertEquals(2048, field.fieldType().ignoreAbove().get()); } public void testDimensionAndNormalizer() { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java index 539786006779d..91848a99dbbd2 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java @@ -9,6 +9,7 @@ package org.elasticsearch.index.mapper; import com.carrotsearch.randomizedtesting.generators.RandomStrings; + import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.LowerCaseFilter; import org.apache.lucene.analysis.TokenFilter; @@ -301,12 +302,12 @@ public void testGetTerms() throws IOException { public void test_ignore_above_index_level_setting() { // given Settings settings = Settings.builder() - .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) - .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD) - .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) - .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) - .put(IndexSettings.IGNORE_ABOVE_SETTING.getKey(), 123) - .build(); + .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) + .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + .put(IndexSettings.IGNORE_ABOVE_SETTING.getKey(), 123) + .build(); IndexSettings indexSettings = new IndexSettings(IndexMetadata.builder("index").settings(settings).build(), settings); MappingParserContext mappingParserContext = mock(MappingParserContext.class); doReturn(settings).when(mappingParserContext).getSettings(); @@ -316,18 +317,18 @@ public void test_ignore_above_index_level_setting() { KeywordFieldMapper.Builder builder = new KeywordFieldMapper.Builder("field", mappingParserContext); KeywordFieldMapper.KeywordFieldType fieldType = new KeywordFieldMapper.KeywordFieldType( - "field", - mock(FieldType.class), - mock(NamedAnalyzer.class), - mock(NamedAnalyzer.class), - mock(NamedAnalyzer.class), - builder, - true + "field", + mock(FieldType.class), + mock(NamedAnalyzer.class), + mock(NamedAnalyzer.class), + mock(NamedAnalyzer.class), + builder, + true ); // when/then - assertFalse(fieldType.isIgnoreAboveSet()); - assertEquals(123, fieldType.ignoreAbove()); + assertFalse(fieldType.ignoreAbove().isSet()); + assertEquals(123, fieldType.ignoreAbove().get()); } public void test_isIgnoreAboveSet_returns_true_when_ignore_above_is_given() { @@ -358,8 +359,8 @@ public void test_isIgnoreAboveSet_returns_true_when_ignore_above_is_given() { ); // when/then - assertTrue(fieldType.isIgnoreAboveSet()); - assertEquals(123, fieldType.ignoreAbove()); + assertTrue(fieldType.ignoreAbove().isSet()); + assertEquals(123, fieldType.ignoreAbove().get()); } public void test_isIgnoreAboveSet_returns_false_when_ignore_above_is_not_given() { @@ -389,8 +390,8 @@ public void test_isIgnoreAboveSet_returns_false_when_ignore_above_is_not_given() ); // when/then - assertFalse(fieldType.isIgnoreAboveSet()); - assertEquals(IGNORE_ABOVE_DEFAULT_STANDARD_INDICES, fieldType.ignoreAbove()); + assertFalse(fieldType.ignoreAbove().isSet()); + assertEquals(IGNORE_ABOVE_DEFAULT_STANDARD_INDICES, fieldType.ignoreAbove().get()); } public void test_isIgnoreAboveSet_returns_false_when_ignore_above_is_given_but_its_the_same_as_default() { @@ -421,8 +422,8 @@ public void test_isIgnoreAboveSet_returns_false_when_ignore_above_is_given_but_i ); // when/then - assertFalse(fieldType.isIgnoreAboveSet()); - assertEquals(IGNORE_ABOVE_DEFAULT_STANDARD_INDICES, fieldType.ignoreAbove()); + assertFalse(fieldType.ignoreAbove().isSet()); + assertEquals(IGNORE_ABOVE_DEFAULT_STANDARD_INDICES, fieldType.ignoreAbove().get()); } public void test_isIgnoreAboveSet_returns_false_when_ignore_above_is_given_but_its_the_same_as_default_for_logsdb_indices() { @@ -453,8 +454,8 @@ public void test_isIgnoreAboveSet_returns_false_when_ignore_above_is_given_but_i ); // when/then - assertFalse(fieldType.isIgnoreAboveSet()); - assertEquals(IGNORE_ABOVE_DEFAULT_LOGSDB_INDICES, fieldType.ignoreAbove()); + assertFalse(fieldType.ignoreAbove().isSet()); + assertEquals(IGNORE_ABOVE_DEFAULT_LOGSDB_INDICES, fieldType.ignoreAbove().get()); } public void test_isIgnoreAboveSet_returns_true_when_ignore_above_is_given_as_logsdb_default_but_index_mod_is_not_logsdb() { @@ -485,8 +486,8 @@ public void test_isIgnoreAboveSet_returns_true_when_ignore_above_is_given_as_log ); // when/then - assertTrue(fieldType.isIgnoreAboveSet()); - assertEquals(IGNORE_ABOVE_DEFAULT_LOGSDB_INDICES, fieldType.ignoreAbove()); + assertTrue(fieldType.ignoreAbove().isSet()); + assertEquals(IGNORE_ABOVE_DEFAULT_LOGSDB_INDICES, fieldType.ignoreAbove().get()); } public void test_isIgnoreAboveSet_returns_false_when_ignore_above_is_configured_at_index_level() { @@ -517,8 +518,8 @@ public void test_isIgnoreAboveSet_returns_false_when_ignore_above_is_configured_ ); // when/then - assertFalse(fieldType.isIgnoreAboveSet()); - assertEquals(123, fieldType.ignoreAbove()); + assertFalse(fieldType.ignoreAbove().isSet()); + assertEquals(123, fieldType.ignoreAbove().get()); } public void test_isIgnoreAboveSet_returns_false_for_non_primary_constructor() { @@ -529,10 +530,10 @@ public void test_isIgnoreAboveSet_returns_false_for_non_primary_constructor() { KeywordFieldType fieldType4 = new KeywordFieldType("field", mock(NamedAnalyzer.class)); // when/then - assertFalse(fieldType1.isIgnoreAboveSet()); - assertFalse(fieldType2.isIgnoreAboveSet()); - assertFalse(fieldType3.isIgnoreAboveSet()); - assertFalse(fieldType4.isIgnoreAboveSet()); + assertFalse(fieldType1.ignoreAbove().isSet()); + assertFalse(fieldType2.ignoreAbove().isSet()); + assertFalse(fieldType3.ignoreAbove().isSet()); + assertFalse(fieldType4.ignoreAbove().isSet()); } private static IndexAnalyzers createIndexAnalyzers() { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java index 0628ca47308ea..d9223a157b2ff 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapperTests.java @@ -86,7 +86,10 @@ protected void registerParameters(ParameterChecker checker) throws IOException { checker.registerConflictCheck("time_series_dimensions", b -> b.field("time_series_dimensions", List.of("one", "two"))); checker.registerUpdateCheck(b -> b.field("eager_global_ordinals", true), m -> assertTrue(m.fieldType().eagerGlobalOrdinals())); - checker.registerUpdateCheck(b -> b.field("ignore_above", 256), m -> assertEquals(256, ((FlattenedFieldMapper) m).ignoreAbove())); + checker.registerUpdateCheck( + b -> b.field("ignore_above", 256), + m -> assertEquals(256, ((FlattenedFieldMapper) m).fieldType().ignoreAbove().get()) + ); checker.registerUpdateCheck( b -> b.field("split_queries_on_whitespace", true), m -> assertEquals("_whitespace", m.fieldType().getTextSearchInfo().searchAnalyzer().name()) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/flattened/RootFlattenedFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/flattened/RootFlattenedFieldTypeTests.java index 7873458eb46ee..0497bf2fa2a9a 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/flattened/RootFlattenedFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/flattened/RootFlattenedFieldTypeTests.java @@ -22,6 +22,7 @@ import org.elasticsearch.common.lucene.search.AutomatonQueries; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.core.Tuple; +import org.elasticsearch.index.IgnoreAbove; import org.elasticsearch.index.mapper.FieldNamesFieldMapper; import org.elasticsearch.index.mapper.FieldTypeTestCase; import org.elasticsearch.index.mapper.flattened.FlattenedFieldMapper.RootFlattenedFieldType; @@ -34,7 +35,15 @@ public class RootFlattenedFieldTypeTests extends FieldTypeTestCase { private static RootFlattenedFieldType createDefaultFieldType(int ignoreAbove) { - return new RootFlattenedFieldType("field", true, true, Collections.emptyMap(), false, false, ignoreAbove); + return new RootFlattenedFieldType( + "field", + true, + true, + Collections.emptyMap(), + false, + false, + IgnoreAbove.builder().value(ignoreAbove).build() + ); } public void testValueForDisplay() { @@ -61,7 +70,7 @@ public void testTermQuery() { Collections.emptyMap(), false, false, - Integer.MAX_VALUE + IgnoreAbove.IGNORE_ABOVE_STANDARD_INDICES ); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> unsearchable.termQuery("field", null)); assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage()); @@ -75,7 +84,7 @@ public void testExistsQuery() { Collections.emptyMap(), false, false, - Integer.MAX_VALUE + IgnoreAbove.IGNORE_ABOVE_STANDARD_INDICES ); assertEquals(new TermQuery(new Term(FieldNamesFieldMapper.NAME, new BytesRef("field"))), ft.existsQuery(null)); @@ -86,7 +95,7 @@ public void testExistsQuery() { Collections.emptyMap(), false, false, - Integer.MAX_VALUE + IgnoreAbove.IGNORE_ABOVE_STANDARD_INDICES ); assertEquals(new FieldExistsQuery("field"), withDv.existsQuery(null)); } diff --git a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java index 4e792ca2b079b..4a1cfad30a5f4 100644 --- a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java +++ b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java @@ -51,7 +51,8 @@ import org.elasticsearch.common.time.DateMathParser; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.core.Nullable; -import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.IgnoreAbove; +import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; import org.elasticsearch.index.analysis.AnalyzerScope; @@ -70,6 +71,7 @@ import org.elasticsearch.index.mapper.LuceneDocument; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.MapperBuilderContext; +import org.elasticsearch.index.mapper.MappingParserContext; import org.elasticsearch.index.mapper.SourceValueFetcher; import org.elasticsearch.index.mapper.TextSearchInfo; import org.elasticsearch.index.mapper.ValueFetcher; @@ -212,18 +214,27 @@ public static class Builder extends FieldMapper.Builder { final Parameter> meta = Parameter.metaParam(); + final IndexMode indexMode; final IndexVersion indexCreatedVersion; - final int ignoreAboveDefault; public Builder(final String name, IndexVersion indexVersionCreated) { - this(name, IndexSettings.IGNORE_ABOVE_DEFAULT_STANDARD_INDICES, indexVersionCreated); + this(name, IndexMode.STANDARD, indexVersionCreated); } - private Builder(String name, int ignoreAboveDefault, IndexVersion indexCreatedVersion) { + private Builder(String name, MappingParserContext mappingParserContext) { + this(name, mappingParserContext.getIndexSettings().getMode(), mappingParserContext.indexVersionCreated()); + + // if ignore_above is configured at index-level, then set it now + if (IGNORE_ABOVE_SETTING.exists(mappingParserContext.getSettings())) { + this.ignoreAbove.setValue(IGNORE_ABOVE_SETTING.get(mappingParserContext.getSettings())); + } + } + + private Builder(String name, IndexMode indexMode, IndexVersion indexCreatedVersion) { super(name); + this.indexMode = indexMode; this.indexCreatedVersion = indexCreatedVersion; - this.ignoreAboveDefault = ignoreAboveDefault; - this.ignoreAbove = Parameter.ignoreAboveParam(m -> toType(m).ignoreAbove, ignoreAboveDefault); + this.ignoreAbove = Parameter.ignoreAboveParam(m -> toType(m).ignoreAbove.get(), indexMode, indexCreatedVersion); } @Override @@ -248,15 +259,12 @@ public WildcardFieldMapper build(MapperBuilderContext context) { new WildcardFieldType(context.buildFullName(leafName()), indexCreatedVersion, meta.get(), this), context.isSourceSynthetic(), builderParams(this, context), - indexCreatedVersion, this ); } } - public static final TypeParser PARSER = new TypeParser( - (n, c) -> new Builder(n, IGNORE_ABOVE_SETTING.get(c.getSettings()), c.indexVersionCreated()) - ); + public static final TypeParser PARSER = createTypeParserWithLegacySupport(Builder::new); public static final char TOKEN_START_OR_END_CHAR = 0; public static final String TOKEN_START_STRING = Character.toString(TOKEN_START_OR_END_CHAR); @@ -268,7 +276,7 @@ public static final class WildcardFieldType extends MappedFieldType { private final String nullValue; private final NamedAnalyzer analyzer; - private final int ignoreAbove; + private final IgnoreAbove ignoreAbove; private WildcardFieldType(String name, IndexVersion version, Map meta, Builder builder) { super(name, true, false, true, Defaults.TEXT_SEARCH_INFO, meta); @@ -278,7 +286,10 @@ private WildcardFieldType(String name, IndexVersion version, Map this.analyzer = WILDCARD_ANALYZER_7_9; } this.nullValue = builder.nullValue.getValue(); - this.ignoreAbove = builder.ignoreAbove.getValue(); + this.ignoreAbove = IgnoreAbove.builder() + .value(builder.ignoreAbove.getValue()) + .defaultValue(builder.ignoreAbove.getDefaultValue()) + .build(); } @Override @@ -977,7 +988,7 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) @Override protected String parseSourceValue(Object value) { String keywordValue = value.toString(); - if (keywordValue.length() > ignoreAbove) { + if (ignoreAbove.isIgnored(keywordValue)) { return null; } return keywordValue; @@ -996,10 +1007,9 @@ protected String parseSourceValue(Object value) { assert NGRAM_FIELD_TYPE.indexOptions() == IndexOptions.DOCS; } private final String nullValue; + private final IndexMode indexMode; private final IndexVersion indexVersionCreated; - - private final int ignoreAbove; - private final int ignoreAboveDefault; + private final IgnoreAbove ignoreAbove; private final boolean storeIgnored; private final String originalName; @@ -1008,15 +1018,17 @@ private WildcardFieldMapper( WildcardFieldType mappedFieldType, boolean storeIgnored, BuilderParams builderParams, - IndexVersion indexVersionCreated, Builder builder ) { super(simpleName, mappedFieldType, builderParams); this.nullValue = builder.nullValue.getValue(); this.storeIgnored = storeIgnored; - this.indexVersionCreated = indexVersionCreated; - this.ignoreAbove = builder.ignoreAbove.getValue(); - this.ignoreAboveDefault = builder.ignoreAboveDefault; + this.indexMode = builder.indexMode; + this.indexVersionCreated = builder.indexCreatedVersion; + this.ignoreAbove = IgnoreAbove.builder() + .value(builder.ignoreAbove.getValue()) + .defaultValue(builder.ignoreAbove.getDefaultValue()) + .build(); this.originalName = storeIgnored ? fullPath() + "._original" : null; } @@ -1028,7 +1040,7 @@ public Map indexAnalyzers() { /** Values that have more chars than the return value of this method will * be skipped at parsing time. */ // pkg-private for testing - int ignoreAbove() { + IgnoreAbove ignoreAbove() { return ignoreAbove; } @@ -1050,13 +1062,13 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio List fields = new ArrayList<>(); if (value != null) { - if (value.length() <= ignoreAbove) { - createFields(value, parseDoc, fields); - } else { + if (ignoreAbove.isIgnored(value)) { context.addIgnoredField(fullPath()); if (storeIgnored) { parseDoc.add(new StoredField(originalName(), new BytesRef(value))); } + } else { + createFields(value, parseDoc, fields); } } parseDoc.addAll(fields); @@ -1092,7 +1104,7 @@ protected String contentType() { @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(leafName(), ignoreAboveDefault, indexVersionCreated).init(this); + return new Builder(leafName(), indexMode, indexVersionCreated).init(this); } @Override @@ -1100,7 +1112,7 @@ protected SyntheticSourceSupport syntheticSourceSupport() { return new SyntheticSourceSupport.Native(() -> { var layers = new ArrayList(); layers.add(new WildcardSyntheticFieldLoader()); - if (ignoreAbove != Integer.MAX_VALUE) { + if (ignoreAbove.isSet()) { layers.add(new CompositeSyntheticFieldLoader.StoredFieldLayer(originalName()) { @Override protected void writeValue(Object value, XContentBuilder b) throws IOException { From ac7f5bf8f629c3678f4f0039c522a0d0700cec41 Mon Sep 17 00:00:00 2001 From: Dmitry Kubikov Date: Mon, 8 Sep 2025 18:57:50 -0700 Subject: [PATCH 12/23] Fixed test typos --- .../org/elasticsearch/index/IgnoreAbove.java | 2 +- .../index/mapper/KeywordFieldTypeTests.java | 19 +++++++++---------- .../index/mapper/ObjectMapperMergeTests.java | 4 ++-- .../mapper/WildcardFieldMapperTests.java | 2 +- 4 files changed, 13 insertions(+), 14 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/IgnoreAbove.java b/server/src/main/java/org/elasticsearch/index/IgnoreAbove.java index 44da20f2e5556..d2496307444a3 100644 --- a/server/src/main/java/org/elasticsearch/index/IgnoreAbove.java +++ b/server/src/main/java/org/elasticsearch/index/IgnoreAbove.java @@ -29,7 +29,7 @@ public int get() { } public boolean isSet() { - return value != defaultValue; + return value != null && value != defaultValue; } public boolean isIgnored(final String s) { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java index 91848a99dbbd2..4eeb9029b6e19 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java @@ -9,7 +9,6 @@ package org.elasticsearch.index.mapper; import com.carrotsearch.randomizedtesting.generators.RandomStrings; - import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.LowerCaseFilter; import org.apache.lucene.analysis.TokenFilter; @@ -327,11 +326,11 @@ public void test_ignore_above_index_level_setting() { ); // when/then - assertFalse(fieldType.ignoreAbove().isSet()); + assertTrue(fieldType.ignoreAbove().isSet()); assertEquals(123, fieldType.ignoreAbove().get()); } - public void test_isIgnoreAboveSet_returns_true_when_ignore_above_is_given() { + public void test_ignore_above_isSet_returns_true_when_ignore_above_is_given() { // given Settings settings = Settings.builder() .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) @@ -363,7 +362,7 @@ public void test_isIgnoreAboveSet_returns_true_when_ignore_above_is_given() { assertEquals(123, fieldType.ignoreAbove().get()); } - public void test_isIgnoreAboveSet_returns_false_when_ignore_above_is_not_given() { + public void test_ignore_above_isSet_returns_false_when_ignore_above_is_not_given() { // given Settings settings = Settings.builder() .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) @@ -394,7 +393,7 @@ public void test_isIgnoreAboveSet_returns_false_when_ignore_above_is_not_given() assertEquals(IGNORE_ABOVE_DEFAULT_STANDARD_INDICES, fieldType.ignoreAbove().get()); } - public void test_isIgnoreAboveSet_returns_false_when_ignore_above_is_given_but_its_the_same_as_default() { + public void test_ignore_above_isSet_returns_false_when_ignore_above_is_given_but_its_the_same_as_default() { // given Settings settings = Settings.builder() .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) @@ -426,7 +425,7 @@ public void test_isIgnoreAboveSet_returns_false_when_ignore_above_is_given_but_i assertEquals(IGNORE_ABOVE_DEFAULT_STANDARD_INDICES, fieldType.ignoreAbove().get()); } - public void test_isIgnoreAboveSet_returns_false_when_ignore_above_is_given_but_its_the_same_as_default_for_logsdb_indices() { + public void test_ignore_above_isSet_returns_false_when_ignore_above_is_given_but_its_the_same_as_default_for_logsdb_indices() { // given Settings settings = Settings.builder() .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) @@ -458,7 +457,7 @@ public void test_isIgnoreAboveSet_returns_false_when_ignore_above_is_given_but_i assertEquals(IGNORE_ABOVE_DEFAULT_LOGSDB_INDICES, fieldType.ignoreAbove().get()); } - public void test_isIgnoreAboveSet_returns_true_when_ignore_above_is_given_as_logsdb_default_but_index_mod_is_not_logsdb() { + public void test_ignore_above_isSet_returns_true_when_ignore_above_is_given_as_logsdb_default_but_index_mod_is_not_logsdb() { // given Settings settings = Settings.builder() .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) @@ -490,7 +489,7 @@ public void test_isIgnoreAboveSet_returns_true_when_ignore_above_is_given_as_log assertEquals(IGNORE_ABOVE_DEFAULT_LOGSDB_INDICES, fieldType.ignoreAbove().get()); } - public void test_isIgnoreAboveSet_returns_false_when_ignore_above_is_configured_at_index_level() { + public void test_ignore_above_isSet_returns_false_when_ignore_above_is_configured_at_index_level() { // given Settings settings = Settings.builder() .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) @@ -518,11 +517,11 @@ public void test_isIgnoreAboveSet_returns_false_when_ignore_above_is_configured_ ); // when/then - assertFalse(fieldType.ignoreAbove().isSet()); + assertTrue(fieldType.ignoreAbove().isSet()); assertEquals(123, fieldType.ignoreAbove().get()); } - public void test_isIgnoreAboveSet_returns_false_for_non_primary_constructor() { + public void test_ignore_above_isSet_returns_false_for_non_primary_constructor() { // given KeywordFieldType fieldType1 = new KeywordFieldType("field"); KeywordFieldType fieldType2 = new KeywordFieldType("field", mock(FieldType.class)); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperMergeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperMergeTests.java index 7402405f5b883..04e8121fdd90e 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperMergeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperMergeTests.java @@ -263,12 +263,12 @@ public void testMergeSameObjectDifferentFields() { ObjectMapper parent0 = (ObjectMapper) mergedAdd0.getMapper("parent"); assertNotNull(parent0.getMapper("child1")); - assertEquals(42, ((KeywordFieldMapper) parent0.getMapper("child1")).fieldType().ignoreAbove()); + assertEquals(42, ((KeywordFieldMapper) parent0.getMapper("child1")).fieldType().ignoreAbove().get()); assertNull(parent0.getMapper("child2")); ObjectMapper parent1 = (ObjectMapper) mergedAdd1.getMapper("parent"); assertNotNull(parent1.getMapper("child1")); - assertEquals(42, ((KeywordFieldMapper) parent1.getMapper("child1")).fieldType().ignoreAbove()); + assertEquals(42, ((KeywordFieldMapper) parent1.getMapper("child1")).fieldType().ignoreAbove().get()); assertNotNull(parent1.getMapper("child2")); } diff --git a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java index 15a6efdff47c4..9dd2889d22068 100644 --- a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java +++ b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java @@ -721,7 +721,7 @@ protected Object getSampleValueForDocument() { @Override protected void registerParameters(ParameterChecker checker) throws IOException { checker.registerConflictCheck("null_value", b -> b.field("null_value", "foo")); - checker.registerUpdateCheck(b -> b.field("ignore_above", 256), m -> assertEquals(256, ((WildcardFieldMapper) m).ignoreAbove())); + checker.registerUpdateCheck(b -> b.field("ignore_above", 256), m -> assertEquals(256, ((WildcardFieldMapper) m).ignoreAbove().get())); } From 634ca0450d1aaab8001d094f67f917c4e639a412 Mon Sep 17 00:00:00 2001 From: Dmitry Kubikov Date: Mon, 8 Sep 2025 23:18:06 -0700 Subject: [PATCH 13/23] Added IgnoreAboveTest --- .../org/elasticsearch/index/IgnoreAbove.java | 28 ++++++- .../elasticsearch/index/IgnoreAboveTests.java | 76 +++++++++++++++++++ .../index/mapper/KeywordFieldTypeTests.java | 1 + .../mapper/WildcardFieldMapperTests.java | 5 +- 4 files changed, 105 insertions(+), 5 deletions(-) create mode 100644 server/src/test/java/org/elasticsearch/index/IgnoreAboveTests.java diff --git a/server/src/main/java/org/elasticsearch/index/IgnoreAbove.java b/server/src/main/java/org/elasticsearch/index/IgnoreAbove.java index d2496307444a3..c5be15c68f320 100644 --- a/server/src/main/java/org/elasticsearch/index/IgnoreAbove.java +++ b/server/src/main/java/org/elasticsearch/index/IgnoreAbove.java @@ -11,10 +11,15 @@ import org.elasticsearch.xcontent.XContentString; +import java.util.Objects; + import static org.elasticsearch.index.IndexSettings.IGNORE_ABOVE_DEFAULT_LOGSDB_INDICES; import static org.elasticsearch.index.IndexSettings.IGNORE_ABOVE_DEFAULT_STANDARD_INDICES; -public record IgnoreAbove(Integer value, int defaultValue) { +/** + * This class models the ignore_above parameter in indices. + */ +public class IgnoreAbove { public static final IgnoreAbove IGNORE_ABOVE_STANDARD_INDICES = IgnoreAbove.builder() .defaultValue(IGNORE_ABOVE_DEFAULT_STANDARD_INDICES) @@ -24,14 +29,29 @@ public record IgnoreAbove(Integer value, int defaultValue) { .defaultValue(IGNORE_ABOVE_DEFAULT_LOGSDB_INDICES) .build(); + private final Integer value; + private final Integer defaultValue; + + public IgnoreAbove(Integer value, Integer defaultValue) { + this.value = value; + this.defaultValue = Objects.requireNonNull(defaultValue); + } + public int get() { return value != null ? value : defaultValue; } + /** + * Returns whether ignore_above is set; at field or index level. + */ public boolean isSet() { - return value != null && value != defaultValue; + // if ignore_above equals default, its not considered to be set, even if it was explicitly set to the default value + return value != null && value.equals(defaultValue) == false; } + /** + * Returns whether the given string will be ignored. + */ public boolean isIgnored(final String s) { if (s == null) return false; return lengthExceedsIgnoreAbove(s.length()); @@ -53,7 +73,7 @@ public static Builder builder() { public static final class Builder { private Integer value; - private int defaultValue; // cannot be null, hence int + private Integer defaultValue; private Builder() {} @@ -62,7 +82,7 @@ public Builder value(Integer value) { return this; } - public Builder defaultValue(int defaultValue) { + public Builder defaultValue(Integer defaultValue) { this.defaultValue = defaultValue; return this; } diff --git a/server/src/test/java/org/elasticsearch/index/IgnoreAboveTests.java b/server/src/test/java/org/elasticsearch/index/IgnoreAboveTests.java new file mode 100644 index 0000000000000..6869f4a868fc8 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/IgnoreAboveTests.java @@ -0,0 +1,76 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index; + +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xcontent.Text; + +public class IgnoreAboveTests extends ESTestCase { + + public void test_ignore_above_with_value_and_default() { + // given + IgnoreAbove ignoreAbove = IgnoreAbove.builder().value(123).defaultValue(456).build(); + + // when/then + assertEquals(123, ignoreAbove.get()); + assertTrue(ignoreAbove.isSet()); + } + + public void test_ignore_above_with_default_only_is_valid() { + // given + IgnoreAbove ignoreAbove = IgnoreAbove.builder().defaultValue(456).build(); + + // when/then + assertEquals(456, ignoreAbove.get()); + assertFalse(ignoreAbove.isSet()); + } + + public void test_ignore_above_with_same_value_and_default() { + // given + IgnoreAbove ignoreAbove = IgnoreAbove.builder().value(123).defaultValue(123).build(); + + // when/then + assertEquals(123, ignoreAbove.get()); + assertFalse(ignoreAbove.isSet()); + } + + public void test_ignore_above_with_value_only_should_throw() { + // given/when/then + assertThrows(NullPointerException.class, () -> IgnoreAbove.builder().value(123).build()); + } + + public void test_ignore_above_with_nothing_should_throw() { + // given/when/then + assertThrows(NullPointerException.class, () -> IgnoreAbove.builder().build()); + } + + public void test_string_isIgnored() { + // given + IgnoreAbove ignoreAbove = IgnoreAbove.builder().value(10).defaultValue(456).build(); + + // when/then + assertFalse(ignoreAbove.isIgnored("potato")); + assertFalse(ignoreAbove.isIgnored("1234567890")); + assertTrue(ignoreAbove.isIgnored("12345678901")); + assertTrue(ignoreAbove.isIgnored("potato potato tomato tomato")); + } + + public void test_XContentString_isIgnored() { + // given + IgnoreAbove ignoreAbove = IgnoreAbove.builder().value(10).defaultValue(456).build(); + + // when/then + assertFalse(ignoreAbove.isIgnored(new Text("potato"))); + assertFalse(ignoreAbove.isIgnored(new Text("1234567890"))); + assertTrue(ignoreAbove.isIgnored(new Text("12345678901"))); + assertTrue(ignoreAbove.isIgnored(new Text("potato potato tomato tomato"))); + } + +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java index 4eeb9029b6e19..9a7993c2f60e5 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java @@ -9,6 +9,7 @@ package org.elasticsearch.index.mapper; import com.carrotsearch.randomizedtesting.generators.RandomStrings; + import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.LowerCaseFilter; import org.apache.lucene.analysis.TokenFilter; diff --git a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java index 9dd2889d22068..31f228ae6bd7e 100644 --- a/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java +++ b/x-pack/plugin/wildcard/src/test/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapperTests.java @@ -721,7 +721,10 @@ protected Object getSampleValueForDocument() { @Override protected void registerParameters(ParameterChecker checker) throws IOException { checker.registerConflictCheck("null_value", b -> b.field("null_value", "foo")); - checker.registerUpdateCheck(b -> b.field("ignore_above", 256), m -> assertEquals(256, ((WildcardFieldMapper) m).ignoreAbove().get())); + checker.registerUpdateCheck( + b -> b.field("ignore_above", 256), + m -> assertEquals(256, ((WildcardFieldMapper) m).ignoreAbove().get()) + ); } From 7e55ebd349a2b3772052492d59a255af6c145ea3 Mon Sep 17 00:00:00 2001 From: Dmitry Kubikov Date: Tue, 9 Sep 2025 12:36:08 -0700 Subject: [PATCH 14/23] Enforce at least one value or defaultValue to always be non-null when IgnoreAbove is initialized --- .../org/elasticsearch/index/IgnoreAbove.java | 9 ++++++--- .../elasticsearch/index/IgnoreAboveTests.java | 18 +++++++++++------- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/IgnoreAbove.java b/server/src/main/java/org/elasticsearch/index/IgnoreAbove.java index c5be15c68f320..7ba1211afc947 100644 --- a/server/src/main/java/org/elasticsearch/index/IgnoreAbove.java +++ b/server/src/main/java/org/elasticsearch/index/IgnoreAbove.java @@ -11,8 +11,6 @@ import org.elasticsearch.xcontent.XContentString; -import java.util.Objects; - import static org.elasticsearch.index.IndexSettings.IGNORE_ABOVE_DEFAULT_LOGSDB_INDICES; import static org.elasticsearch.index.IndexSettings.IGNORE_ABOVE_DEFAULT_STANDARD_INDICES; @@ -33,8 +31,13 @@ public class IgnoreAbove { private final Integer defaultValue; public IgnoreAbove(Integer value, Integer defaultValue) { + if (value == null && defaultValue == null) { + throw new IllegalArgumentException( + "IgnoreAbove must be initialized with at least one non-null argument: either 'value' or 'defaultValue'" + ); + } this.value = value; - this.defaultValue = Objects.requireNonNull(defaultValue); + this.defaultValue = defaultValue; } public int get() { diff --git a/server/src/test/java/org/elasticsearch/index/IgnoreAboveTests.java b/server/src/test/java/org/elasticsearch/index/IgnoreAboveTests.java index 6869f4a868fc8..d36a785489ff6 100644 --- a/server/src/test/java/org/elasticsearch/index/IgnoreAboveTests.java +++ b/server/src/test/java/org/elasticsearch/index/IgnoreAboveTests.java @@ -23,7 +23,16 @@ public void test_ignore_above_with_value_and_default() { assertTrue(ignoreAbove.isSet()); } - public void test_ignore_above_with_default_only_is_valid() { + public void test_ignore_above_with_value_only() { + // given + IgnoreAbove ignoreAbove = IgnoreAbove.builder().value(123).build(); + + // when/then + assertEquals(123, ignoreAbove.get()); + assertTrue(ignoreAbove.isSet()); + } + + public void test_ignore_above_with_default_only() { // given IgnoreAbove ignoreAbove = IgnoreAbove.builder().defaultValue(456).build(); @@ -41,14 +50,9 @@ public void test_ignore_above_with_same_value_and_default() { assertFalse(ignoreAbove.isSet()); } - public void test_ignore_above_with_value_only_should_throw() { - // given/when/then - assertThrows(NullPointerException.class, () -> IgnoreAbove.builder().value(123).build()); - } - public void test_ignore_above_with_nothing_should_throw() { // given/when/then - assertThrows(NullPointerException.class, () -> IgnoreAbove.builder().build()); + assertThrows(IllegalArgumentException.class, () -> IgnoreAbove.builder().build()); } public void test_string_isIgnored() { From e03dcff4c67fb35c7d5bcd96c529bb2d8f86fac9 Mon Sep 17 00:00:00 2001 From: Dmitry Kubikov Date: Tue, 9 Sep 2025 18:43:52 -0700 Subject: [PATCH 15/23] When initializing IgnoreAbove, dont use defaultValue from builder - this fixes failing BWC test --- .../icu/ICUCollationKeywordFieldMapper.java | 6 +++- .../org/elasticsearch/index/IgnoreAbove.java | 10 +++++- .../index/mapper/FieldMapper.java | 11 +------ .../index/mapper/KeywordFieldMapper.java | 32 ++++++++++++++----- .../flattened/FlattenedFieldMapper.java | 23 +++++-------- .../elasticsearch/index/IgnoreAboveTests.java | 8 +++++ .../index/mapper/ParameterTests.java | 8 ++--- .../wildcard/mapper/WildcardFieldMapper.java | 27 +++++++++------- 8 files changed, 74 insertions(+), 51 deletions(-) diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapper.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapper.java index 8b0183ede9ff0..53e3b08b0cfdd 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapper.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapper.java @@ -25,6 +25,7 @@ import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.core.Nullable; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.fielddata.FieldData; import org.elasticsearch.index.fielddata.FieldDataContext; @@ -250,7 +251,10 @@ public static class Builder extends FieldMapper.Builder { false ).acceptsNull(); - final Parameter ignoreAbove = Parameter.ignoreAboveParam(m -> toType(m).ignoreAbove); + final Parameter ignoreAbove = Parameter.ignoreAboveParam( + m -> toType(m).ignoreAbove, + IndexSettings.IGNORE_ABOVE_DEFAULT_STANDARD_INDICES + ); final Parameter nullValue = Parameter.stringParam("null_value", false, m -> toType(m).nullValue, null).acceptsNull(); public Builder(String name) { diff --git a/server/src/main/java/org/elasticsearch/index/IgnoreAbove.java b/server/src/main/java/org/elasticsearch/index/IgnoreAbove.java index 7ba1211afc947..e170bc5969fd9 100644 --- a/server/src/main/java/org/elasticsearch/index/IgnoreAbove.java +++ b/server/src/main/java/org/elasticsearch/index/IgnoreAbove.java @@ -49,7 +49,7 @@ public int get() { */ public boolean isSet() { // if ignore_above equals default, its not considered to be set, even if it was explicitly set to the default value - return value != null && value.equals(defaultValue) == false; + return Integer.valueOf(get()).equals(defaultValue) == false; } /** @@ -81,11 +81,19 @@ public static final class Builder { private Builder() {} public Builder value(Integer value) { + if (value != null && value < 0) { + throw new IllegalArgumentException("[ignore_above] must be positive, got [" + value + "]"); + } + this.value = value; return this; } public Builder defaultValue(Integer defaultValue) { + if (defaultValue != null && defaultValue < 0) { + throw new IllegalArgumentException("[ignore_above] must be positive, got [" + defaultValue + "]"); + } + this.defaultValue = defaultValue; return this; } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java index 048f5a2254cbe..018cd6ab6a287 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/FieldMapper.java @@ -1337,16 +1337,7 @@ public static Parameter normsParam(Function initi .setMergeValidator((prev, curr, c) -> prev == curr || (prev && curr == false)); } - public static Parameter ignoreAboveParam(Function initializer) { - return ignoreAboveParam(initializer, null, null); - } - - public static Parameter ignoreAboveParam( - Function initializer, - final IndexMode indexMode, - final IndexVersion indexCreatedVersion - ) { - final int defaultValue = IndexSettings.getIgnoreAboveDefaultValue(indexMode, indexCreatedVersion); + public static Parameter ignoreAboveParam(Function initializer, int defaultValue) { return Parameter.intParam("ignore_above", true, initializer, defaultValue).addValidator(v -> { if (v < 0) { throw new IllegalArgumentException("[ignore_above] must be positive, got [" + v + "]"); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 8966b478ba466..e74247281238a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -43,6 +43,7 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.index.IgnoreAbove; import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexSortConfig; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; @@ -92,6 +93,7 @@ import static org.elasticsearch.core.Strings.format; import static org.elasticsearch.index.IndexSettings.IGNORE_ABOVE_SETTING; import static org.elasticsearch.index.IndexSettings.USE_DOC_VALUES_SKIPPER; +import static org.elasticsearch.index.IndexSettings.getIgnoreAboveDefaultValue; import static org.elasticsearch.index.mapper.FieldArrayContext.getOffsetsFieldName; /** @@ -180,6 +182,7 @@ public static final class Builder extends FieldMapper.DimensionBuilder { false ); private final Parameter ignoreAbove; + private final int ignoreAboveDefault; private final IndexSortConfig indexSortConfig; private final IndexMode indexMode; private final Parameter indexOptions = TextParams.keywordIndexOptions(m -> toType(m).indexOptions); @@ -218,6 +221,7 @@ public Builder(final String name, final MappingParserContext mappingParserContex name, mappingParserContext.getIndexAnalyzers(), mappingParserContext.scriptCompiler(), + IGNORE_ABOVE_SETTING.get(mappingParserContext.getSettings()), mappingParserContext.getIndexSettings().getIndexVersionCreated(), mappingParserContext.getIndexSettings().getMode(), mappingParserContext.getIndexSettings().getIndexSortConfig(), @@ -225,11 +229,6 @@ public Builder(final String name, final MappingParserContext mappingParserContex false, mappingParserContext.getIndexSettings().sourceKeepMode() ); - - // if ignore_above is configured at index-level, then set it now - if (IGNORE_ABOVE_SETTING.exists(mappingParserContext.getSettings())) { - this.ignoreAbove.setValue(IGNORE_ABOVE_SETTING.get(mappingParserContext.getSettings())); - } } Builder( @@ -239,13 +238,25 @@ public Builder(final String name, final MappingParserContext mappingParserContex IndexVersion indexCreatedVersion, SourceKeepMode sourceKeepMode ) { - this(name, indexAnalyzers, scriptCompiler, indexCreatedVersion, IndexMode.STANDARD, null, false, false, sourceKeepMode); + this( + name, + indexAnalyzers, + scriptCompiler, + getIgnoreAboveDefaultValue(IndexMode.STANDARD, indexCreatedVersion), + indexCreatedVersion, + IndexMode.STANDARD, + null, + false, + false, + sourceKeepMode + ); } private Builder( String name, IndexAnalyzers indexAnalyzers, ScriptCompiler scriptCompiler, + int ignoreAboveDefault, IndexVersion indexCreatedVersion, IndexMode indexMode, IndexSortConfig indexSortConfig, @@ -279,7 +290,8 @@ private Builder( ); } }).precludesParameters(normalizer); - this.ignoreAbove = Parameter.ignoreAboveParam(m -> toType(m).fieldType().ignoreAbove().get(), indexMode, indexCreatedVersion); + this.ignoreAboveDefault = ignoreAboveDefault; + this.ignoreAbove = Parameter.ignoreAboveParam(m -> toType(m).fieldType().ignoreAbove().get(), ignoreAboveDefault); this.indexSortConfig = indexSortConfig; this.indexMode = indexMode; this.enableDocValuesSkipper = enableDocValuesSkipper; @@ -301,6 +313,7 @@ public static Builder buildWithDocValuesSkipper( name, null, ScriptCompiler.NONE, + IndexSettings.getIgnoreAboveDefaultValue(indexMode, indexCreatedVersion), indexCreatedVersion, indexMode, // Sort config is used to decide if DocValueSkippers can be used. Since skippers are forced, a sort config is not needed. @@ -553,7 +566,7 @@ public KeywordFieldType( this.normalizer = normalizer; this.ignoreAbove = IgnoreAbove.builder() .value(builder.ignoreAbove.getValue()) - .defaultValue(builder.ignoreAbove.getDefaultValue()) + .defaultValue(IndexSettings.getIgnoreAboveDefaultValue(builder.indexMode, builder.indexCreatedVersion)) .build(); this.nullValue = builder.nullValue.getValue(); this.scriptValues = builder.scriptValues(); @@ -1098,6 +1111,7 @@ public String originalName() { private final boolean isSyntheticSource; private final IndexAnalyzers indexAnalyzers; + private final int ignoreAboveDefault; private final IndexMode indexMode; private final IndexSortConfig indexSortConfig; private final boolean enableDocValuesSkipper; @@ -1129,6 +1143,7 @@ private KeywordFieldMapper( this.scriptCompiler = builder.scriptCompiler; this.indexCreatedVersion = builder.indexCreatedVersion; this.isSyntheticSource = isSyntheticSource; + this.ignoreAboveDefault = builder.ignoreAboveDefault; this.indexMode = builder.indexMode; this.indexSortConfig = builder.indexSortConfig; this.enableDocValuesSkipper = builder.enableDocValuesSkipper; @@ -1284,6 +1299,7 @@ public FieldMapper.Builder getMergeBuilder() { leafName(), indexAnalyzers, scriptCompiler, + ignoreAboveDefault, indexCreatedVersion, indexMode, indexSortConfig, diff --git a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java index 2835de6d7edab..c2e94e472e8e8 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java @@ -87,6 +87,7 @@ import java.util.function.Function; import static org.elasticsearch.index.IndexSettings.IGNORE_ABOVE_SETTING; +import static org.elasticsearch.index.IndexSettings.getIgnoreAboveDefaultValue; /** * A field mapper that accepts a JSON object and flattens it into a single field. This data type @@ -153,6 +154,7 @@ public static class Builder extends FieldMapper.Builder { m -> builder(m).eagerGlobalOrdinals.get(), false ); + private final int ignoreAboveDefault; private final Parameter ignoreAbove; private final Parameter indexOptions = TextParams.keywordIndexOptions(m -> builder(m).indexOptions.get()); @@ -180,31 +182,22 @@ public static class Builder extends FieldMapper.Builder { private final Parameter> meta = Parameter.metaParam(); - private final IndexMode indexMode; - private final IndexVersion indexCreatedVersion; - public static FieldMapper.Parameter> dimensionsParam(Function> initializer) { return FieldMapper.Parameter.stringArrayParam(TIME_SERIES_DIMENSIONS_ARRAY_PARAM, false, initializer); } public Builder(final String name) { - this(name, IndexMode.STANDARD, IndexVersion.current()); + this(name, getIgnoreAboveDefaultValue(IndexMode.STANDARD, IndexVersion.current())); } private Builder(String name, MappingParserContext mappingParserContext) { - this(name, mappingParserContext.getIndexSettings().getMode(), mappingParserContext.indexVersionCreated()); - - // if ignore_above is configured at index-level, then set it now - if (IGNORE_ABOVE_SETTING.exists(mappingParserContext.getSettings())) { - this.ignoreAbove.setValue(IGNORE_ABOVE_SETTING.get(mappingParserContext.getSettings())); - } + this(name, IGNORE_ABOVE_SETTING.get(mappingParserContext.getSettings())); } - private Builder(String name, IndexMode indexMode, IndexVersion indexCreatedVersion) { + private Builder(String name, int ignoreAboveDefault) { super(name); - this.indexMode = indexMode; - this.indexCreatedVersion = indexCreatedVersion; - this.ignoreAbove = Parameter.ignoreAboveParam(m -> builder(m).ignoreAbove.get(), indexMode, indexCreatedVersion); + this.ignoreAboveDefault = ignoreAboveDefault; + this.ignoreAbove = Parameter.ignoreAboveParam(m -> builder(m).ignoreAbove.get(), ignoreAboveDefault); this.dimensions.precludesParameters(ignoreAbove); } @@ -905,7 +898,7 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(leafName(), builder.indexMode, builder.indexCreatedVersion).init(this); + return new Builder(leafName(), builder.ignoreAboveDefault).init(this); } @Override diff --git a/server/src/test/java/org/elasticsearch/index/IgnoreAboveTests.java b/server/src/test/java/org/elasticsearch/index/IgnoreAboveTests.java index d36a785489ff6..0374ab878aebb 100644 --- a/server/src/test/java/org/elasticsearch/index/IgnoreAboveTests.java +++ b/server/src/test/java/org/elasticsearch/index/IgnoreAboveTests.java @@ -55,6 +55,14 @@ public void test_ignore_above_with_nothing_should_throw() { assertThrows(IllegalArgumentException.class, () -> IgnoreAbove.builder().build()); } + public void test_negative_value_throws() { + assertThrows(IllegalArgumentException.class, () -> IgnoreAbove.builder().value(-1).build()); + } + + public void test_negative_default_value_throws() { + assertThrows(IllegalArgumentException.class, () -> IgnoreAbove.builder().defaultValue(-1).build()); + } + public void test_string_isIgnored() { // given IgnoreAbove ignoreAbove = IgnoreAbove.builder().value(10).defaultValue(456).build(); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ParameterTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ParameterTests.java index 23ef333fa1e89..d27ec182db7e7 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ParameterTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ParameterTests.java @@ -11,21 +11,19 @@ import org.elasticsearch.test.ESTestCase; -import static org.elasticsearch.index.IndexSettings.IGNORE_ABOVE_DEFAULT_STANDARD_INDICES; - public class ParameterTests extends ESTestCase { public void test_ignore_above_param_default() { // when - FieldMapper.Parameter ignoreAbove = FieldMapper.Parameter.ignoreAboveParam((FieldMapper fm) -> 123); + FieldMapper.Parameter ignoreAbove = FieldMapper.Parameter.ignoreAboveParam((FieldMapper fm) -> 123, 456); // then - assertEquals(IGNORE_ABOVE_DEFAULT_STANDARD_INDICES, ignoreAbove.getValue().intValue()); + assertEquals(456, ignoreAbove.getValue().intValue()); } public void test_ignore_above_param_invalid_value() { // when - FieldMapper.Parameter ignoreAbove = FieldMapper.Parameter.ignoreAboveParam((FieldMapper fm) -> -1); + FieldMapper.Parameter ignoreAbove = FieldMapper.Parameter.ignoreAboveParam((FieldMapper fm) -> -1, 456); ignoreAbove.setValue(-1); // then diff --git a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java index 4a1cfad30a5f4..12574d45ff9f6 100644 --- a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java +++ b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java @@ -95,6 +95,7 @@ import java.util.TreeSet; import static org.elasticsearch.index.IndexSettings.IGNORE_ABOVE_SETTING; +import static org.elasticsearch.index.IndexSettings.getIgnoreAboveDefaultValue; /** * A {@link FieldMapper} for indexing fields with ngrams for efficient wildcard matching @@ -209,6 +210,7 @@ private static WildcardFieldMapper toType(FieldMapper in) { public static class Builder extends FieldMapper.Builder { + final int ignoreAboveDefault; final Parameter ignoreAbove; final Parameter nullValue = Parameter.stringParam("null_value", false, m -> toType(m).nullValue, null).acceptsNull(); @@ -218,23 +220,24 @@ public static class Builder extends FieldMapper.Builder { final IndexVersion indexCreatedVersion; public Builder(final String name, IndexVersion indexVersionCreated) { - this(name, IndexMode.STANDARD, indexVersionCreated); + this(name, getIgnoreAboveDefaultValue(IndexMode.STANDARD, indexVersionCreated), IndexMode.STANDARD, indexVersionCreated); } private Builder(String name, MappingParserContext mappingParserContext) { - this(name, mappingParserContext.getIndexSettings().getMode(), mappingParserContext.indexVersionCreated()); - - // if ignore_above is configured at index-level, then set it now - if (IGNORE_ABOVE_SETTING.exists(mappingParserContext.getSettings())) { - this.ignoreAbove.setValue(IGNORE_ABOVE_SETTING.get(mappingParserContext.getSettings())); - } + this( + name, + IGNORE_ABOVE_SETTING.get(mappingParserContext.getSettings()), + mappingParserContext.getIndexSettings().getMode(), + mappingParserContext.indexVersionCreated() + ); } - private Builder(String name, IndexMode indexMode, IndexVersion indexCreatedVersion) { + private Builder(String name, int ignoreAboveDefault, IndexMode indexMode, IndexVersion indexCreatedVersion) { super(name); + this.ignoreAboveDefault = ignoreAboveDefault; this.indexMode = indexMode; this.indexCreatedVersion = indexCreatedVersion; - this.ignoreAbove = Parameter.ignoreAboveParam(m -> toType(m).ignoreAbove.get(), indexMode, indexCreatedVersion); + this.ignoreAbove = Parameter.ignoreAboveParam(m -> toType(m).ignoreAbove.get(), ignoreAboveDefault); } @Override @@ -1009,6 +1012,7 @@ protected String parseSourceValue(Object value) { private final String nullValue; private final IndexMode indexMode; private final IndexVersion indexVersionCreated; + private final int ignoreAboveDefault; private final IgnoreAbove ignoreAbove; private final boolean storeIgnored; private final String originalName; @@ -1025,9 +1029,10 @@ private WildcardFieldMapper( this.storeIgnored = storeIgnored; this.indexMode = builder.indexMode; this.indexVersionCreated = builder.indexCreatedVersion; + this.ignoreAboveDefault = builder.ignoreAboveDefault; this.ignoreAbove = IgnoreAbove.builder() .value(builder.ignoreAbove.getValue()) - .defaultValue(builder.ignoreAbove.getDefaultValue()) + .defaultValue(getIgnoreAboveDefaultValue(builder.indexMode, builder.indexCreatedVersion)) .build(); this.originalName = storeIgnored ? fullPath() + "._original" : null; } @@ -1104,7 +1109,7 @@ protected String contentType() { @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(leafName(), indexMode, indexVersionCreated).init(this); + return new Builder(leafName(), ignoreAboveDefault, indexMode, indexVersionCreated).init(this); } @Override From 2143b07dce02b0645510ce51a88113af2d8f2ad7 Mon Sep 17 00:00:00 2001 From: Dmitry Kubikov Date: Tue, 9 Sep 2025 19:13:25 -0700 Subject: [PATCH 16/23] Fixed typo --- .../flattened/FlattenedFieldMapper.java | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java index c2e94e472e8e8..4e0e495f61fa2 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java @@ -40,6 +40,7 @@ import org.elasticsearch.core.Nullable; import org.elasticsearch.index.IgnoreAbove; import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.fielddata.FieldData; @@ -182,21 +183,31 @@ public static class Builder extends FieldMapper.Builder { private final Parameter> meta = Parameter.metaParam(); + private final IndexMode indexMode; + private final IndexVersion indexCreatedVersion; + public static FieldMapper.Parameter> dimensionsParam(Function> initializer) { return FieldMapper.Parameter.stringArrayParam(TIME_SERIES_DIMENSIONS_ARRAY_PARAM, false, initializer); } public Builder(final String name) { - this(name, getIgnoreAboveDefaultValue(IndexMode.STANDARD, IndexVersion.current())); + this(name, IndexSettings.IGNORE_ABOVE_DEFAULT_STANDARD_INDICES, IndexMode.STANDARD, IndexVersion.current()); } private Builder(String name, MappingParserContext mappingParserContext) { - this(name, IGNORE_ABOVE_SETTING.get(mappingParserContext.getSettings())); + this( + name, + IGNORE_ABOVE_SETTING.get(mappingParserContext.getSettings()), + mappingParserContext.getIndexSettings().getMode(), + mappingParserContext.indexVersionCreated() + ); } - private Builder(String name, int ignoreAboveDefault) { + private Builder(String name, int ignoreAboveDefault, IndexMode indexMode, IndexVersion indexCreatedVersion) { super(name); this.ignoreAboveDefault = ignoreAboveDefault; + this.indexMode = indexMode; + this.indexCreatedVersion = indexCreatedVersion; this.ignoreAbove = Parameter.ignoreAboveParam(m -> builder(m).ignoreAbove.get(), ignoreAboveDefault); this.dimensions.precludesParameters(ignoreAbove); } @@ -234,7 +245,10 @@ public FlattenedFieldMapper build(MapperBuilderContext context) { splitQueriesOnWhitespace.get(), eagerGlobalOrdinals.get(), dimensions.get(), - IgnoreAbove.builder().value(ignoreAbove.getValue()).defaultValue(ignoreAbove.getDefaultValue()).build() + IgnoreAbove.builder() + .value(ignoreAbove.getValue()) + .defaultValue(getIgnoreAboveDefaultValue(indexMode, indexCreatedVersion)) + .build() ); return new FlattenedFieldMapper(leafName(), ft, builderParams(this, context), this); } @@ -898,7 +912,7 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio @Override public FieldMapper.Builder getMergeBuilder() { - return new Builder(leafName(), builder.ignoreAboveDefault).init(this); + return new Builder(leafName(), builder.ignoreAboveDefault, builder.indexMode, builder.indexCreatedVersion).init(this); } @Override From a7c2fbb27b3c44379d7a59874ae5d491c569acf0 Mon Sep 17 00:00:00 2001 From: Dmitry Kubikov Date: Tue, 9 Sep 2025 20:59:47 -0700 Subject: [PATCH 17/23] Switched IgnoreAbove to constructor only, removed the ability to set default directly --- .../org/elasticsearch/index/IgnoreAbove.java | 60 ++++--------------- .../index/mapper/KeywordFieldMapper.java | 5 +- .../flattened/FlattenedFieldMapper.java | 6 +- .../elasticsearch/index/IgnoreAboveTests.java | 59 ++++++++++++------ .../index/mapper/KeywordFieldTypeTests.java | 2 +- .../index/mapper/TextFieldTypeTests.java | 48 +++++++++++++++ .../RootFlattenedFieldTypeTests.java | 10 +--- .../wildcard/mapper/WildcardFieldMapper.java | 10 +--- 8 files changed, 107 insertions(+), 93 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/IgnoreAbove.java b/server/src/main/java/org/elasticsearch/index/IgnoreAbove.java index e170bc5969fd9..39f4328bdd002 100644 --- a/server/src/main/java/org/elasticsearch/index/IgnoreAbove.java +++ b/server/src/main/java/org/elasticsearch/index/IgnoreAbove.java @@ -11,33 +11,30 @@ import org.elasticsearch.xcontent.XContentString; -import static org.elasticsearch.index.IndexSettings.IGNORE_ABOVE_DEFAULT_LOGSDB_INDICES; -import static org.elasticsearch.index.IndexSettings.IGNORE_ABOVE_DEFAULT_STANDARD_INDICES; +import java.util.Objects; /** * This class models the ignore_above parameter in indices. */ public class IgnoreAbove { - public static final IgnoreAbove IGNORE_ABOVE_STANDARD_INDICES = IgnoreAbove.builder() - .defaultValue(IGNORE_ABOVE_DEFAULT_STANDARD_INDICES) - .build(); - - public static final IgnoreAbove IGNORE_ABOVE_LOGSDB_INDICES = IgnoreAbove.builder() - .defaultValue(IGNORE_ABOVE_DEFAULT_LOGSDB_INDICES) - .build(); + public static final IgnoreAbove IGNORE_ABOVE_STANDARD_INDICES = new IgnoreAbove(null, IndexMode.STANDARD, IndexVersion.current()); + public static final IgnoreAbove IGNORE_ABOVE_LOGSDB_INDICES = new IgnoreAbove(null, IndexMode.LOGSDB, IndexVersion.current()); private final Integer value; private final Integer defaultValue; - public IgnoreAbove(Integer value, Integer defaultValue) { - if (value == null && defaultValue == null) { - throw new IllegalArgumentException( - "IgnoreAbove must be initialized with at least one non-null argument: either 'value' or 'defaultValue'" - ); + public IgnoreAbove(Integer value) { + this(Objects.requireNonNull(value), null, null); + } + + public IgnoreAbove(Integer value, IndexMode indexMode, IndexVersion indexCreatedVersion) { + if (value != null && value < 0) { + throw new IllegalArgumentException("[ignore_above] must be positive, got [" + value + "]"); } + this.value = value; - this.defaultValue = defaultValue; + this.defaultValue = IndexSettings.getIgnoreAboveDefaultValue(indexMode, indexCreatedVersion); } public int get() { @@ -69,37 +66,4 @@ private boolean lengthExceedsIgnoreAbove(int strLength) { return strLength > get(); } - public static Builder builder() { - return new Builder(); - } - - public static final class Builder { - - private Integer value; - private Integer defaultValue; - - private Builder() {} - - public Builder value(Integer value) { - if (value != null && value < 0) { - throw new IllegalArgumentException("[ignore_above] must be positive, got [" + value + "]"); - } - - this.value = value; - return this; - } - - public Builder defaultValue(Integer defaultValue) { - if (defaultValue != null && defaultValue < 0) { - throw new IllegalArgumentException("[ignore_above] must be positive, got [" + defaultValue + "]"); - } - - this.defaultValue = defaultValue; - return this; - } - - public IgnoreAbove build() { - return new IgnoreAbove(value, defaultValue); - } - } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index e74247281238a..94076ebe17f4b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -564,10 +564,7 @@ public KeywordFieldType( ); this.eagerGlobalOrdinals = builder.eagerGlobalOrdinals.getValue(); this.normalizer = normalizer; - this.ignoreAbove = IgnoreAbove.builder() - .value(builder.ignoreAbove.getValue()) - .defaultValue(IndexSettings.getIgnoreAboveDefaultValue(builder.indexMode, builder.indexCreatedVersion)) - .build(); + this.ignoreAbove = new IgnoreAbove(builder.ignoreAbove.getValue(), builder.indexMode, builder.indexCreatedVersion); this.nullValue = builder.nullValue.getValue(); this.scriptValues = builder.scriptValues(); this.isDimension = builder.dimension.getValue(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java index 4e0e495f61fa2..56758dd07161e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java @@ -88,7 +88,6 @@ import java.util.function.Function; import static org.elasticsearch.index.IndexSettings.IGNORE_ABOVE_SETTING; -import static org.elasticsearch.index.IndexSettings.getIgnoreAboveDefaultValue; /** * A field mapper that accepts a JSON object and flattens it into a single field. This data type @@ -245,10 +244,7 @@ public FlattenedFieldMapper build(MapperBuilderContext context) { splitQueriesOnWhitespace.get(), eagerGlobalOrdinals.get(), dimensions.get(), - IgnoreAbove.builder() - .value(ignoreAbove.getValue()) - .defaultValue(getIgnoreAboveDefaultValue(indexMode, indexCreatedVersion)) - .build() + new IgnoreAbove(ignoreAbove.getValue(), indexMode, indexCreatedVersion) ); return new FlattenedFieldMapper(leafName(), ft, builderParams(this, context), this); } diff --git a/server/src/test/java/org/elasticsearch/index/IgnoreAboveTests.java b/server/src/test/java/org/elasticsearch/index/IgnoreAboveTests.java index 0374ab878aebb..fc189dbb19cfc 100644 --- a/server/src/test/java/org/elasticsearch/index/IgnoreAboveTests.java +++ b/server/src/test/java/org/elasticsearch/index/IgnoreAboveTests.java @@ -14,9 +14,9 @@ public class IgnoreAboveTests extends ESTestCase { - public void test_ignore_above_with_value_and_default() { + public void test_ignore_above_with_value_and_index_mode_and_index_version() { // given - IgnoreAbove ignoreAbove = IgnoreAbove.builder().value(123).defaultValue(456).build(); + IgnoreAbove ignoreAbove = new IgnoreAbove(123, IndexMode.STANDARD, IndexVersion.current()); // when/then assertEquals(123, ignoreAbove.get()); @@ -25,47 +25,70 @@ public void test_ignore_above_with_value_and_default() { public void test_ignore_above_with_value_only() { // given - IgnoreAbove ignoreAbove = IgnoreAbove.builder().value(123).build(); + IgnoreAbove ignoreAbove = new IgnoreAbove(123); // when/then assertEquals(123, ignoreAbove.get()); assertTrue(ignoreAbove.isSet()); } - public void test_ignore_above_with_default_only() { + public void test_ignore_above_with_null_value_should_throw() { + assertThrows(NullPointerException.class, () -> new IgnoreAbove(null)); + } + + public void test_ignore_above_with_negative_value_should_throw() { + assertThrows(IllegalArgumentException.class, () -> new IgnoreAbove(-1)); + assertThrows(IllegalArgumentException.class, () -> new IgnoreAbove(-1, IndexMode.STANDARD, IndexVersion.current())); + } + + public void test_ignore_above_with_null_value() { // given - IgnoreAbove ignoreAbove = IgnoreAbove.builder().defaultValue(456).build(); + IgnoreAbove ignoreAbove = new IgnoreAbove(null, IndexMode.STANDARD, IndexVersion.current()); // when/then - assertEquals(456, ignoreAbove.get()); + assertEquals(IndexSettings.IGNORE_ABOVE_DEFAULT_STANDARD_INDICES, ignoreAbove.get()); assertFalse(ignoreAbove.isSet()); } - public void test_ignore_above_with_same_value_and_default() { + public void test_ignore_above_with_null_value_and_logsdb_index_mode() { // given - IgnoreAbove ignoreAbove = IgnoreAbove.builder().value(123).defaultValue(123).build(); + IgnoreAbove ignoreAbove = new IgnoreAbove(null, IndexMode.LOGSDB, IndexVersion.current()); // when/then - assertEquals(123, ignoreAbove.get()); + assertEquals(IndexSettings.IGNORE_ABOVE_DEFAULT_LOGSDB_INDICES, ignoreAbove.get()); assertFalse(ignoreAbove.isSet()); } - public void test_ignore_above_with_nothing_should_throw() { - // given/when/then - assertThrows(IllegalArgumentException.class, () -> IgnoreAbove.builder().build()); + public void test_ignore_above_with_null_everything() { + // given + IgnoreAbove ignoreAbove = new IgnoreAbove(null, null, null); + + // when/then + assertEquals(IndexSettings.IGNORE_ABOVE_DEFAULT_STANDARD_INDICES, ignoreAbove.get()); + assertFalse(ignoreAbove.isSet()); } - public void test_negative_value_throws() { - assertThrows(IllegalArgumentException.class, () -> IgnoreAbove.builder().value(-1).build()); + public void test_ignore_above_default_for_standard_indices() { + // given + IgnoreAbove ignoreAbove = IgnoreAbove.IGNORE_ABOVE_STANDARD_INDICES; + + // when/then + assertEquals(IndexSettings.IGNORE_ABOVE_DEFAULT_STANDARD_INDICES, ignoreAbove.get()); + assertFalse(ignoreAbove.isSet()); } - public void test_negative_default_value_throws() { - assertThrows(IllegalArgumentException.class, () -> IgnoreAbove.builder().defaultValue(-1).build()); + public void test_ignore_above_default_for_logsdb_indices() { + // given + IgnoreAbove ignoreAbove = IgnoreAbove.IGNORE_ABOVE_LOGSDB_INDICES; + + // when/then + assertEquals(IndexSettings.IGNORE_ABOVE_DEFAULT_LOGSDB_INDICES, ignoreAbove.get()); + assertFalse(ignoreAbove.isSet()); } public void test_string_isIgnored() { // given - IgnoreAbove ignoreAbove = IgnoreAbove.builder().value(10).defaultValue(456).build(); + IgnoreAbove ignoreAbove = new IgnoreAbove(10); // when/then assertFalse(ignoreAbove.isIgnored("potato")); @@ -76,7 +99,7 @@ public void test_string_isIgnored() { public void test_XContentString_isIgnored() { // given - IgnoreAbove ignoreAbove = IgnoreAbove.builder().value(10).defaultValue(456).build(); + IgnoreAbove ignoreAbove = new IgnoreAbove(10); // when/then assertFalse(ignoreAbove.isIgnored(new Text("potato"))); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java index 9a7993c2f60e5..f68a3f8b8ae9a 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java @@ -490,7 +490,7 @@ public void test_ignore_above_isSet_returns_true_when_ignore_above_is_given_as_l assertEquals(IGNORE_ABOVE_DEFAULT_LOGSDB_INDICES, fieldType.ignoreAbove().get()); } - public void test_ignore_above_isSet_returns_false_when_ignore_above_is_configured_at_index_level() { + public void test_ignore_above_isSet_returns_true_when_ignore_above_is_configured_at_index_level() { // given Settings settings = Settings.builder() .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) diff --git a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java index 0c7013e198af3..50074c50d51c4 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/TextFieldTypeTests.java @@ -379,4 +379,52 @@ public void test_block_loader_does_not_use_synthetic_source_delegate_when_ignore // verify that we don't delegate anything assertFalse(blockLoader instanceof BlockLoader.Delegating); } + + public void test_block_loader_does_not_use_synthetic_source_delegate_when_ignore_above_is_set_at_index_level() { + // given + Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) + .put(IndexSettings.MODE.getKey(), IndexMode.STANDARD) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + .put(IndexSettings.IGNORE_ABOVE_SETTING.getKey(), 123) + .build(); + IndexSettings indexSettings = new IndexSettings(IndexMetadata.builder("index").settings(settings).build(), settings); + MappingParserContext mappingParserContext = mock(MappingParserContext.class); + doReturn(settings).when(mappingParserContext).getSettings(); + doReturn(indexSettings).when(mappingParserContext).getIndexSettings(); + doReturn(mock(ScriptCompiler.class)).when(mappingParserContext).scriptCompiler(); + + KeywordFieldMapper.Builder builder = new KeywordFieldMapper.Builder("child", mappingParserContext); + + KeywordFieldMapper.KeywordFieldType syntheticSourceDelegate = new KeywordFieldMapper.KeywordFieldType( + "child", + mock(FieldType.class), + mock(NamedAnalyzer.class), + mock(NamedAnalyzer.class), + mock(NamedAnalyzer.class), + builder, + true + ); + + TextFieldType ft = new TextFieldType( + "parent", + true, + false, + new TextSearchInfo(TextFieldMapper.Defaults.FIELD_TYPE, null, Lucene.STANDARD_ANALYZER, Lucene.STANDARD_ANALYZER), + true, + syntheticSourceDelegate, + Collections.singletonMap("potato", "tomato"), + false, + false + ); + + // when + ft.blockLoader(mock(MappedFieldType.BlockLoaderContext.class)); + BlockLoader blockLoader = ft.blockLoader(mock(MappedFieldType.BlockLoaderContext.class)); + + // then + // verify that we don't delegate anything + assertFalse(blockLoader instanceof BlockLoader.Delegating); + } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/flattened/RootFlattenedFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/flattened/RootFlattenedFieldTypeTests.java index 0497bf2fa2a9a..336dabf65d476 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/flattened/RootFlattenedFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/flattened/RootFlattenedFieldTypeTests.java @@ -35,15 +35,7 @@ public class RootFlattenedFieldTypeTests extends FieldTypeTestCase { private static RootFlattenedFieldType createDefaultFieldType(int ignoreAbove) { - return new RootFlattenedFieldType( - "field", - true, - true, - Collections.emptyMap(), - false, - false, - IgnoreAbove.builder().value(ignoreAbove).build() - ); + return new RootFlattenedFieldType("field", true, true, Collections.emptyMap(), false, false, new IgnoreAbove(ignoreAbove)); } public void testValueForDisplay() { diff --git a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java index 12574d45ff9f6..d4239f976c198 100644 --- a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java +++ b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java @@ -289,10 +289,7 @@ private WildcardFieldType(String name, IndexVersion version, Map this.analyzer = WILDCARD_ANALYZER_7_9; } this.nullValue = builder.nullValue.getValue(); - this.ignoreAbove = IgnoreAbove.builder() - .value(builder.ignoreAbove.getValue()) - .defaultValue(builder.ignoreAbove.getDefaultValue()) - .build(); + this.ignoreAbove = new IgnoreAbove(builder.ignoreAbove.getValue(), builder.indexMode, builder.indexCreatedVersion); } @Override @@ -1030,10 +1027,7 @@ private WildcardFieldMapper( this.indexMode = builder.indexMode; this.indexVersionCreated = builder.indexCreatedVersion; this.ignoreAboveDefault = builder.ignoreAboveDefault; - this.ignoreAbove = IgnoreAbove.builder() - .value(builder.ignoreAbove.getValue()) - .defaultValue(getIgnoreAboveDefaultValue(builder.indexMode, builder.indexCreatedVersion)) - .build(); + this.ignoreAbove = new IgnoreAbove(builder.ignoreAbove.getValue(), builder.indexMode, builder.indexCreatedVersion); this.originalName = storeIgnored ? fullPath() + "._original" : null; } From 798b9a52998bdc8adabcf2fe1d57139224f94eb9 Mon Sep 17 00:00:00 2001 From: Dmitry Kubikov Date: Tue, 9 Sep 2025 21:06:38 -0700 Subject: [PATCH 18/23] Update docs/changelog/134253.yaml --- docs/changelog/134253.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/134253.yaml diff --git a/docs/changelog/134253.yaml b/docs/changelog/134253.yaml new file mode 100644 index 0000000000000..5073b28e9c228 --- /dev/null +++ b/docs/changelog/134253.yaml @@ -0,0 +1,5 @@ +pr: 134253 +summary: Fixed text fields block loader +area: Mapping +type: bug +issues: [] From ba7fc0e620a0652387a417b3eafe0ed52bf9e90f Mon Sep 17 00:00:00 2001 From: Dmitry Kubikov Date: Wed, 10 Sep 2025 09:24:07 -0700 Subject: [PATCH 19/23] Update 134253.yaml --- docs/changelog/134253.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog/134253.yaml b/docs/changelog/134253.yaml index 5073b28e9c228..fd12f4c6a83b2 100644 --- a/docs/changelog/134253.yaml +++ b/docs/changelog/134253.yaml @@ -1,5 +1,5 @@ pr: 134253 -summary: Fixed text fields block loader +summary: Fixed a bug where text fields in LogsDB indices did not use their keyword multi fields for block loading area: Mapping type: bug issues: [] From 186776bc0b6e4dfad05bf5f4e65204060e7a00e9 Mon Sep 17 00:00:00 2001 From: Dmitry Kubikov Date: Wed, 10 Sep 2025 13:22:45 -0700 Subject: [PATCH 20/23] Move IgnoreAbove into Mapper and make it final, move everything new out of IndexSettings and into IgnoreAbove --- .../icu/ICUCollationKeywordFieldMapper.java | 3 +- .../org/elasticsearch/index/IgnoreAbove.java | 69 ------------------ .../elasticsearch/index/IndexSettings.java | 23 ++---- .../index/mapper/KeywordFieldMapper.java | 14 ++-- .../elasticsearch/index/mapper/Mapper.java | 71 +++++++++++++++++++ .../flattened/FlattenedFieldMapper.java | 9 ++- .../elasticsearch/index/IgnoreAboveTests.java | 42 ++++++----- .../index/mapper/KeywordFieldTypeTests.java | 16 ++--- .../RootFlattenedFieldTypeTests.java | 30 +++----- .../org/elasticsearch/test/ESTestCase.java | 12 ---- .../upgrades/TextRollingUpgradeIT.java | 13 ++++ .../wildcard/mapper/WildcardFieldMapper.java | 3 +- 12 files changed, 145 insertions(+), 160 deletions(-) delete mode 100644 server/src/main/java/org/elasticsearch/index/IgnoreAbove.java diff --git a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapper.java b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapper.java index 53e3b08b0cfdd..9dd959981701c 100644 --- a/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapper.java +++ b/plugins/analysis-icu/src/main/java/org/elasticsearch/plugin/analysis/icu/ICUCollationKeywordFieldMapper.java @@ -25,7 +25,6 @@ import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.core.Nullable; -import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.fielddata.FieldData; import org.elasticsearch.index.fielddata.FieldDataContext; @@ -253,7 +252,7 @@ public static class Builder extends FieldMapper.Builder { final Parameter ignoreAbove = Parameter.ignoreAboveParam( m -> toType(m).ignoreAbove, - IndexSettings.IGNORE_ABOVE_DEFAULT_STANDARD_INDICES + IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE ); final Parameter nullValue = Parameter.stringParam("null_value", false, m -> toType(m).nullValue, null).acceptsNull(); diff --git a/server/src/main/java/org/elasticsearch/index/IgnoreAbove.java b/server/src/main/java/org/elasticsearch/index/IgnoreAbove.java deleted file mode 100644 index 39f4328bdd002..0000000000000 --- a/server/src/main/java/org/elasticsearch/index/IgnoreAbove.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the "Elastic License - * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side - * Public License v 1"; you may not use this file except in compliance with, at - * your election, the "Elastic License 2.0", the "GNU Affero General Public - * License v3.0 only", or the "Server Side Public License, v 1". - */ - -package org.elasticsearch.index; - -import org.elasticsearch.xcontent.XContentString; - -import java.util.Objects; - -/** - * This class models the ignore_above parameter in indices. - */ -public class IgnoreAbove { - - public static final IgnoreAbove IGNORE_ABOVE_STANDARD_INDICES = new IgnoreAbove(null, IndexMode.STANDARD, IndexVersion.current()); - public static final IgnoreAbove IGNORE_ABOVE_LOGSDB_INDICES = new IgnoreAbove(null, IndexMode.LOGSDB, IndexVersion.current()); - - private final Integer value; - private final Integer defaultValue; - - public IgnoreAbove(Integer value) { - this(Objects.requireNonNull(value), null, null); - } - - public IgnoreAbove(Integer value, IndexMode indexMode, IndexVersion indexCreatedVersion) { - if (value != null && value < 0) { - throw new IllegalArgumentException("[ignore_above] must be positive, got [" + value + "]"); - } - - this.value = value; - this.defaultValue = IndexSettings.getIgnoreAboveDefaultValue(indexMode, indexCreatedVersion); - } - - public int get() { - return value != null ? value : defaultValue; - } - - /** - * Returns whether ignore_above is set; at field or index level. - */ - public boolean isSet() { - // if ignore_above equals default, its not considered to be set, even if it was explicitly set to the default value - return Integer.valueOf(get()).equals(defaultValue) == false; - } - - /** - * Returns whether the given string will be ignored. - */ - public boolean isIgnored(final String s) { - if (s == null) return false; - return lengthExceedsIgnoreAbove(s.length()); - } - - public boolean isIgnored(final XContentString s) { - if (s == null) return false; - return lengthExceedsIgnoreAbove(s.stringLength()); - } - - private boolean lengthExceedsIgnoreAbove(int strLength) { - return strLength > get(); - } - -} diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index 2bb44b26a1716..b396e1ca206e3 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -803,9 +803,6 @@ public Iterator> settings() { * occupy at most 4 bytes. */ - public static final int IGNORE_ABOVE_DEFAULT_STANDARD_INDICES = Integer.MAX_VALUE; - public static final int IGNORE_ABOVE_DEFAULT_LOGSDB_INDICES = 8191; - public static final Setting IGNORE_ABOVE_SETTING = Setting.intSetting( "index.mapping.ignore_above", settings -> String.valueOf(getIgnoreAboveDefaultValue(settings)), @@ -817,22 +814,12 @@ public Iterator> settings() { private static int getIgnoreAboveDefaultValue(final Settings settings) { if (settings == null) { - return IGNORE_ABOVE_DEFAULT_STANDARD_INDICES; - } - return getIgnoreAboveDefaultValue(IndexSettings.MODE.get(settings), IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings)); - } - - public static int getIgnoreAboveDefaultValue(final IndexMode indexMode, final IndexVersion indexCreatedVersion) { - if (diffIgnoreAboveDefaultForLogs(indexMode, indexCreatedVersion)) { - return IGNORE_ABOVE_DEFAULT_LOGSDB_INDICES; - } else { - return IGNORE_ABOVE_DEFAULT_STANDARD_INDICES; + return Mapper.IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE; } - } - - private static boolean diffIgnoreAboveDefaultForLogs(final IndexMode indexMode, final IndexVersion indexCreatedVersion) { - return indexMode == IndexMode.LOGSDB - && (indexCreatedVersion != null && indexCreatedVersion.onOrAfter(IndexVersions.ENABLE_IGNORE_ABOVE_LOGSDB)); + return Mapper.IgnoreAbove.getIgnoreAboveDefaultValue( + IndexSettings.MODE.get(settings), + IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings) + ); } public static final Setting SEQ_NO_INDEX_OPTIONS_SETTING = Setting.enumSetting( diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 94076ebe17f4b..b989a03ba09cf 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -41,9 +41,7 @@ import org.elasticsearch.common.lucene.search.AutomatonQueries; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.core.Nullable; -import org.elasticsearch.index.IgnoreAbove; import org.elasticsearch.index.IndexMode; -import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexSortConfig; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; @@ -93,8 +91,8 @@ import static org.elasticsearch.core.Strings.format; import static org.elasticsearch.index.IndexSettings.IGNORE_ABOVE_SETTING; import static org.elasticsearch.index.IndexSettings.USE_DOC_VALUES_SKIPPER; -import static org.elasticsearch.index.IndexSettings.getIgnoreAboveDefaultValue; import static org.elasticsearch.index.mapper.FieldArrayContext.getOffsetsFieldName; +import static org.elasticsearch.index.mapper.Mapper.IgnoreAbove.getIgnoreAboveDefaultValue; /** * A field mapper for keywords. This mapper accepts strings and indexes them as-is. @@ -313,7 +311,7 @@ public static Builder buildWithDocValuesSkipper( name, null, ScriptCompiler.NONE, - IndexSettings.getIgnoreAboveDefaultValue(indexMode, indexCreatedVersion), + getIgnoreAboveDefaultValue(indexMode, indexCreatedVersion), indexCreatedVersion, indexMode, // Sort config is used to decide if DocValueSkippers can be used. Since skippers are forced, a sort config is not needed. @@ -534,6 +532,8 @@ private static boolean indexSortConfigByHostName(final IndexSortConfig indexSort public static final class KeywordFieldType extends StringFieldType { + private static final IgnoreAbove IGNORE_ABOVE_DEFAULT = new IgnoreAbove(null, IndexMode.STANDARD, IndexVersion.current()); + private final IgnoreAbove ignoreAbove; private final String nullValue; private final NamedAnalyzer normalizer; @@ -581,7 +581,7 @@ public KeywordFieldType(String name) { public KeywordFieldType(String name, boolean isIndexed, boolean hasDocValues, Map meta) { super(name, isIndexed, false, hasDocValues, TextSearchInfo.SIMPLE_MATCH_ONLY, meta); this.normalizer = Lucene.KEYWORD_ANALYZER; - this.ignoreAbove = IgnoreAbove.IGNORE_ABOVE_STANDARD_INDICES; + this.ignoreAbove = IGNORE_ABOVE_DEFAULT; this.nullValue = null; this.eagerGlobalOrdinals = false; this.scriptValues = null; @@ -602,7 +602,7 @@ public KeywordFieldType(String name, FieldType fieldType) { Collections.emptyMap() ); this.normalizer = Lucene.KEYWORD_ANALYZER; - this.ignoreAbove = IgnoreAbove.IGNORE_ABOVE_STANDARD_INDICES; + this.ignoreAbove = IGNORE_ABOVE_DEFAULT; this.nullValue = null; this.eagerGlobalOrdinals = false; this.scriptValues = null; @@ -616,7 +616,7 @@ public KeywordFieldType(String name, FieldType fieldType) { public KeywordFieldType(String name, NamedAnalyzer analyzer) { super(name, true, false, true, textSearchInfo(Defaults.FIELD_TYPE, null, analyzer, analyzer), Collections.emptyMap()); this.normalizer = Lucene.KEYWORD_ANALYZER; - this.ignoreAbove = IgnoreAbove.IGNORE_ABOVE_STANDARD_INDICES; + this.ignoreAbove = IGNORE_ABOVE_DEFAULT; this.nullValue = null; this.eagerGlobalOrdinals = false; this.scriptValues = null; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java index cf3261d88bf10..5824c87d0b1a5 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java @@ -19,6 +19,7 @@ import org.elasticsearch.index.IndexVersions; import org.elasticsearch.xcontent.ToXContentFragment; import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentString; import java.io.IOException; import java.util.Arrays; @@ -131,6 +132,76 @@ default boolean supportsVersion(IndexVersion indexCreatedVersion) { } } + /** + * This class models the ignore_above parameter in indices. + */ + public static final class IgnoreAbove { + + public static final int I + + public static final int IGNORE_ABOVE_DEFAULT_VALUE = Integer.MAX_VALUE; + public static final int IGNORE_ABOVE_DEFAULT_VALUE_FOR_LOGSDB_INDICES = 8191; + + private final Integer value; + private final Integer defaultValue; + + public IgnoreAbove(Integer value) { + this(Objects.requireNonNull(value), null, null); + } + + public IgnoreAbove(Integer value, IndexMode indexMode, IndexVersion indexCreatedVersion) { + if (value != null && value < 0) { + throw new IllegalArgumentException("[ignore_above] must be positive, got [" + value + "]"); + } + + this.value = value; + this.defaultValue = getIgnoreAboveDefaultValue(indexMode, indexCreatedVersion); + } + + public int get() { + return value != null ? value : defaultValue; + } + + /** + * Returns whether ignore_above is set; at field or index level. + */ + public boolean isSet() { + // if ignore_above equals default, its not considered to be set, even if it was explicitly set to the default value + return Integer.valueOf(get()).equals(defaultValue) == false; + } + + /** + * Returns whether the given string will be ignored. + */ + public boolean isIgnored(final String s) { + if (s == null) return false; + return lengthExceedsIgnoreAbove(s.length()); + } + + public boolean isIgnored(final XContentString s) { + if (s == null) return false; + return lengthExceedsIgnoreAbove(s.stringLength()); + } + + private boolean lengthExceedsIgnoreAbove(int strLength) { + return strLength > get(); + } + + public static int getIgnoreAboveDefaultValue(final IndexMode indexMode, final IndexVersion indexCreatedVersion) { + if (diffIgnoreAboveDefaultForLogs(indexMode, indexCreatedVersion)) { + return IGNORE_ABOVE_DEFAULT_VALUE_FOR_LOGSDB_INDICES; + } else { + return IGNORE_ABOVE_DEFAULT_VALUE; + } + } + + private static boolean diffIgnoreAboveDefaultForLogs(final IndexMode indexMode, final IndexVersion indexCreatedVersion) { + return indexMode == IndexMode.LOGSDB + && (indexCreatedVersion != null && indexCreatedVersion.onOrAfter(IndexVersions.ENABLE_IGNORE_ABOVE_LOGSDB)); + } + + } + private final String leafName; @SuppressWarnings("this-escape") diff --git a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java index 56758dd07161e..9b818570ba543 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java @@ -38,9 +38,7 @@ import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.core.Nullable; -import org.elasticsearch.index.IgnoreAbove; import org.elasticsearch.index.IndexMode; -import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.fielddata.FieldData; @@ -190,7 +188,12 @@ public static FieldMapper.Parameter> dimensionsParam(Function new IgnoreAbove(null)); + assertThrows(NullPointerException.class, () -> new Mapper.IgnoreAbove(null)); } public void test_ignore_above_with_negative_value_should_throw() { - assertThrows(IllegalArgumentException.class, () -> new IgnoreAbove(-1)); - assertThrows(IllegalArgumentException.class, () -> new IgnoreAbove(-1, IndexMode.STANDARD, IndexVersion.current())); + assertThrows(IllegalArgumentException.class, () -> new Mapper.IgnoreAbove(-1)); + assertThrows(IllegalArgumentException.class, () -> new Mapper.IgnoreAbove(-1, IndexMode.STANDARD, IndexVersion.current())); } public void test_ignore_above_with_null_value() { // given - IgnoreAbove ignoreAbove = new IgnoreAbove(null, IndexMode.STANDARD, IndexVersion.current()); + Mapper.IgnoreAbove ignoreAbove = new Mapper.IgnoreAbove(null, IndexMode.STANDARD, IndexVersion.current()); // when/then - assertEquals(IndexSettings.IGNORE_ABOVE_DEFAULT_STANDARD_INDICES, ignoreAbove.get()); + assertEquals(Mapper.IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE, ignoreAbove.get()); assertFalse(ignoreAbove.isSet()); } public void test_ignore_above_with_null_value_and_logsdb_index_mode() { // given - IgnoreAbove ignoreAbove = new IgnoreAbove(null, IndexMode.LOGSDB, IndexVersion.current()); + Mapper.IgnoreAbove ignoreAbove = new Mapper.IgnoreAbove(null, IndexMode.LOGSDB, IndexVersion.current()); // when/then - assertEquals(IndexSettings.IGNORE_ABOVE_DEFAULT_LOGSDB_INDICES, ignoreAbove.get()); + assertEquals(Mapper.IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE_FOR_LOGSDB_INDICES, ignoreAbove.get()); assertFalse(ignoreAbove.isSet()); } public void test_ignore_above_with_null_everything() { // given - IgnoreAbove ignoreAbove = new IgnoreAbove(null, null, null); + Mapper.IgnoreAbove ignoreAbove = new Mapper.IgnoreAbove(null, null, null); // when/then - assertEquals(IndexSettings.IGNORE_ABOVE_DEFAULT_STANDARD_INDICES, ignoreAbove.get()); + assertEquals(Mapper.IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE, ignoreAbove.get()); assertFalse(ignoreAbove.isSet()); } public void test_ignore_above_default_for_standard_indices() { // given - IgnoreAbove ignoreAbove = IgnoreAbove.IGNORE_ABOVE_STANDARD_INDICES; + Mapper.IgnoreAbove ignoreAbove = IGNORE_ABOVE_DEFAULT; // when/then - assertEquals(IndexSettings.IGNORE_ABOVE_DEFAULT_STANDARD_INDICES, ignoreAbove.get()); + assertEquals(Mapper.IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE, ignoreAbove.get()); assertFalse(ignoreAbove.isSet()); } public void test_ignore_above_default_for_logsdb_indices() { // given - IgnoreAbove ignoreAbove = IgnoreAbove.IGNORE_ABOVE_LOGSDB_INDICES; + Mapper.IgnoreAbove ignoreAbove = IGNORE_ABOVE_DEFAULT_LOGS; // when/then - assertEquals(IndexSettings.IGNORE_ABOVE_DEFAULT_LOGSDB_INDICES, ignoreAbove.get()); + assertEquals(Mapper.IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE_FOR_LOGSDB_INDICES, ignoreAbove.get()); assertFalse(ignoreAbove.isSet()); } public void test_string_isIgnored() { // given - IgnoreAbove ignoreAbove = new IgnoreAbove(10); + Mapper.IgnoreAbove ignoreAbove = new Mapper.IgnoreAbove(10); // when/then assertFalse(ignoreAbove.isIgnored("potato")); @@ -99,7 +107,7 @@ public void test_string_isIgnored() { public void test_XContentString_isIgnored() { // given - IgnoreAbove ignoreAbove = new IgnoreAbove(10); + Mapper.IgnoreAbove ignoreAbove = new Mapper.IgnoreAbove(10); // when/then assertFalse(ignoreAbove.isIgnored(new Text("potato"))); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java index f68a3f8b8ae9a..7cc886d45dc6b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/KeywordFieldTypeTests.java @@ -62,8 +62,6 @@ import java.util.List; import java.util.Map; -import static org.elasticsearch.index.IndexSettings.IGNORE_ABOVE_DEFAULT_LOGSDB_INDICES; -import static org.elasticsearch.index.IndexSettings.IGNORE_ABOVE_DEFAULT_STANDARD_INDICES; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; @@ -391,7 +389,7 @@ public void test_ignore_above_isSet_returns_false_when_ignore_above_is_not_given // when/then assertFalse(fieldType.ignoreAbove().isSet()); - assertEquals(IGNORE_ABOVE_DEFAULT_STANDARD_INDICES, fieldType.ignoreAbove().get()); + assertEquals(Mapper.IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE, fieldType.ignoreAbove().get()); } public void test_ignore_above_isSet_returns_false_when_ignore_above_is_given_but_its_the_same_as_default() { @@ -409,7 +407,7 @@ public void test_ignore_above_isSet_returns_false_when_ignore_above_is_given_but doReturn(mock(ScriptCompiler.class)).when(mappingParserContext).scriptCompiler(); KeywordFieldMapper.Builder builder = new KeywordFieldMapper.Builder("field", mappingParserContext); - builder.ignoreAbove(IGNORE_ABOVE_DEFAULT_STANDARD_INDICES); + builder.ignoreAbove(Mapper.IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE); KeywordFieldMapper.KeywordFieldType fieldType = new KeywordFieldMapper.KeywordFieldType( "field", @@ -423,7 +421,7 @@ public void test_ignore_above_isSet_returns_false_when_ignore_above_is_given_but // when/then assertFalse(fieldType.ignoreAbove().isSet()); - assertEquals(IGNORE_ABOVE_DEFAULT_STANDARD_INDICES, fieldType.ignoreAbove().get()); + assertEquals(Mapper.IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE, fieldType.ignoreAbove().get()); } public void test_ignore_above_isSet_returns_false_when_ignore_above_is_given_but_its_the_same_as_default_for_logsdb_indices() { @@ -441,7 +439,7 @@ public void test_ignore_above_isSet_returns_false_when_ignore_above_is_given_but doReturn(mock(ScriptCompiler.class)).when(mappingParserContext).scriptCompiler(); KeywordFieldMapper.Builder builder = new KeywordFieldMapper.Builder("field", mappingParserContext); - builder.ignoreAbove(IGNORE_ABOVE_DEFAULT_LOGSDB_INDICES); + builder.ignoreAbove(Mapper.IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE_FOR_LOGSDB_INDICES); KeywordFieldMapper.KeywordFieldType fieldType = new KeywordFieldMapper.KeywordFieldType( "field", @@ -455,7 +453,7 @@ public void test_ignore_above_isSet_returns_false_when_ignore_above_is_given_but // when/then assertFalse(fieldType.ignoreAbove().isSet()); - assertEquals(IGNORE_ABOVE_DEFAULT_LOGSDB_INDICES, fieldType.ignoreAbove().get()); + assertEquals(Mapper.IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE_FOR_LOGSDB_INDICES, fieldType.ignoreAbove().get()); } public void test_ignore_above_isSet_returns_true_when_ignore_above_is_given_as_logsdb_default_but_index_mod_is_not_logsdb() { @@ -473,7 +471,7 @@ public void test_ignore_above_isSet_returns_true_when_ignore_above_is_given_as_l doReturn(mock(ScriptCompiler.class)).when(mappingParserContext).scriptCompiler(); KeywordFieldMapper.Builder builder = new KeywordFieldMapper.Builder("field", mappingParserContext); - builder.ignoreAbove(IGNORE_ABOVE_DEFAULT_LOGSDB_INDICES); + builder.ignoreAbove(Mapper.IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE_FOR_LOGSDB_INDICES); KeywordFieldMapper.KeywordFieldType fieldType = new KeywordFieldMapper.KeywordFieldType( "field", @@ -487,7 +485,7 @@ public void test_ignore_above_isSet_returns_true_when_ignore_above_is_given_as_l // when/then assertTrue(fieldType.ignoreAbove().isSet()); - assertEquals(IGNORE_ABOVE_DEFAULT_LOGSDB_INDICES, fieldType.ignoreAbove().get()); + assertEquals(Mapper.IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE_FOR_LOGSDB_INDICES, fieldType.ignoreAbove().get()); } public void test_ignore_above_isSet_returns_true_when_ignore_above_is_configured_at_index_level() { diff --git a/server/src/test/java/org/elasticsearch/index/mapper/flattened/RootFlattenedFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/flattened/RootFlattenedFieldTypeTests.java index 336dabf65d476..c462684e7e2d1 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/flattened/RootFlattenedFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/flattened/RootFlattenedFieldTypeTests.java @@ -22,9 +22,11 @@ import org.elasticsearch.common.lucene.search.AutomatonQueries; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.core.Tuple; -import org.elasticsearch.index.IgnoreAbove; +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.mapper.FieldNamesFieldMapper; import org.elasticsearch.index.mapper.FieldTypeTestCase; +import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.flattened.FlattenedFieldMapper.RootFlattenedFieldType; import java.io.IOException; @@ -34,8 +36,10 @@ public class RootFlattenedFieldTypeTests extends FieldTypeTestCase { + private static final Mapper.IgnoreAbove IGNORE_ABOVE = new Mapper.IgnoreAbove(null, IndexMode.STANDARD, IndexVersion.current()); + private static RootFlattenedFieldType createDefaultFieldType(int ignoreAbove) { - return new RootFlattenedFieldType("field", true, true, Collections.emptyMap(), false, false, new IgnoreAbove(ignoreAbove)); + return new RootFlattenedFieldType("field", true, true, Collections.emptyMap(), false, false, new Mapper.IgnoreAbove(ignoreAbove)); } public void testValueForDisplay() { @@ -62,33 +66,17 @@ public void testTermQuery() { Collections.emptyMap(), false, false, - IgnoreAbove.IGNORE_ABOVE_STANDARD_INDICES + IGNORE_ABOVE ); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> unsearchable.termQuery("field", null)); assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage()); } public void testExistsQuery() { - RootFlattenedFieldType ft = new RootFlattenedFieldType( - "field", - true, - false, - Collections.emptyMap(), - false, - false, - IgnoreAbove.IGNORE_ABOVE_STANDARD_INDICES - ); + RootFlattenedFieldType ft = new RootFlattenedFieldType("field", true, false, Collections.emptyMap(), false, false, IGNORE_ABOVE); assertEquals(new TermQuery(new Term(FieldNamesFieldMapper.NAME, new BytesRef("field"))), ft.existsQuery(null)); - RootFlattenedFieldType withDv = new RootFlattenedFieldType( - "field", - true, - true, - Collections.emptyMap(), - false, - false, - IgnoreAbove.IGNORE_ABOVE_STANDARD_INDICES - ); + RootFlattenedFieldType withDv = new RootFlattenedFieldType("field", true, true, Collections.emptyMap(), false, false, IGNORE_ABOVE); assertEquals(new FieldExistsQuery("field"), withDv.existsQuery(null)); } diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java index 5b35d736ed314..43243e431db48 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java @@ -1242,18 +1242,6 @@ public static String randomAlphaOfLength(int codeUnits) { return RandomizedTest.randomAsciiOfLength(codeUnits); } - /** - * Generates a string containing a random number of random length alphas, all delimited by space. - */ - public static String randomAlphasDelimitedBySpace(int maxAlphas, int minCodeUnits, int maxCodeUnits) { - int numAlphas = randomIntBetween(1, maxAlphas); - List alphas = new ArrayList<>(numAlphas); - for (int i = 0; i < numAlphas; i++) { - alphas.add(randomAlphaOfLengthBetween(minCodeUnits, maxCodeUnits)); - } - return String.join(" ", alphas); - } - /** * Generate a random string containing only alphanumeric characters. * The locale for the string is {@link Locale#ROOT}. diff --git a/x-pack/plugin/logsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/TextRollingUpgradeIT.java b/x-pack/plugin/logsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/TextRollingUpgradeIT.java index c7b2b99dc7632..b99c1a69fab0a 100644 --- a/x-pack/plugin/logsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/TextRollingUpgradeIT.java +++ b/x-pack/plugin/logsdb/qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/TextRollingUpgradeIT.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.io.InputStream; import java.time.Instant; +import java.util.ArrayList; import java.util.List; import java.util.Locale; import java.util.Map; @@ -213,6 +214,18 @@ private String bulkIndexRequestBody(int numDocs, Instant startTime) { return requestBody.toString(); } + /** + * Generates a string containing a random number of random length alphas, all delimited by space. + */ + public static String randomAlphasDelimitedBySpace(int maxAlphas, int minCodeUnits, int maxCodeUnits) { + int numAlphas = randomIntBetween(1, maxAlphas); + List alphas = new ArrayList<>(numAlphas); + for (int i = 0; i < numAlphas; i++) { + alphas.add(randomAlphaOfLengthBetween(minCodeUnits, maxCodeUnits)); + } + return String.join(" ", alphas); + } + private void recordSmallestMessage(final String message) { if (smallestMessage == null || message.compareTo(smallestMessage) < 0) { smallestMessage = message; diff --git a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java index d4239f976c198..874ea05c00e34 100644 --- a/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java +++ b/x-pack/plugin/wildcard/src/main/java/org/elasticsearch/xpack/wildcard/mapper/WildcardFieldMapper.java @@ -51,7 +51,6 @@ import org.elasticsearch.common.time.DateMathParser; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.core.Nullable; -import org.elasticsearch.index.IgnoreAbove; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; @@ -95,7 +94,7 @@ import java.util.TreeSet; import static org.elasticsearch.index.IndexSettings.IGNORE_ABOVE_SETTING; -import static org.elasticsearch.index.IndexSettings.getIgnoreAboveDefaultValue; +import static org.elasticsearch.index.mapper.Mapper.IgnoreAbove.getIgnoreAboveDefaultValue; /** * A {@link FieldMapper} for indexing fields with ngrams for efficient wildcard matching From 44d9ce881cb82945995c8f74c9ca10432ec6b11b Mon Sep 17 00:00:00 2001 From: Dmitry Kubikov Date: Wed, 10 Sep 2025 13:28:41 -0700 Subject: [PATCH 21/23] Fixed typo --- server/src/main/java/org/elasticsearch/index/mapper/Mapper.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java index 5824c87d0b1a5..9dd8069155a5a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java @@ -137,8 +137,6 @@ default boolean supportsVersion(IndexVersion indexCreatedVersion) { */ public static final class IgnoreAbove { - public static final int I - public static final int IGNORE_ABOVE_DEFAULT_VALUE = Integer.MAX_VALUE; public static final int IGNORE_ABOVE_DEFAULT_VALUE_FOR_LOGSDB_INDICES = 8191; From 42c82a951e85c349f83d1771438deab4de013298 Mon Sep 17 00:00:00 2001 From: Dmitry Kubikov Date: Wed, 10 Sep 2025 13:55:25 -0700 Subject: [PATCH 22/23] Added helpful constructor to IgnoreAbove --- .../index/mapper/KeywordFieldMapper.java | 2 +- .../org/elasticsearch/index/mapper/Mapper.java | 6 +++++- .../elasticsearch/index/IgnoreAboveTests.java | 16 ++++++---------- .../flattened/RootFlattenedFieldTypeTests.java | 3 +-- 4 files changed, 13 insertions(+), 14 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index b989a03ba09cf..f730e5188f55d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -532,7 +532,7 @@ private static boolean indexSortConfigByHostName(final IndexSortConfig indexSort public static final class KeywordFieldType extends StringFieldType { - private static final IgnoreAbove IGNORE_ABOVE_DEFAULT = new IgnoreAbove(null, IndexMode.STANDARD, IndexVersion.current()); + private static final IgnoreAbove IGNORE_ABOVE_DEFAULT = new IgnoreAbove(null, IndexMode.STANDARD); private final IgnoreAbove ignoreAbove; private final String nullValue; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java index 9dd8069155a5a..1453d8fcb65a2 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/Mapper.java @@ -144,7 +144,11 @@ public static final class IgnoreAbove { private final Integer defaultValue; public IgnoreAbove(Integer value) { - this(Objects.requireNonNull(value), null, null); + this(Objects.requireNonNull(value), IndexMode.STANDARD, IndexVersion.current()); + } + + public IgnoreAbove(Integer value, IndexMode indexMode) { + this(value, indexMode, IndexVersion.current()); } public IgnoreAbove(Integer value, IndexMode indexMode, IndexVersion indexCreatedVersion) { diff --git a/server/src/test/java/org/elasticsearch/index/IgnoreAboveTests.java b/server/src/test/java/org/elasticsearch/index/IgnoreAboveTests.java index 925179d2b617c..1b384f36cb4d7 100644 --- a/server/src/test/java/org/elasticsearch/index/IgnoreAboveTests.java +++ b/server/src/test/java/org/elasticsearch/index/IgnoreAboveTests.java @@ -15,16 +15,12 @@ public class IgnoreAboveTests extends ESTestCase { - private static final Mapper.IgnoreAbove IGNORE_ABOVE_DEFAULT = new Mapper.IgnoreAbove(null, IndexMode.STANDARD, IndexVersion.current()); - private static final Mapper.IgnoreAbove IGNORE_ABOVE_DEFAULT_LOGS = new Mapper.IgnoreAbove( - null, - IndexMode.LOGSDB, - IndexVersion.current() - ); + private static final Mapper.IgnoreAbove IGNORE_ABOVE_DEFAULT = new Mapper.IgnoreAbove(null, IndexMode.STANDARD); + private static final Mapper.IgnoreAbove IGNORE_ABOVE_DEFAULT_LOGS = new Mapper.IgnoreAbove(null, IndexMode.LOGSDB); public void test_ignore_above_with_value_and_index_mode_and_index_version() { // given - Mapper.IgnoreAbove ignoreAbove = new Mapper.IgnoreAbove(123, IndexMode.STANDARD, IndexVersion.current()); + Mapper.IgnoreAbove ignoreAbove = new Mapper.IgnoreAbove(123, IndexMode.STANDARD); // when/then assertEquals(123, ignoreAbove.get()); @@ -46,12 +42,12 @@ public void test_ignore_above_with_null_value_should_throw() { public void test_ignore_above_with_negative_value_should_throw() { assertThrows(IllegalArgumentException.class, () -> new Mapper.IgnoreAbove(-1)); - assertThrows(IllegalArgumentException.class, () -> new Mapper.IgnoreAbove(-1, IndexMode.STANDARD, IndexVersion.current())); + assertThrows(IllegalArgumentException.class, () -> new Mapper.IgnoreAbove(-1, IndexMode.STANDARD)); } public void test_ignore_above_with_null_value() { // given - Mapper.IgnoreAbove ignoreAbove = new Mapper.IgnoreAbove(null, IndexMode.STANDARD, IndexVersion.current()); + Mapper.IgnoreAbove ignoreAbove = new Mapper.IgnoreAbove(null, IndexMode.STANDARD); // when/then assertEquals(Mapper.IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE, ignoreAbove.get()); @@ -60,7 +56,7 @@ public void test_ignore_above_with_null_value() { public void test_ignore_above_with_null_value_and_logsdb_index_mode() { // given - Mapper.IgnoreAbove ignoreAbove = new Mapper.IgnoreAbove(null, IndexMode.LOGSDB, IndexVersion.current()); + Mapper.IgnoreAbove ignoreAbove = new Mapper.IgnoreAbove(null, IndexMode.LOGSDB); // when/then assertEquals(Mapper.IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE_FOR_LOGSDB_INDICES, ignoreAbove.get()); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/flattened/RootFlattenedFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/flattened/RootFlattenedFieldTypeTests.java index c462684e7e2d1..3127b2c60d0f5 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/flattened/RootFlattenedFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/flattened/RootFlattenedFieldTypeTests.java @@ -23,7 +23,6 @@ import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.core.Tuple; import org.elasticsearch.index.IndexMode; -import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.mapper.FieldNamesFieldMapper; import org.elasticsearch.index.mapper.FieldTypeTestCase; import org.elasticsearch.index.mapper.Mapper; @@ -36,7 +35,7 @@ public class RootFlattenedFieldTypeTests extends FieldTypeTestCase { - private static final Mapper.IgnoreAbove IGNORE_ABOVE = new Mapper.IgnoreAbove(null, IndexMode.STANDARD, IndexVersion.current()); + private static final Mapper.IgnoreAbove IGNORE_ABOVE = new Mapper.IgnoreAbove(null, IndexMode.STANDARD); private static RootFlattenedFieldType createDefaultFieldType(int ignoreAbove) { return new RootFlattenedFieldType("field", true, true, Collections.emptyMap(), false, false, new Mapper.IgnoreAbove(ignoreAbove)); From d26fad80ae7508539c7ab3b9012950bf0c8496f8 Mon Sep 17 00:00:00 2001 From: Dmitry Kubikov Date: Wed, 10 Sep 2025 13:55:25 -0700 Subject: [PATCH 23/23] Added helpful constructor to IgnoreAbove --- .../org/elasticsearch/index/IgnoreAboveTests.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/server/src/test/java/org/elasticsearch/index/IgnoreAboveTests.java b/server/src/test/java/org/elasticsearch/index/IgnoreAboveTests.java index 1b384f36cb4d7..f43b0eb95683e 100644 --- a/server/src/test/java/org/elasticsearch/index/IgnoreAboveTests.java +++ b/server/src/test/java/org/elasticsearch/index/IgnoreAboveTests.java @@ -112,4 +112,19 @@ public void test_XContentString_isIgnored() { assertTrue(ignoreAbove.isIgnored(new Text("potato potato tomato tomato"))); } + public void test_default_value() { + assertEquals( + Mapper.IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE, + Mapper.IgnoreAbove.getIgnoreAboveDefaultValue(IndexMode.STANDARD, IndexVersion.current()) + ); + assertEquals( + Mapper.IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE_FOR_LOGSDB_INDICES, + Mapper.IgnoreAbove.getIgnoreAboveDefaultValue(IndexMode.LOGSDB, IndexVersion.current()) + ); + assertEquals( + Mapper.IgnoreAbove.IGNORE_ABOVE_DEFAULT_VALUE, + Mapper.IgnoreAbove.getIgnoreAboveDefaultValue(IndexMode.LOGSDB, IndexVersions.ENABLE_IGNORE_MALFORMED_LOGSDB) + ); + } + }