diff --git a/docs/changelog/126623.yaml b/docs/changelog/126623.yaml new file mode 100644 index 0000000000000..9acdb04b7da3d --- /dev/null +++ b/docs/changelog/126623.yaml @@ -0,0 +1,16 @@ +pr: 126623 +summary: Enable synthetic source on normalized keyword mappings +area: Mapping +type: "breaking, enhancement" +issues: + - 124369 + - 121358 +breaking: + title: Enable synthetic source on normalized keyword mappings + area: Mapping + details: |- + This changes the default behavior for Synthetic Source on keyword fields using normalizers. Prior to this change, normalized keywords were always stored to allow returning the non-normalized values. Under this change, such field will NOT be stored (i.e they will be synthesized from the index when returning source, like all other synthetic source fields). This should result in considerable space improvement for this use case. + Users can opt out of this behavior on a per-field basis by setting `synthetic_source_keep` to `all` on the field. + impact: "By default, normalized keyword fields in synthetic source indices will\ + \ no longer return the non-normalized value in the source." + notable: false diff --git a/rest-api-spec/build.gradle b/rest-api-spec/build.gradle index 8ae4999647fe1..6d5a48a98adbd 100644 --- a/rest-api-spec/build.gradle +++ b/rest-api-spec/build.gradle @@ -89,4 +89,5 @@ tasks.named("yamlRestCompatTestTransform").configure ({ task -> task.skipTest("indices.create/21_synthetic_source_stored/field param - keep root array", "Synthetic source keep arrays now stores leaf arrays natively") task.skipTest("cluster.info/30_info_thread_pool/Cluster HTTP Info", "The search_throttled thread pool has been removed") task.skipTest("synonyms/80_synonyms_from_index/Fail loading synonyms from index if synonyms_set doesn't exist", "Synonyms do no longer fail if the synonyms_set doesn't exist") + task.skipTest("mget/90_synthetic_source/keyword with normalizer", "Normalized keywords now use synthetic source") }) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/mget/90_synthetic_source.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/mget/90_synthetic_source.yml index 8485aba0ecc6a..f63d867e1fff5 100644 --- a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/mget/90_synthetic_source.yml +++ b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/mget/90_synthetic_source.yml @@ -6,7 +6,7 @@ setup: --- keyword: - requires: - cluster_features: ["gte_v8.4.0"] + cluster_features: [ "gte_v8.4.0" ] reason: introduced in 8.4.0 - do: @@ -23,15 +23,15 @@ keyword: - do: index: - index: test - id: 1 + index: test + id: 1 body: kwd: foo - do: index: - index: test - id: 2 + index: test + id: 2 body: kwd: bar @@ -40,21 +40,24 @@ keyword: mget: index: test body: - ids: [1, 2] - - match: {docs.0._index: "test"} - - match: {docs.0._id: "1"} + ids: [ 1, 2 ] + - match: { docs.0._index: "test" } + - match: { docs.0._id: "1" } - match: docs.0._source: kwd: foo - - match: {docs.1._index: "test"} - - match: {docs.1._id: "2"} + - match: { docs.1._index: "test" } + - match: { docs.1._id: "2" } - match: docs.1._source: kwd: bar --- keyword with normalizer: + - requires: + cluster_features: "mapper.keyword.keyword_normalizer_synthetic_source" + reason: "Behavior changed in #126623" - do: indices.create: index: test-keyword-with-normalizer @@ -113,7 +116,94 @@ keyword with normalizer: mget: index: test-keyword-with-normalizer body: - ids: [ 1, 2, 3 ] + ids: [ 1, 2, 3 ] + - match: { docs.0._index: "test-keyword-with-normalizer" } + - match: { docs.0._id: "1" } + - match: + docs.0._source: + keyword: "the quick brown fox jumps over the lazy dog" + keyword_with_ignore_above: "the Quick Brown Fox jumps over the lazy Dog" + keyword_without_doc_values: "the Quick Brown Fox jumps over the lazy Dog" + + - match: { docs.1._index: "test-keyword-with-normalizer" } + - match: { docs.1._id: "2" } + - match: + docs.1._source: + keyword: "the five boxing wizards jump quickly" + keyword_with_ignore_above: "The five BOXING wizards jump Quickly" + keyword_without_doc_values: "The five BOXING wizards jump Quickly" + + - match: { docs.2._index: "test-keyword-with-normalizer" } + - match: { docs.2._id: "3" } + - match: + docs.2._source: + keyword: [ "do or do not, there is no try", "may the force be with you!" ] + keyword_with_ignore_above: [ "May the FORCE be with You!", "Do or Do Not, There is no Try" ] + keyword_without_doc_values: [ "May the FORCE be with You!", "Do or Do Not, There is no Try" ] + +--- +keyword with normalizer, source keep mode all: + - do: + indices.create: + index: test-keyword-with-normalizer + body: + settings: + analysis: + normalizer: + lowercase: + type: custom + filter: + - lowercase + index: + mapping.source.mode: synthetic + + mappings: + properties: + keyword: + type: keyword + normalizer: lowercase + synthetic_source_keep: all + keyword_with_ignore_above: + type: keyword + normalizer: lowercase + ignore_above: 10 + keyword_without_doc_values: + type: keyword + normalizer: lowercase + doc_values: false + + - do: + index: + index: test-keyword-with-normalizer + id: 1 + body: + keyword: "the Quick Brown Fox jumps over the lazy Dog" + keyword_with_ignore_above: "the Quick Brown Fox jumps over the lazy Dog" + keyword_without_doc_values: "the Quick Brown Fox jumps over the lazy Dog" + + - do: + index: + index: test-keyword-with-normalizer + id: 2 + body: + keyword: "The five BOXING wizards jump Quickly" + keyword_with_ignore_above: "The five BOXING wizards jump Quickly" + keyword_without_doc_values: "The five BOXING wizards jump Quickly" + + - do: + index: + index: test-keyword-with-normalizer + id: 3 + body: + keyword: [ "May the FORCE be with You!", "Do or Do Not, There is no Try" ] + keyword_with_ignore_above: [ "May the FORCE be with You!", "Do or Do Not, There is no Try" ] + keyword_without_doc_values: [ "May the FORCE be with You!", "Do or Do Not, There is no Try" ] + + - do: + mget: + index: test-keyword-with-normalizer + body: + ids: [ 1, 2, 3 ] - match: { docs.0._index: "test-keyword-with-normalizer" } - match: { docs.0._id: "1" } - match: @@ -141,7 +231,7 @@ keyword with normalizer: --- stored text: - requires: - cluster_features: ["gte_v8.5.0"] + cluster_features: [ "gte_v8.5.0" ] reason: introduced in 8.5.0 - do: @@ -159,15 +249,15 @@ stored text: - do: index: - index: test - id: 1 + index: test + id: 1 body: text: the quick brown fox - do: index: - index: test - id: 2 + index: test + id: 2 body: text: jumped over the lazy dog @@ -175,15 +265,15 @@ stored text: mget: index: test body: - ids: [1, 2] - - match: {docs.0._index: "test"} - - match: {docs.0._id: "1"} + ids: [ 1, 2 ] + - match: { docs.0._index: "test" } + - match: { docs.0._id: "1" } - match: docs.0._source: text: the quick brown fox - - match: {docs.1._index: "test"} - - match: {docs.1._id: "2"} + - match: { docs.1._index: "test" } + - match: { docs.1._id: "2" } - match: docs.1._source: text: jumped over the lazy dog @@ -191,7 +281,7 @@ stored text: --- force_synthetic_source_ok: - requires: - cluster_features: ["gte_v8.4.0"] + cluster_features: [ "gte_v8.4.0" ] reason: introduced in 8.4.0 - do: @@ -210,15 +300,15 @@ force_synthetic_source_ok: - do: index: - index: test - id: 1 + index: test + id: 1 body: obj.kwd: foo - do: index: - index: test - id: 2 + index: test + id: 2 body: obj: kwd: bar @@ -228,7 +318,7 @@ force_synthetic_source_ok: mget: index: test body: - ids: [1, 2] + ids: [ 1, 2 ] - match: docs.0._source: obj.kwd: foo diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index f8af7043b13dd..491783b40a004 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -39,6 +39,7 @@ import org.elasticsearch.common.lucene.search.AutomatonQueries; import org.elasticsearch.common.unit.Fuzziness; import org.elasticsearch.core.Nullable; +import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSortConfig; import org.elasticsearch.index.IndexVersion; @@ -94,6 +95,10 @@ */ public final class KeywordFieldMapper extends FieldMapper { + public static final NodeFeature KEYWORD_NORMALIZER_SYNTHETIC_SOURCE = new NodeFeature( + "mapper.keyword.keyword_normalizer_synthetic_source" + ); + private static final Logger logger = LogManager.getLogger(KeywordFieldMapper.class); public static final String CONTENT_TYPE = "keyword"; @@ -1276,11 +1281,10 @@ private String originalName() { @Override protected SyntheticSourceSupport syntheticSourceSupport() { - if (hasNormalizer()) { - // NOTE: no matter if we have doc values or not we use fallback synthetic source - // to store the original value whose doc values would be altered by the normalizer - return SyntheticSourceSupport.FALLBACK; - } + /* NOTE: we allow enabling synthetic source on Keyword fields with a Normalizer, even though the returned synthetic value + may not perfectly match the original, pre-normalization, value. This reduces the storage space required for normalized keyword + fields, which otherwise perform quite poorly in terms of storage. + */ if (fieldType.stored() || hasDocValues) { return new SyntheticSourceSupport.Native(() -> syntheticFieldLoader(fullPath(), leafName())); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java index 70b6790c4dc1d..9b54d6e217c58 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java @@ -70,7 +70,8 @@ public Set getTestFeatures() { NPE_ON_DIMS_UPDATE_FIX, RESCORE_ZERO_VECTOR_QUANTIZED_VECTOR_MAPPING, USE_DEFAULT_OVERSAMPLE_VALUE_FOR_BBQ, - IVF_FORMAT_CLUSTER_FEATURE + IVF_FORMAT_CLUSTER_FEATURE, + KeywordFieldMapper.KEYWORD_NORMALIZER_SYNTHETIC_SOURCE ); } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java index 08cef586e1438..9935bc0f19072 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/TextFieldMapper.java @@ -300,6 +300,12 @@ public Builder(String name, IndexVersion indexCreatedVersion, IndexAnalyzers ind // storing the field without requiring users to explicitly set 'store'. // // If 'store' parameter was explicitly provided we'll reject the request. + + /* NOTE: I am fairly sure the above is strictly not true. Testing seems + show that we do not reject the mapping, even when there is no compatible keyword + field and the mapping sets store to false. --MT 2025-06-11 + */ + this.store = Parameter.storeParam( m -> ((TextFieldMapper) m).store, () -> isSyntheticSourceEnabled && multiFieldsBuilder.hasSyntheticSourceCompatibleKeywordField() == false diff --git a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/KeywordFieldBlockLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/KeywordFieldBlockLoaderTests.java index 54004022c32b6..9b77886018b83 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/KeywordFieldBlockLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/KeywordFieldBlockLoaderTests.java @@ -15,6 +15,7 @@ import org.elasticsearch.index.mapper.MappedFieldType; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.function.Function; @@ -34,19 +35,21 @@ protected Object expected(Map fieldMapping, Object value, TestCo public static Object expectedValue(Map fieldMapping, Object value, Params params, TestContext testContext) { var nullValue = (String) fieldMapping.get("null_value"); - var ignoreAbove = fieldMapping.get("ignore_above") == null + int ignoreAbove = fieldMapping.get("ignore_above") == null ? Integer.MAX_VALUE : ((Number) fieldMapping.get("ignore_above")).intValue(); + String normalizerName = (String) fieldMapping.get("normalizer"); + if (value == null) { - return convert(null, nullValue, ignoreAbove); + return convert(null, nullValue, ignoreAbove, normalizerName); } if (value instanceof String s) { - return convert(s, nullValue, ignoreAbove); + return convert(s, nullValue, ignoreAbove, normalizerName); } - Function, Stream> convertValues = s -> s.map(v -> convert(v, nullValue, ignoreAbove)) + Function, Stream> convertValues = s -> s.map(v -> convert(v, nullValue, ignoreAbove, normalizerName)) .filter(Objects::nonNull); boolean hasDocValues = hasDocValues(fieldMapping, true); @@ -67,7 +70,7 @@ public static Object expectedValue(Map fieldMapping, Object valu return maybeFoldList(resultList); } - private static BytesRef convert(String value, String nullValue, int ignoreAbove) { + private static BytesRef convert(String value, String nullValue, int ignoreAbove, String normalizer) { if (value == null) { if (nullValue != null) { value = nullValue; @@ -75,7 +78,13 @@ private static BytesRef convert(String value, String nullValue, int ignoreAbove) return null; } } - + if (Objects.equals(normalizer, "lowercase")) { + // hopefully not Turkish... + value = value.toLowerCase(Locale.ROOT); + } else if (normalizer != null) { + // we probably can't get here anyway, since MapperServiceTestCase only initializes the lowercase normalizer + throw new IllegalArgumentException("normalizer [" + normalizer + "] not supported for block loader tests"); + } return value.length() <= ignoreAbove ? new BytesRef(value) : null; } } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/TextFieldWithParentBlockLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/TextFieldWithParentBlockLoaderTests.java index 6343aeea2d9de..b985b4583677b 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/TextFieldWithParentBlockLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/TextFieldWithParentBlockLoaderTests.java @@ -11,6 +11,7 @@ import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import org.elasticsearch.datageneration.DataGeneratorSpecification; import org.elasticsearch.datageneration.DocumentGenerator; import org.elasticsearch.datageneration.FieldType; import org.elasticsearch.datageneration.MappingGenerator; @@ -49,7 +50,7 @@ public TextFieldWithParentBlockLoaderTests(BlockLoaderTestCase.Params params) { // of text multi field in a keyword field. public void testBlockLoaderOfParentField() throws IOException { var template = new Template(Map.of("parent", new Template.Leaf("parent", FieldType.KEYWORD.toString()))); - var specification = buildSpecification(List.of(new DataSourceHandler() { + DataGeneratorSpecification specification = buildSpecification(List.of(new DataSourceHandler() { @Override public DataSourceResponse.LeafMappingParametersGenerator handle(DataSourceRequest.LeafMappingParametersGenerator request) { // This is a bit tricky meta-logic. @@ -101,7 +102,7 @@ public DataSourceResponse.LeafMappingParametersGenerator handle(DataSourceReques var fieldValue = document.get("parent"); Object expected = expected(fieldMapping, fieldValue, new BlockLoaderTestCase.TestContext(false, true)); - var mappingXContent = XContentBuilder.builder(XContentType.JSON.xContent()).map(mapping.raw()); + XContentBuilder mappingXContent = XContentBuilder.builder(XContentType.JSON.xContent()).map(mapping.raw()); var mapperService = params.syntheticSource() ? createSytheticSourceMapperService(mappingXContent) : createMapperService(mappingXContent); diff --git a/test/framework/src/main/java/org/elasticsearch/datageneration/datasource/DefaultMappingParametersHandler.java b/test/framework/src/main/java/org/elasticsearch/datageneration/datasource/DefaultMappingParametersHandler.java index 2e234f8aec41c..3cb5c9be701f4 100644 --- a/test/framework/src/main/java/org/elasticsearch/datageneration/datasource/DefaultMappingParametersHandler.java +++ b/test/framework/src/main/java/org/elasticsearch/datageneration/datasource/DefaultMappingParametersHandler.java @@ -102,7 +102,9 @@ private Supplier> keywordMapping(DataSourceRequest.LeafMappi if (ESTestCase.randomDouble() <= 0.2) { mapping.put("null_value", ESTestCase.randomAlphaOfLengthBetween(0, 10)); } - + if (ESTestCase.randomBoolean()) { + mapping.put("normalizer", "lowercase"); + } return mapping; }; } diff --git a/test/framework/src/main/java/org/elasticsearch/datageneration/matchers/source/FieldSpecificMatcher.java b/test/framework/src/main/java/org/elasticsearch/datageneration/matchers/source/FieldSpecificMatcher.java index aa7b1e203c89e..689bee2d166d0 100644 --- a/test/framework/src/main/java/org/elasticsearch/datageneration/matchers/source/FieldSpecificMatcher.java +++ b/test/framework/src/main/java/org/elasticsearch/datageneration/matchers/source/FieldSpecificMatcher.java @@ -331,6 +331,7 @@ Object convert(Object value, Object nullValue) { } class KeywordMatcher extends GenericMappingAwareMatcher { + KeywordMatcher( XContentBuilder actualMappings, Settings.Builder actualSettings, @@ -340,12 +341,31 @@ class KeywordMatcher extends GenericMappingAwareMatcher { super("keyword", actualMappings, actualSettings, expectedMappings, expectedSettings); } + @Override + public MatchResult match( + List actual, + List expected, + Map actualMapping, + Map expectedMapping + ) { + String normalizer = (String) FieldSpecificMatcher.getMappingParameter("normalizer", actualMapping, expectedMapping); + // Account for normalization + if (normalizer != null) { + if (normalizer.equals("lowercase") == false) { + throw new UnsupportedOperationException("Test framework currently only supports lowercase normalizer"); + } + expected = expected.stream() + .map(s -> s instanceof String ? ((String) s).toLowerCase(Locale.ROOT) : s) + .collect(Collectors.toList()); + } + return super.match(actual, expected, actualMapping, expectedMapping); + } + @Override Object convert(Object value, Object nullValue) { if (value == null) { return nullValue; } - return value; } } @@ -766,10 +786,10 @@ public MatchResult match( Map actualMapping, Map expectedMapping ) { - var nullValue = getNullValue(actualMapping, expectedMapping); + Object nullValue = getNullValue(actualMapping, expectedMapping); - var expectedNormalized = normalize(expected, nullValue); - var actualNormalized = normalize(actual, nullValue); + Set expectedNormalized = normalize(expected, nullValue); + Set actualNormalized = normalize(actual, nullValue); return actualNormalized.equals(expectedNormalized) ? MatchResult.match() diff --git a/test/framework/src/main/java/org/elasticsearch/datageneration/matchers/source/SourceMatcher.java b/test/framework/src/main/java/org/elasticsearch/datageneration/matchers/source/SourceMatcher.java index 6583a181dacb5..c99cbf72fef6f 100644 --- a/test/framework/src/main/java/org/elasticsearch/datageneration/matchers/source/SourceMatcher.java +++ b/test/framework/src/main/java/org/elasticsearch/datageneration/matchers/source/SourceMatcher.java @@ -76,10 +76,10 @@ public MatchResult match() { var sortedAndFlattenedExpected = expected.stream().map(s -> SourceTransforms.normalize(s, mappingLookup)).toList(); for (int i = 0; i < sortedAndFlattenedActual.size(); i++) { - var actual = sortedAndFlattenedActual.get(i); - var expected = sortedAndFlattenedExpected.get(i); + Map> actual = sortedAndFlattenedActual.get(i); + Map> expected = sortedAndFlattenedExpected.get(i); - var result = compareSource(actual, expected); + MatchResult result = compareSource(actual, expected); if (result.isMatch() == false) { var message = "Source matching failed at document id [" + i + "]. " + result.getMessage(); return MatchResult.noMatch(message); @@ -90,15 +90,18 @@ public MatchResult match() { } private MatchResult compareSource(Map> actual, Map> expected) { - for (var expectedFieldEntry : expected.entrySet()) { - var name = expectedFieldEntry.getKey(); + for (Map.Entry> expectedFieldEntry : expected.entrySet()) { + String name = expectedFieldEntry.getKey(); - var actualValues = actual.get(name); - var expectedValues = expectedFieldEntry.getValue(); + List actualValues = actual.get(name); + List expectedValues = expectedFieldEntry.getValue(); - var matchIncludingFieldSpecificMatchers = matchWithFieldSpecificMatcher(name, actualValues, expectedValues); + MatchResult matchIncludingFieldSpecificMatchers = matchWithFieldSpecificMatcher(name, actualValues, expectedValues); if (matchIncludingFieldSpecificMatchers.isMatch() == false) { - var message = "Source documents don't match for field [" + name + "]: " + matchIncludingFieldSpecificMatchers.getMessage(); + String message = "Source documents don't match for field [" + + name + + "]: " + + matchIncludingFieldSpecificMatchers.getMessage(); return MatchResult.noMatch(message); } } @@ -106,7 +109,7 @@ private MatchResult compareSource(Map> actual, Map actualValues, List expectedValues) { - var actualFieldMapping = actualNormalizedMapping.get(fieldName); + MappingTransforms.FieldMapping actualFieldMapping = actualNormalizedMapping.get(fieldName); if (actualFieldMapping == null) { if (expectedNormalizedMapping.get(fieldName) != null // Special cases due to fields being defined in default mapping for logsdb index mode @@ -126,7 +129,7 @@ private MatchResult matchWithFieldSpecificMatcher(String fieldName, List throw new IllegalStateException("Field type is missing from leaf field Leaf field [" + fieldName + "] mapping parameters"); } - var expectedFieldMapping = expectedNormalizedMapping.get(fieldName); + MappingTransforms.FieldMapping expectedFieldMapping = expectedNormalizedMapping.get(fieldName); if (expectedFieldMapping == null) { throw new IllegalStateException("Leaf field [" + fieldName + "] is present in actual mapping but absent in expected mapping"); } else { @@ -144,7 +147,7 @@ private MatchResult matchWithFieldSpecificMatcher(String fieldName, List } } - var fieldSpecificMatcher = fieldSpecificMatchers.get(actualFieldType); + FieldSpecificMatcher fieldSpecificMatcher = fieldSpecificMatchers.get(actualFieldType); assert fieldSpecificMatcher != null : "Missing matcher for field type [" + actualFieldType + "]"; return fieldSpecificMatcher.match( diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java index 4a1d33595eb79..ddf4e4ec79157 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java @@ -41,6 +41,7 @@ import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.analysis.AnalyzerScope; import org.elasticsearch.index.analysis.IndexAnalyzers; +import org.elasticsearch.index.analysis.LowercaseNormalizer; import org.elasticsearch.index.analysis.NameOrDefinition; import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.cache.bitset.BitsetFilterCache; @@ -132,7 +133,11 @@ protected IndexAnalyzers createIndexAnalyzers(IndexSettings indexSettings) { } protected static IndexAnalyzers createIndexAnalyzers() { - return IndexAnalyzers.of(Map.of("default", new NamedAnalyzer("default", AnalyzerScope.INDEX, new StandardAnalyzer()))); + return IndexAnalyzers.of( + Map.of("default", new NamedAnalyzer("default", AnalyzerScope.INDEX, new StandardAnalyzer())), + Map.of("lowercase", new NamedAnalyzer("lowercase", AnalyzerScope.INDEX, new LowercaseNormalizer())), + Map.of() + ); } protected static String randomIndexOptions() { diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index 51a99c3d47c4f..10f378ec24b2a 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -134,6 +134,12 @@ tasks.named("yamlRestCompatTestTransform").configure({ task -> task.skipTest("esql/90_non_indexed/fetch", "Temporary until backported") task.skipTest("esql/63_enrich_int_range/Invalid age as double", "TODO: require disable allow_partial_results") task.skipTest("esql/191_lookup_join_on_datastreams/data streams not supported in LOOKUP JOIN", "Added support for aliases in JOINs") + task.skipTest("esql/81_text_exact_subfields/filter ignore above", "Normalized keywords now use synthetic source") + task.skipTest("esql/81_text_exact_subfields/filter with normalizer", "Normalized keywords now use synthetic source") + task.skipTest("esql/81_text_exact_subfields/non indexed", "Normalized keywords now use synthetic source") + task.skipTest("esql/81_text_exact_subfields/extract", "Normalized keywords now use synthetic source") + task.skipTest("esql/81_text_exact_subfields/sort ignore above", "Normalized keywords now use synthetic source") + task.skipTest("esql/81_text_exact_subfields/sort normalizer", "Normalized keywords now use synthetic source") }) tasks.named('yamlRestCompatTest').configure {