diff --git a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/TextFieldBlockLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/TextFieldBlockLoaderTests.java index ce5482b15b0ee..d78c7eba351e2 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/TextFieldBlockLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/TextFieldBlockLoaderTests.java @@ -41,7 +41,7 @@ public static Object expectedValue(Map fieldMapping, Object valu var fields = (Map) fieldMapping.get("fields"); if (fields != null) { - var keywordMultiFieldMapping = (Map) fields.get("kwd"); + var keywordMultiFieldMapping = (Map) fields.get("subfield_keyword"); Object normalizer = fields.get("normalizer"); boolean docValues = hasDocValues(keywordMultiFieldMapping, true); boolean store = keywordMultiFieldMapping.getOrDefault("store", false).equals(true); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/TextFieldWithParentBlockLoaderTests.java b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/TextFieldWithParentBlockLoaderTests.java index 6343aeea2d9de..c74e133611071 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/blockloader/TextFieldWithParentBlockLoaderTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/blockloader/TextFieldWithParentBlockLoaderTests.java @@ -15,9 +15,7 @@ import org.elasticsearch.datageneration.FieldType; import org.elasticsearch.datageneration.MappingGenerator; import org.elasticsearch.datageneration.Template; -import org.elasticsearch.datageneration.datasource.DataSourceHandler; -import org.elasticsearch.datageneration.datasource.DataSourceRequest; -import org.elasticsearch.datageneration.datasource.DataSourceResponse; +import org.elasticsearch.datageneration.datasource.MultifieldAddonHandler; import org.elasticsearch.index.mapper.BlockLoaderTestCase; import org.elasticsearch.index.mapper.BlockLoaderTestRunner; import org.elasticsearch.index.mapper.MapperServiceTestCase; @@ -49,51 +47,8 @@ public TextFieldWithParentBlockLoaderTests(BlockLoaderTestCase.Params params) { // of text multi field in a keyword field. public void testBlockLoaderOfParentField() throws IOException { var template = new Template(Map.of("parent", new Template.Leaf("parent", FieldType.KEYWORD.toString()))); - var specification = buildSpecification(List.of(new DataSourceHandler() { - @Override - public DataSourceResponse.LeafMappingParametersGenerator handle(DataSourceRequest.LeafMappingParametersGenerator request) { - // This is a bit tricky meta-logic. - // We want to customize mapping but to do this we need the mapping for the same field type - // so we use name to untangle this. - if (request.fieldName().equals("parent") == false) { - return null; - } - - return new DataSourceResponse.LeafMappingParametersGenerator(() -> { - var dataSource = request.dataSource(); - - var keywordParentMapping = dataSource.get( - new DataSourceRequest.LeafMappingParametersGenerator( - dataSource, - "_field", - FieldType.KEYWORD.toString(), - request.eligibleCopyToFields(), - request.dynamicMapping() - ) - ).mappingGenerator().get(); - - var textMultiFieldMapping = dataSource.get( - new DataSourceRequest.LeafMappingParametersGenerator( - dataSource, - "_field", - FieldType.TEXT.toString(), - request.eligibleCopyToFields(), - request.dynamicMapping() - ) - ).mappingGenerator().get(); - - // we don't need this here - keywordParentMapping.remove("copy_to"); - - textMultiFieldMapping.put("type", "text"); - textMultiFieldMapping.remove("fields"); - - keywordParentMapping.put("fields", Map.of("mf", textMultiFieldMapping)); - - return keywordParentMapping; - }); - } - })); + var specification = buildSpecification(List.of(new MultifieldAddonHandler(Map.of(FieldType.KEYWORD, List.of(FieldType.TEXT)), 1f))); + var mapping = new MappingGenerator(specification).generate(template); var fieldMapping = mapping.lookup().get("parent"); @@ -106,7 +61,7 @@ public DataSourceResponse.LeafMappingParametersGenerator handle(DataSourceReques ? createSytheticSourceMapperService(mappingXContent) : createMapperService(mappingXContent); - runner.runTest(mapperService, document, expected, "parent.mf"); + runner.runTest(mapperService, document, expected, "parent.subfield_text"); } @SuppressWarnings("unchecked") @@ -123,7 +78,7 @@ private Object expected(Map fieldMapping, Object value, BlockLoa } // we are using block loader of the text field itself - var textFieldMapping = (Map) ((Map) fieldMapping.get("fields")).get("mf"); + var textFieldMapping = (Map) ((Map) fieldMapping.get("fields")).get("subfield_text"); return TextFieldBlockLoaderTests.expectedValue(textFieldMapping, value, params, testContext); } } diff --git a/test/framework/src/main/java/org/elasticsearch/datageneration/FieldType.java b/test/framework/src/main/java/org/elasticsearch/datageneration/FieldType.java index eab2149019204..ef6cc21e90d8a 100644 --- a/test/framework/src/main/java/org/elasticsearch/datageneration/FieldType.java +++ b/test/framework/src/main/java/org/elasticsearch/datageneration/FieldType.java @@ -23,6 +23,7 @@ import org.elasticsearch.datageneration.fields.leaf.IpFieldDataGenerator; import org.elasticsearch.datageneration.fields.leaf.KeywordFieldDataGenerator; import org.elasticsearch.datageneration.fields.leaf.LongFieldDataGenerator; +import org.elasticsearch.datageneration.fields.leaf.MatchOnlyTextFieldDataGenerator; import org.elasticsearch.datageneration.fields.leaf.ScaledFloatFieldDataGenerator; import org.elasticsearch.datageneration.fields.leaf.ShortFieldDataGenerator; import org.elasticsearch.datageneration.fields.leaf.TextFieldDataGenerator; @@ -50,7 +51,8 @@ public enum FieldType { TEXT("text"), IP("ip"), CONSTANT_KEYWORD("constant_keyword"), - WILDCARD("wildcard"); + WILDCARD("wildcard"), + MATCH_ONLY_TEXT("match_only_text"); private final String name; @@ -78,6 +80,7 @@ public FieldDataGenerator generator(String fieldName, DataSource dataSource) { case IP -> new IpFieldDataGenerator(dataSource); case CONSTANT_KEYWORD -> new ConstantKeywordFieldDataGenerator(); case WILDCARD -> new WildcardFieldDataGenerator(dataSource); + case MATCH_ONLY_TEXT -> new MatchOnlyTextFieldDataGenerator(dataSource); }; } @@ -101,6 +104,7 @@ public static FieldType tryParse(String name) { case "ip" -> FieldType.IP; case "constant_keyword" -> FieldType.CONSTANT_KEYWORD; case "wildcard" -> FieldType.WILDCARD; + case "match_only_text" -> FieldType.MATCH_ONLY_TEXT; default -> null; }; } diff --git a/test/framework/src/main/java/org/elasticsearch/datageneration/datasource/DefaultMappingParametersHandler.java b/test/framework/src/main/java/org/elasticsearch/datageneration/datasource/DefaultMappingParametersHandler.java index 2e234f8aec41c..8e759946f2ee4 100644 --- a/test/framework/src/main/java/org/elasticsearch/datageneration/datasource/DefaultMappingParametersHandler.java +++ b/test/framework/src/main/java/org/elasticsearch/datageneration/datasource/DefaultMappingParametersHandler.java @@ -44,10 +44,11 @@ public DataSourceResponse.LeafMappingParametersGenerator handle(DataSourceReques case BOOLEAN -> booleanMapping(); case DATE -> dateMapping(); case GEO_POINT -> geoPointMapping(); - case TEXT -> textMapping(request); + case TEXT -> textMapping(); case IP -> ipMapping(); case CONSTANT_KEYWORD -> constantKeywordMapping(); case WILDCARD -> wildcardMapping(); + case MATCH_ONLY_TEXT -> matchOnlyTextMapping(); }); } @@ -96,8 +97,8 @@ private Supplier> keywordMapping(DataSourceRequest.LeafMappi } } - if (ESTestCase.randomDouble() <= 0.2) { - mapping.put("ignore_above", ESTestCase.randomIntBetween(1, 100)); + if (ESTestCase.randomDouble() <= 0.3) { + mapping.put("ignore_above", ESTestCase.randomIntBetween(1, 50)); } if (ESTestCase.randomDouble() <= 0.2) { mapping.put("null_value", ESTestCase.randomAlphaOfLengthBetween(0, 10)); @@ -196,21 +197,13 @@ private Supplier> geoPointMapping() { }; } - private Supplier> textMapping(DataSourceRequest.LeafMappingParametersGenerator request) { + private Supplier> textMapping() { return () -> { var mapping = new HashMap(); mapping.put("store", ESTestCase.randomBoolean()); mapping.put("index", ESTestCase.randomBoolean()); - if (ESTestCase.randomDouble() <= 0.1) { - var keywordMultiFieldMapping = keywordMapping(request).get(); - keywordMultiFieldMapping.put("type", "keyword"); - keywordMultiFieldMapping.remove("copy_to"); - - mapping.put("fields", Map.of("kwd", keywordMultiFieldMapping)); - } - return mapping; }; } @@ -247,8 +240,8 @@ private Supplier> wildcardMapping() { return () -> { var mapping = new HashMap(); - if (ESTestCase.randomDouble() <= 0.2) { - mapping.put("ignore_above", ESTestCase.randomIntBetween(1, 100)); + if (ESTestCase.randomDouble() <= 0.3) { + mapping.put("ignore_above", ESTestCase.randomIntBetween(1, 50)); } if (ESTestCase.randomDouble() <= 0.2) { mapping.put("null_value", ESTestCase.randomAlphaOfLengthBetween(0, 10)); @@ -258,6 +251,10 @@ private Supplier> wildcardMapping() { }; } + private Supplier> matchOnlyTextMapping() { + return HashMap::new; + } + public static HashMap commonMappingParameters() { var map = new HashMap(); map.put("store", ESTestCase.randomBoolean()); diff --git a/test/framework/src/main/java/org/elasticsearch/datageneration/datasource/DefaultObjectGenerationHandler.java b/test/framework/src/main/java/org/elasticsearch/datageneration/datasource/DefaultObjectGenerationHandler.java index bf660779186ca..0938e59903099 100644 --- a/test/framework/src/main/java/org/elasticsearch/datageneration/datasource/DefaultObjectGenerationHandler.java +++ b/test/framework/src/main/java/org/elasticsearch/datageneration/datasource/DefaultObjectGenerationHandler.java @@ -20,6 +20,14 @@ import static org.elasticsearch.test.ESTestCase.randomRealisticUnicodeOfCodepointLengthBetween; public class DefaultObjectGenerationHandler implements DataSourceHandler { + + /** + * Field names will not be generated which start with `_reserved_`. Handlers can safely + * create field names starting with this prefix without the concern of randomly generated + * fields having the same name. + */ + public static final String RESERVED_FIELD_NAME_PREFIX = "_reserved_"; + @Override public DataSourceResponse.ChildFieldGenerator handle(DataSourceRequest.ChildFieldGenerator request) { return new DataSourceResponse.ChildFieldGenerator() { @@ -57,6 +65,9 @@ public String generateFieldName() { if (fieldName.indexOf('.') != -1) { continue; } + if (fieldName.startsWith(RESERVED_FIELD_NAME_PREFIX)) { + continue; + } return fieldName; } diff --git a/test/framework/src/main/java/org/elasticsearch/datageneration/datasource/MultifieldAddonHandler.java b/test/framework/src/main/java/org/elasticsearch/datageneration/datasource/MultifieldAddonHandler.java new file mode 100644 index 0000000000000..886629beaf9d2 --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/datageneration/datasource/MultifieldAddonHandler.java @@ -0,0 +1,97 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.datageneration.datasource; + +import org.elasticsearch.datageneration.FieldType; +import org.elasticsearch.test.ESTestCase; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +public class MultifieldAddonHandler implements DataSourceHandler { + + private static final String PLACEHOLDER = DefaultObjectGenerationHandler.RESERVED_FIELD_NAME_PREFIX + "multifield"; + private static final float DEFAULT_CHANCE_OF_CHILD_FIELD = 0.5f; + private final Map> subfieldTypes; + private final float chanceOfChildField; + + private static final List STRING_TYPES = List.of( + FieldType.TEXT, + FieldType.KEYWORD, + FieldType.MATCH_ONLY_TEXT, + FieldType.WILDCARD + ); + public static MultifieldAddonHandler STRING_TYPE_HANDLER = new MultifieldAddonHandler( + STRING_TYPES.stream().collect(Collectors.toMap(t -> t, t -> STRING_TYPES.stream().filter(s -> s != t).toList())) + ); + + public MultifieldAddonHandler(Map> subfieldTypes, float chanceOfChildField) { + this.subfieldTypes = subfieldTypes; + this.chanceOfChildField = chanceOfChildField; + } + + public MultifieldAddonHandler(Map> subfieldTypes) { + this(subfieldTypes, DEFAULT_CHANCE_OF_CHILD_FIELD); + } + + @Override + public DataSourceResponse.LeafMappingParametersGenerator handle(DataSourceRequest.LeafMappingParametersGenerator request) { + + // Need to delegate creation of the same type of field to other handlers. So skip request + // if it's for the placeholder name used when creating the child and parent fields. + if (request.fieldName().equals(PLACEHOLDER)) { + return null; + } + + FieldType parentType = FieldType.tryParse(request.fieldType()); + List childTypes = subfieldTypes.get(parentType); + if (childTypes == null) { + return null; + } + + return new DataSourceResponse.LeafMappingParametersGenerator(() -> { + assert parentType != null; + var parent = getMappingForType(parentType, request); + if (ESTestCase.randomFloat() > chanceOfChildField) { + return parent; + } + + var childType = ESTestCase.randomFrom(childTypes); + var child = getChildMappingForType(childType, request); + + child.put("type", childType.toString()); + String childName = "subfield_" + childType; + parent.put("fields", Map.of(childName, child)); + return parent; + }); + } + + private static Map getChildMappingForType(FieldType type, DataSourceRequest.LeafMappingParametersGenerator request) { + Map mapping = getMappingForType(type, request); + mapping.remove("copy_to"); + return mapping; + } + + private static Map getMappingForType(FieldType type, DataSourceRequest.LeafMappingParametersGenerator request) { + return request.dataSource() + .get( + new DataSourceRequest.LeafMappingParametersGenerator( + request.dataSource(), + PLACEHOLDER, + type.toString(), + request.eligibleCopyToFields(), + request.dynamicMapping() + ) + ) + .mappingGenerator() + .get(); + } +} diff --git a/test/framework/src/main/java/org/elasticsearch/datageneration/fields/leaf/MatchOnlyTextFieldDataGenerator.java b/test/framework/src/main/java/org/elasticsearch/datageneration/fields/leaf/MatchOnlyTextFieldDataGenerator.java new file mode 100644 index 0000000000000..f4493fd9b4ee9 --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/datageneration/fields/leaf/MatchOnlyTextFieldDataGenerator.java @@ -0,0 +1,28 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.datageneration.fields.leaf; + +import org.elasticsearch.datageneration.FieldDataGenerator; +import org.elasticsearch.datageneration.datasource.DataSource; + +import java.util.Map; + +public class MatchOnlyTextFieldDataGenerator implements FieldDataGenerator { + private final FieldDataGenerator textGenerator; + + public MatchOnlyTextFieldDataGenerator(DataSource dataSource) { + this.textGenerator = new TextFieldDataGenerator(dataSource); + } + + @Override + public Object generateValue(Map fieldMapping) { + return textGenerator.generateValue(fieldMapping); + } +} diff --git a/test/framework/src/main/java/org/elasticsearch/datageneration/matchers/source/FieldSpecificMatcher.java b/test/framework/src/main/java/org/elasticsearch/datageneration/matchers/source/FieldSpecificMatcher.java index 7adf98ef9d6ee..1503cbd8e9adb 100644 --- a/test/framework/src/main/java/org/elasticsearch/datageneration/matchers/source/FieldSpecificMatcher.java +++ b/test/framework/src/main/java/org/elasticsearch/datageneration/matchers/source/FieldSpecificMatcher.java @@ -63,6 +63,7 @@ static Map matchers( put("shape", new ExactMatcher("shape", actualMappings, actualSettings, expectedMappings, expectedSettings)); put("geo_point", new GeoPointMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings)); put("text", new TextMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings)); + put("match_only_text", new MatchOnlyTextMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings)); put("ip", new IpMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings)); put("constant_keyword", new ConstantKeywordMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings)); put("wildcard", new WildcardMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings)); @@ -621,6 +622,10 @@ class TextMatcher implements FieldSpecificMatcher { this.expectedSettings = expectedSettings; } + public String type() { + return "text"; + } + @Override @SuppressWarnings("unchecked") public MatchResult match( @@ -643,7 +648,7 @@ public MatchResult match( if (multiFields != null) { var keywordMatcher = new KeywordMatcher(actualMappings, actualSettings, expectedMappings, expectedSettings); - var keywordFieldMapping = (Map) multiFields.get("kwd"); + var keywordFieldMapping = (Map) multiFields.get("subfield_keyword"); var keywordMatchResult = keywordMatcher.match(actual, expected, keywordFieldMapping, keywordFieldMapping); if (keywordMatchResult.isMatch()) { return MatchResult.match(); @@ -656,7 +661,7 @@ public MatchResult match( actualSettings, expectedMappings, expectedSettings, - "Values of type [text] don't match, " + prettyPrintCollections(actual, expected) + "Values of type [" + type() + "] don't match, " + prettyPrintCollections(actual, expected) ) ); } @@ -670,6 +675,22 @@ private Set normalize(List values) { } } + class MatchOnlyTextMatcher extends TextMatcher { + MatchOnlyTextMatcher( + XContentBuilder actualMappings, + Settings.Builder actualSettings, + XContentBuilder expectedMappings, + Settings.Builder expectedSettings + ) { + super(actualMappings, actualSettings, expectedMappings, expectedSettings); + } + + @Override + public String type() { + return "match_only_text"; + } + } + class IpMatcher extends GenericMappingAwareMatcher { IpMatcher( XContentBuilder actualMappings, diff --git a/test/framework/src/main/java/org/elasticsearch/datageneration/queries/LeafQueryGenerator.java b/test/framework/src/main/java/org/elasticsearch/datageneration/queries/LeafQueryGenerator.java index be26db580edf6..cb102e185c9b5 100644 --- a/test/framework/src/main/java/org/elasticsearch/datageneration/queries/LeafQueryGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/datageneration/queries/LeafQueryGenerator.java @@ -12,10 +12,7 @@ import org.elasticsearch.datageneration.FieldType; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.test.ESTestCase; -import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Map; @@ -28,7 +25,7 @@ public interface LeafQueryGenerator { * @param type the type to build a query for * @return a generator that can build queries for this type */ - static LeafQueryGenerator buildForType(String type) { + static LeafQueryGenerator buildForType(String type, MappingPredicates mappingPredicates) { LeafQueryGenerator noQueries = (Map fieldMapping, String path, Object value) -> List.of(); FieldType fieldType = FieldType.tryParse(type); @@ -38,8 +35,9 @@ static LeafQueryGenerator buildForType(String type) { return switch (fieldType) { case KEYWORD -> new KeywordQueryGenerator(); - case TEXT -> new TextQueryGenerator(); case WILDCARD -> new WildcardQueryGenerator(); + case TEXT -> new TextQueryGenerator(); + case MATCH_ONLY_TEXT -> new MatchOnlyTextQueryGenerator(mappingPredicates); default -> noQueries; }; } @@ -53,13 +51,14 @@ public List generate(Map fieldMapping, String path return List.of(); } } - return List.of(QueryBuilders.termQuery(path, value)); + return List.of(QueryBuilders.termQuery(path, value), QueryBuilders.matchQuery(path, value)); } } class WildcardQueryGenerator implements LeafQueryGenerator { public List generate(Map fieldMapping, String path, Object value) { - // Queries with emojis can currently fail due to https://github.com/elastic/elasticsearch/issues/132144 + // TODO remove when fixed + // queries with emojis can currently fail due to https://github.com/elastic/elasticsearch/issues/132144 if (containsHighSurrogates((String) value)) { return List.of(); } @@ -76,25 +75,20 @@ public List generate(Map fieldMapping, String path } } - var results = new ArrayList(); - results.add(QueryBuilders.matchQuery(path, value)); - var phraseQuery = buildPhraseQuery(path, (String) value); - if (phraseQuery != null) { - results.add(phraseQuery); - } - return results; + return List.of(QueryBuilders.matchQuery(path, value), QueryBuilders.matchPhraseQuery(path, value)); } + } + + record MatchOnlyTextQueryGenerator(MappingPredicates mappingPredicates) implements LeafQueryGenerator { - private static QueryBuilder buildPhraseQuery(String path, String value) { - var tokens = Arrays.asList(value.split("[^a-zA-Z0-9]")); - if (tokens.isEmpty()) { - return null; + public List generate(Map fieldMapping, String path, Object value) { + // TODO remove when fixed + // match_only_text in nested context fails for synthetic source https://github.com/elastic/elasticsearch/issues/132352 + if (mappingPredicates.inNestedContext(path)) { + return List.of(QueryBuilders.matchQuery(path, value)); } - int low = ESTestCase.randomIntBetween(0, tokens.size() - 1); - int hi = ESTestCase.randomIntBetween(low + 1, tokens.size()); - var phrase = String.join(" ", tokens.subList(low, hi)); - return QueryBuilders.matchPhraseQuery(path, phrase); + return List.of(QueryBuilders.matchQuery(path, value), QueryBuilders.matchPhraseQuery(path, value)); } } diff --git a/test/framework/src/main/java/org/elasticsearch/datageneration/queries/MappingPredicates.java b/test/framework/src/main/java/org/elasticsearch/datageneration/queries/MappingPredicates.java new file mode 100644 index 0000000000000..a1500e6612c0f --- /dev/null +++ b/test/framework/src/main/java/org/elasticsearch/datageneration/queries/MappingPredicates.java @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.datageneration.queries; + +import org.elasticsearch.datageneration.Mapping; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +public class MappingPredicates { + + private final Mapping mapping; + + public MappingPredicates(Mapping mapping) { + this.mapping = mapping; + } + + record PathMapping(String path, Map mapping) {} + + private List getPathMapping(String path) { + String[] parts = path.split("\\."); + var result = new ArrayList(); + for (int i = 0; i < parts.length; i++) { + var pathToHere = String.join(".", Arrays.copyOfRange(parts, 0, i + 1)); + Map fieldMapping = mapping.lookup().get(pathToHere); + if (fieldMapping == null) { + break; + } + result.add(new PathMapping(pathToHere, fieldMapping)); + } + return result; + } + + public List getNestedPathPrefixes(String fullPath) { + return getPathMapping(fullPath).stream().filter(pm -> "nested".equals(pm.mapping().get("type"))).map(PathMapping::path).toList(); + } + + public boolean inNestedContext(String fullPath) { + return getPathMapping(fullPath).stream().anyMatch(pm -> "nested".equals(pm.mapping().get("type"))); + } + + @SuppressWarnings("unchecked") + public boolean isRuntimeField(String path) { + var topLevelMapping = (Map) mapping.raw().get("_doc"); + boolean inRuntimeContext = "runtime".equals(topLevelMapping.get("dynamic")); + for (var pm : getPathMapping(path)) { + if (pm.mapping().containsKey("dynamic")) { + // lower down dynamic definitions override higher up behavior + inRuntimeContext = "runtime".equals(pm.mapping().get("dynamic")); + } + } + return inRuntimeContext; + } + +} diff --git a/test/framework/src/main/java/org/elasticsearch/datageneration/queries/QueryGenerator.java b/test/framework/src/main/java/org/elasticsearch/datageneration/queries/QueryGenerator.java index 9db0b628f85da..7630d810acdc5 100644 --- a/test/framework/src/main/java/org/elasticsearch/datageneration/queries/QueryGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/datageneration/queries/QueryGenerator.java @@ -14,17 +14,16 @@ import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; -import java.util.ArrayList; -import java.util.Arrays; import java.util.List; -import java.util.Map; public class QueryGenerator { private final Mapping mapping; + private final MappingPredicates mappingPredicates; public QueryGenerator(Mapping mapping) { this.mapping = mapping; + this.mappingPredicates = new MappingPredicates(mapping); } public List generateQueries(String type, String path, Object value) { @@ -33,66 +32,21 @@ public List generateQueries(String type, String path, Object value return List.of(); } // Can handle dynamically mapped fields, but not runtime fields - if (isRuntimeField(path)) { + if (mappingPredicates.isRuntimeField(path)) { return List.of(); } - var leafQueryGenerator = LeafQueryGenerator.buildForType(type); + var leafQueryGenerator = LeafQueryGenerator.buildForType(type, mappingPredicates); var fieldMapping = mapping.lookup().get(path); var leafQueries = leafQueryGenerator.generate(fieldMapping, path, value); return leafQueries.stream().map(q -> wrapInNestedQuery(path, q)).toList(); } private QueryBuilder wrapInNestedQuery(String path, QueryBuilder leafQuery) { - String[] parts = path.split("\\."); - List nestedPaths = getNestedPathPrefixes(parts); + List nestedPaths = mappingPredicates.getNestedPathPrefixes(path); QueryBuilder query = leafQuery; for (String nestedPath : nestedPaths.reversed()) { query = QueryBuilders.nestedQuery(nestedPath, query, ScoreMode.Max); } return query; } - - @SuppressWarnings("unchecked") - private List getNestedPathPrefixes(String[] path) { - Map mapping = this.mapping.raw(); - mapping = (Map) mapping.get("_doc"); - mapping = (Map) mapping.get("properties"); - - var result = new ArrayList(); - for (int i = 0; i < path.length - 1; i++) { - var field = path[i]; - mapping = (Map) mapping.get(field); - - // dynamic field - if (mapping == null) { - break; - } - - boolean nested = "nested".equals(mapping.get("type")); - if (nested) { - result.add(String.join(".", Arrays.copyOfRange(path, 0, i + 1))); - } - mapping = (Map) mapping.get("properties"); - } - return result; - } - - @SuppressWarnings("unchecked") - private boolean isRuntimeField(String path) { - String[] parts = path.split("\\."); - var topLevelMapping = (Map) mapping.raw().get("_doc"); - boolean inRuntimeContext = "runtime".equals(topLevelMapping.get("dynamic")); - for (int i = 0; i < parts.length - 1; i++) { - var pathToHere = String.join(".", Arrays.copyOfRange(parts, 0, i + 1)); - Map fieldMapping = mapping.lookup().get(pathToHere); - if (fieldMapping == null) { - break; - } - if (fieldMapping.containsKey("dynamic")) { - // lower down dynamic definitions override higher up behavior - inRuntimeContext = "runtime".equals(fieldMapping.get("dynamic")); - } - } - return inRuntimeContext; - } } diff --git a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/DataGenerationHelper.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/DataGenerationHelper.java index 86c435ba2a4e8..257689c8aa558 100644 --- a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/DataGenerationHelper.java +++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/DataGenerationHelper.java @@ -18,6 +18,7 @@ import org.elasticsearch.datageneration.datasource.DataSourceHandler; import org.elasticsearch.datageneration.datasource.DataSourceRequest; import org.elasticsearch.datageneration.datasource.DataSourceResponse; +import org.elasticsearch.datageneration.datasource.MultifieldAddonHandler; import org.elasticsearch.datageneration.fields.PredefinedField; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.test.ESTestCase; @@ -105,7 +106,8 @@ public DataSourceResponse.FieldTypeGenerator.FieldTypeInfo get() { } }); } - })); + })) + .withDataSourceHandlers(List.of(MultifieldAddonHandler.STRING_TYPE_HANDLER)); // Customize builder if necessary builderConfigurator.accept(specificationBuilder); diff --git a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/StandardVersusLogsIndexModeChallengeRestIT.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/StandardVersusLogsIndexModeChallengeRestIT.java index db48748bba302..c4eb399a39243 100644 --- a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/StandardVersusLogsIndexModeChallengeRestIT.java +++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/qa/StandardVersusLogsIndexModeChallengeRestIT.java @@ -143,6 +143,7 @@ public void testMatchAllQuery() throws IOException { assertTrue(matchResult.getMessage(), matchResult.isMatch()); } + @SuppressWarnings("unchecked") public void testRandomQueries() throws IOException { int numberOfDocuments = ESTestCase.randomIntBetween(10, 50); final List documents = generateDocuments(numberOfDocuments);