Skip to content

Commit 8ca0947

Browse files
parkertimminsparkertimmins
authored
Add random tests with match_only_text multi-field (elastic#132380)
Add tests to the logsdb randomized tests which use multi-fields for each of the text, keyword, match_only_text, and wildcard types. For each type allow each other type of multi-field.
1 parent da49b80 commit 8ca0947

File tree

13 files changed

+270
-142
lines changed

13 files changed

+270
-142
lines changed

‎server/src/test/java/org/elasticsearch/index/mapper/blockloader/TextFieldBlockLoaderTests.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ public static Object expectedValue(Map<String, Object> fieldMapping, Object valu
4141

4242
var fields = (Map<String, Object>) fieldMapping.get("fields");
4343
if (fields != null) {
44-
var keywordMultiFieldMapping = (Map<String, Object>) fields.get("kwd");
44+
var keywordMultiFieldMapping = (Map<String, Object>) fields.get("subfield_keyword");
4545
Object normalizer = fields.get("normalizer");
4646
boolean docValues = hasDocValues(keywordMultiFieldMapping, true);
4747
boolean store = keywordMultiFieldMapping.getOrDefault("store", false).equals(true);

‎server/src/test/java/org/elasticsearch/index/mapper/blockloader/TextFieldWithParentBlockLoaderTests.java‎

Lines changed: 5 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,7 @@
1515
import org.elasticsearch.datageneration.FieldType;
1616
import org.elasticsearch.datageneration.MappingGenerator;
1717
import org.elasticsearch.datageneration.Template;
18-
import org.elasticsearch.datageneration.datasource.DataSourceHandler;
19-
import org.elasticsearch.datageneration.datasource.DataSourceRequest;
20-
import org.elasticsearch.datageneration.datasource.DataSourceResponse;
18+
import org.elasticsearch.datageneration.datasource.MultifieldAddonHandler;
2119
import org.elasticsearch.index.mapper.BlockLoaderTestCase;
2220
import org.elasticsearch.index.mapper.BlockLoaderTestRunner;
2321
import org.elasticsearch.index.mapper.MapperServiceTestCase;
@@ -49,51 +47,8 @@ public TextFieldWithParentBlockLoaderTests(BlockLoaderTestCase.Params params) {
4947
// of text multi field in a keyword field.
5048
public void testBlockLoaderOfParentField() throws IOException {
5149
var template = new Template(Map.of("parent", new Template.Leaf("parent", FieldType.KEYWORD.toString())));
52-
var specification = buildSpecification(List.of(new DataSourceHandler() {
53-
@Override
54-
public DataSourceResponse.LeafMappingParametersGenerator handle(DataSourceRequest.LeafMappingParametersGenerator request) {
55-
// This is a bit tricky meta-logic.
56-
// We want to customize mapping but to do this we need the mapping for the same field type
57-
// so we use name to untangle this.
58-
if (request.fieldName().equals("parent") == false) {
59-
return null;
60-
}
61-
62-
return new DataSourceResponse.LeafMappingParametersGenerator(() -> {
63-
var dataSource = request.dataSource();
64-
65-
var keywordParentMapping = dataSource.get(
66-
new DataSourceRequest.LeafMappingParametersGenerator(
67-
dataSource,
68-
"_field",
69-
FieldType.KEYWORD.toString(),
70-
request.eligibleCopyToFields(),
71-
request.dynamicMapping()
72-
)
73-
).mappingGenerator().get();
74-
75-
var textMultiFieldMapping = dataSource.get(
76-
new DataSourceRequest.LeafMappingParametersGenerator(
77-
dataSource,
78-
"_field",
79-
FieldType.TEXT.toString(),
80-
request.eligibleCopyToFields(),
81-
request.dynamicMapping()
82-
)
83-
).mappingGenerator().get();
84-
85-
// we don't need this here
86-
keywordParentMapping.remove("copy_to");
87-
88-
textMultiFieldMapping.put("type", "text");
89-
textMultiFieldMapping.remove("fields");
90-
91-
keywordParentMapping.put("fields", Map.of("mf", textMultiFieldMapping));
92-
93-
return keywordParentMapping;
94-
});
95-
}
96-
}));
50+
var specification = buildSpecification(List.of(new MultifieldAddonHandler(Map.of(FieldType.KEYWORD, List.of(FieldType.TEXT)), 1f)));
51+
9752
var mapping = new MappingGenerator(specification).generate(template);
9853
var fieldMapping = mapping.lookup().get("parent");
9954

@@ -106,7 +61,7 @@ public DataSourceResponse.LeafMappingParametersGenerator handle(DataSourceReques
10661
? createSytheticSourceMapperService(mappingXContent)
10762
: createMapperService(mappingXContent);
10863

109-
runner.runTest(mapperService, document, expected, "parent.mf");
64+
runner.runTest(mapperService, document, expected, "parent.subfield_text");
11065
}
11166

11267
@SuppressWarnings("unchecked")
@@ -123,7 +78,7 @@ private Object expected(Map<String, Object> fieldMapping, Object value, BlockLoa
12378
}
12479

12580
// we are using block loader of the text field itself
126-
var textFieldMapping = (Map<String, Object>) ((Map<String, Object>) fieldMapping.get("fields")).get("mf");
81+
var textFieldMapping = (Map<String, Object>) ((Map<String, Object>) fieldMapping.get("fields")).get("subfield_text");
12782
return TextFieldBlockLoaderTests.expectedValue(textFieldMapping, value, params, testContext);
12883
}
12984
}

‎test/framework/src/main/java/org/elasticsearch/datageneration/FieldType.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import org.elasticsearch.datageneration.fields.leaf.IpFieldDataGenerator;
2424
import org.elasticsearch.datageneration.fields.leaf.KeywordFieldDataGenerator;
2525
import org.elasticsearch.datageneration.fields.leaf.LongFieldDataGenerator;
26+
import org.elasticsearch.datageneration.fields.leaf.MatchOnlyTextFieldDataGenerator;
2627
import org.elasticsearch.datageneration.fields.leaf.ScaledFloatFieldDataGenerator;
2728
import org.elasticsearch.datageneration.fields.leaf.ShortFieldDataGenerator;
2829
import org.elasticsearch.datageneration.fields.leaf.TextFieldDataGenerator;
@@ -50,7 +51,8 @@ public enum FieldType {
5051
TEXT("text"),
5152
IP("ip"),
5253
CONSTANT_KEYWORD("constant_keyword"),
53-
WILDCARD("wildcard");
54+
WILDCARD("wildcard"),
55+
MATCH_ONLY_TEXT("match_only_text");
5456

5557
private final String name;
5658

@@ -78,6 +80,7 @@ public FieldDataGenerator generator(String fieldName, DataSource dataSource) {
7880
case IP -> new IpFieldDataGenerator(dataSource);
7981
case CONSTANT_KEYWORD -> new ConstantKeywordFieldDataGenerator();
8082
case WILDCARD -> new WildcardFieldDataGenerator(dataSource);
83+
case MATCH_ONLY_TEXT -> new MatchOnlyTextFieldDataGenerator(dataSource);
8184
};
8285
}
8386

@@ -101,6 +104,7 @@ public static FieldType tryParse(String name) {
101104
case "ip" -> FieldType.IP;
102105
case "constant_keyword" -> FieldType.CONSTANT_KEYWORD;
103106
case "wildcard" -> FieldType.WILDCARD;
107+
case "match_only_text" -> FieldType.MATCH_ONLY_TEXT;
104108
default -> null;
105109
};
106110
}

‎test/framework/src/main/java/org/elasticsearch/datageneration/datasource/DefaultMappingParametersHandler.java‎

Lines changed: 11 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -44,10 +44,11 @@ public DataSourceResponse.LeafMappingParametersGenerator handle(DataSourceReques
4444
case BOOLEAN -> booleanMapping();
4545
case DATE -> dateMapping();
4646
case GEO_POINT -> geoPointMapping();
47-
case TEXT -> textMapping(request);
47+
case TEXT -> textMapping();
4848
case IP -> ipMapping();
4949
case CONSTANT_KEYWORD -> constantKeywordMapping();
5050
case WILDCARD -> wildcardMapping();
51+
case MATCH_ONLY_TEXT -> matchOnlyTextMapping();
5152
});
5253
}
5354

@@ -96,8 +97,8 @@ private Supplier<Map<String, Object>> keywordMapping(DataSourceRequest.LeafMappi
9697
}
9798
}
9899

99-
if (ESTestCase.randomDouble() <= 0.2) {
100-
mapping.put("ignore_above", ESTestCase.randomIntBetween(1, 100));
100+
if (ESTestCase.randomDouble() <= 0.3) {
101+
mapping.put("ignore_above", ESTestCase.randomIntBetween(1, 50));
101102
}
102103
if (ESTestCase.randomDouble() <= 0.2) {
103104
mapping.put("null_value", ESTestCase.randomAlphaOfLengthBetween(0, 10));
@@ -196,21 +197,13 @@ private Supplier<Map<String, Object>> geoPointMapping() {
196197
};
197198
}
198199

199-
private Supplier<Map<String, Object>> textMapping(DataSourceRequest.LeafMappingParametersGenerator request) {
200+
private Supplier<Map<String, Object>> textMapping() {
200201
return () -> {
201202
var mapping = new HashMap<String, Object>();
202203

203204
mapping.put("store", ESTestCase.randomBoolean());
204205
mapping.put("index", ESTestCase.randomBoolean());
205206

206-
if (ESTestCase.randomDouble() <= 0.1) {
207-
var keywordMultiFieldMapping = keywordMapping(request).get();
208-
keywordMultiFieldMapping.put("type", "keyword");
209-
keywordMultiFieldMapping.remove("copy_to");
210-
211-
mapping.put("fields", Map.of("kwd", keywordMultiFieldMapping));
212-
}
213-
214207
return mapping;
215208
};
216209
}
@@ -247,8 +240,8 @@ private Supplier<Map<String, Object>> wildcardMapping() {
247240
return () -> {
248241
var mapping = new HashMap<String, Object>();
249242

250-
if (ESTestCase.randomDouble() <= 0.2) {
251-
mapping.put("ignore_above", ESTestCase.randomIntBetween(1, 100));
243+
if (ESTestCase.randomDouble() <= 0.3) {
244+
mapping.put("ignore_above", ESTestCase.randomIntBetween(1, 50));
252245
}
253246
if (ESTestCase.randomDouble() <= 0.2) {
254247
mapping.put("null_value", ESTestCase.randomAlphaOfLengthBetween(0, 10));
@@ -258,6 +251,10 @@ private Supplier<Map<String, Object>> wildcardMapping() {
258251
};
259252
}
260253

254+
private Supplier<Map<String, Object>> matchOnlyTextMapping() {
255+
return HashMap::new;
256+
}
257+
261258
public static HashMap<String, Object> commonMappingParameters() {
262259
var map = new HashMap<String, Object>();
263260
map.put("store", ESTestCase.randomBoolean());

‎test/framework/src/main/java/org/elasticsearch/datageneration/datasource/DefaultObjectGenerationHandler.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,14 @@
2020
import static org.elasticsearch.test.ESTestCase.randomRealisticUnicodeOfCodepointLengthBetween;
2121

2222
public class DefaultObjectGenerationHandler implements DataSourceHandler {
23+
24+
/**
25+
* Field names will not be generated which start with `_reserved_`. Handlers can safely
26+
* create field names starting with this prefix without the concern of randomly generated
27+
* fields having the same name.
28+
*/
29+
public static final String RESERVED_FIELD_NAME_PREFIX = "_reserved_";
30+
2331
@Override
2432
public DataSourceResponse.ChildFieldGenerator handle(DataSourceRequest.ChildFieldGenerator request) {
2533
return new DataSourceResponse.ChildFieldGenerator() {
@@ -57,6 +65,9 @@ public String generateFieldName() {
5765
if (fieldName.indexOf('.') != -1) {
5866
continue;
5967
}
68+
if (fieldName.startsWith(RESERVED_FIELD_NAME_PREFIX)) {
69+
continue;
70+
}
6071

6172
return fieldName;
6273
}

‎test/framework/src/main/java/org/elasticsearch/datageneration/datasource/MultifieldAddonHandler.java‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.datageneration.datasource;
11+
12+
import org.elasticsearch.datageneration.FieldType;
13+
import org.elasticsearch.test.ESTestCase;
14+
15+
import java.util.List;
16+
import java.util.Map;
17+
import java.util.stream.Collectors;
18+
19+
public class MultifieldAddonHandler implements DataSourceHandler {
20+
21+
private static final String PLACEHOLDER = DefaultObjectGenerationHandler.RESERVED_FIELD_NAME_PREFIX + "multifield";
22+
private static final float DEFAULT_CHANCE_OF_CHILD_FIELD = 0.5f;
23+
private final Map<FieldType, List<FieldType>> subfieldTypes;
24+
private final float chanceOfChildField;
25+
26+
private static final List<FieldType> STRING_TYPES = List.of(
27+
FieldType.TEXT,
28+
FieldType.KEYWORD,
29+
FieldType.MATCH_ONLY_TEXT,
30+
FieldType.WILDCARD
31+
);
32+
public static MultifieldAddonHandler STRING_TYPE_HANDLER = new MultifieldAddonHandler(
33+
STRING_TYPES.stream().collect(Collectors.toMap(t -> t, t -> STRING_TYPES.stream().filter(s -> s != t).toList()))
34+
);
35+
36+
public MultifieldAddonHandler(Map<FieldType, List<FieldType>> subfieldTypes, float chanceOfChildField) {
37+
this.subfieldTypes = subfieldTypes;
38+
this.chanceOfChildField = chanceOfChildField;
39+
}
40+
41+
public MultifieldAddonHandler(Map<FieldType, List<FieldType>> subfieldTypes) {
42+
this(subfieldTypes, DEFAULT_CHANCE_OF_CHILD_FIELD);
43+
}
44+
45+
@Override
46+
public DataSourceResponse.LeafMappingParametersGenerator handle(DataSourceRequest.LeafMappingParametersGenerator request) {
47+
48+
// Need to delegate creation of the same type of field to other handlers. So skip request
49+
// if it's for the placeholder name used when creating the child and parent fields.
50+
if (request.fieldName().equals(PLACEHOLDER)) {
51+
return null;
52+
}
53+
54+
FieldType parentType = FieldType.tryParse(request.fieldType());
55+
List<FieldType> childTypes = subfieldTypes.get(parentType);
56+
if (childTypes == null) {
57+
return null;
58+
}
59+
60+
return new DataSourceResponse.LeafMappingParametersGenerator(() -> {
61+
assert parentType != null;
62+
var parent = getMappingForType(parentType, request);
63+
if (ESTestCase.randomFloat() > chanceOfChildField) {
64+
return parent;
65+
}
66+
67+
var childType = ESTestCase.randomFrom(childTypes);
68+
var child = getChildMappingForType(childType, request);
69+
70+
child.put("type", childType.toString());
71+
String childName = "subfield_" + childType;
72+
parent.put("fields", Map.of(childName, child));
73+
return parent;
74+
});
75+
}
76+
77+
private static Map<String, Object> getChildMappingForType(FieldType type, DataSourceRequest.LeafMappingParametersGenerator request) {
78+
Map<String, Object> mapping = getMappingForType(type, request);
79+
mapping.remove("copy_to");
80+
return mapping;
81+
}
82+
83+
private static Map<String, Object> getMappingForType(FieldType type, DataSourceRequest.LeafMappingParametersGenerator request) {
84+
return request.dataSource()
85+
.get(
86+
new DataSourceRequest.LeafMappingParametersGenerator(
87+
request.dataSource(),
88+
PLACEHOLDER,
89+
type.toString(),
90+
request.eligibleCopyToFields(),
91+
request.dynamicMapping()
92+
)
93+
)
94+
.mappingGenerator()
95+
.get();
96+
}
97+
}

‎test/framework/src/main/java/org/elasticsearch/datageneration/fields/leaf/MatchOnlyTextFieldDataGenerator.java‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.datageneration.fields.leaf;
11+
12+
import org.elasticsearch.datageneration.FieldDataGenerator;
13+
import org.elasticsearch.datageneration.datasource.DataSource;
14+
15+
import java.util.Map;
16+
17+
public class MatchOnlyTextFieldDataGenerator implements FieldDataGenerator {
18+
private final FieldDataGenerator textGenerator;
19+
20+
public MatchOnlyTextFieldDataGenerator(DataSource dataSource) {
21+
this.textGenerator = new TextFieldDataGenerator(dataSource);
22+
}
23+
24+
@Override
25+
public Object generateValue(Map<String, Object> fieldMapping) {
26+
return textGenerator.generateValue(fieldMapping);
27+
}
28+
}

0 commit comments

Comments
 (0)