Skip to content

Commit 1760919

Browse files
parkertimminsparkertimmins
committed
Move multifield into separate data source handler
1 parent 4c2843c commit 1760919

File tree

8 files changed

+113
-123
lines changed

8 files changed

+113
-123
lines changed

‎server/src/test/java/org/elasticsearch/index/mapper/blockloader/TextFieldWithParentBlockLoaderTests.java‎

Lines changed: 5 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,7 @@
1515
import org.elasticsearch.datageneration.FieldType;
1616
import org.elasticsearch.datageneration.MappingGenerator;
1717
import org.elasticsearch.datageneration.Template;
18-
import org.elasticsearch.datageneration.datasource.DataSourceHandler;
19-
import org.elasticsearch.datageneration.datasource.DataSourceRequest;
20-
import org.elasticsearch.datageneration.datasource.DataSourceResponse;
18+
import org.elasticsearch.datageneration.datasource.MultifieldAddonHandler;
2119
import org.elasticsearch.index.mapper.BlockLoaderTestCase;
2220
import org.elasticsearch.index.mapper.BlockLoaderTestRunner;
2321
import org.elasticsearch.index.mapper.MapperServiceTestCase;
@@ -49,53 +47,8 @@ public TextFieldWithParentBlockLoaderTests(BlockLoaderTestCase.Params params) {
4947
// of text multi field in a keyword field.
5048
public void testBlockLoaderOfParentField() throws IOException {
5149
var template = new Template(Map.of("parent", new Template.Leaf("parent", FieldType.KEYWORD.toString())));
52-
var specification = buildSpecification(List.of(new DataSourceHandler() {
53-
@Override
54-
public DataSourceResponse.LeafMappingParametersGenerator handle(DataSourceRequest.LeafMappingParametersGenerator request) {
55-
// This is a bit tricky meta-logic.
56-
// We want to customize mapping but to do this we need the mapping for the same field type
57-
// so we use name to untangle this.
58-
if (request.fieldName().equals("parent") == false) {
59-
return null;
60-
}
61-
62-
return new DataSourceResponse.LeafMappingParametersGenerator(() -> {
63-
var dataSource = request.dataSource();
64-
65-
var keywordParentMapping = dataSource.get(
66-
new DataSourceRequest.LeafMappingParametersGenerator(
67-
dataSource,
68-
"_field",
69-
FieldType.KEYWORD.toString(),
70-
request.eligibleCopyToFields(),
71-
request.dynamicMapping(),
72-
request.includePluginTypesInMultiFields()
73-
)
74-
).mappingGenerator().get();
75-
76-
var textMultiFieldMapping = dataSource.get(
77-
new DataSourceRequest.LeafMappingParametersGenerator(
78-
dataSource,
79-
"_field",
80-
FieldType.TEXT.toString(),
81-
request.eligibleCopyToFields(),
82-
request.dynamicMapping(),
83-
request.includePluginTypesInMultiFields()
84-
)
85-
).mappingGenerator().get();
86-
87-
// we don't need this here
88-
keywordParentMapping.remove("copy_to");
89-
90-
textMultiFieldMapping.put("type", "text");
91-
textMultiFieldMapping.remove("fields");
92-
93-
keywordParentMapping.put("fields", Map.of("mf", textMultiFieldMapping));
94-
95-
return keywordParentMapping;
96-
});
97-
}
98-
}));
50+
var specification = buildSpecification(List.of(new MultifieldAddonHandler(Map.of(FieldType.KEYWORD, List.of(FieldType.TEXT)), 1f)));
51+
9952
var mapping = new MappingGenerator(specification).generate(template);
10053
var fieldMapping = mapping.lookup().get("parent");
10154

@@ -108,7 +61,7 @@ public DataSourceResponse.LeafMappingParametersGenerator handle(DataSourceReques
10861
? createSytheticSourceMapperService(mappingXContent)
10962
: createMapperService(mappingXContent);
11063

111-
runner.runTest(mapperService, document, expected, "parent.mf");
64+
runner.runTest(mapperService, document, expected, "parent.subfield_text");
11265
}
11366

11467
@SuppressWarnings("unchecked")
@@ -125,7 +78,7 @@ private Object expected(Map<String, Object> fieldMapping, Object value, BlockLoa
12578
}
12679

12780
// we are using block loader of the text field itself
128-
var textFieldMapping = (Map<String, Object>) ((Map<String, Object>) fieldMapping.get("fields")).get("mf");
81+
var textFieldMapping = (Map<String, Object>) ((Map<String, Object>) fieldMapping.get("fields")).get("subfield_text");
12982
return TextFieldBlockLoaderTests.expectedValue(textFieldMapping, value, params, testContext);
13083
}
13184
}

‎test/framework/src/main/java/org/elasticsearch/datageneration/DataGeneratorSpecification.java‎

Lines changed: 2 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -26,16 +26,14 @@
2626
* @param nestedFieldsLimit how many total nested fields can be present in a produced mapping
2727
* @param fullyDynamicMapping if the mapping is fully dynamic, meaning none of the fields are mapped (essentially mapping is empty)
2828
* @param predefinedFields predefined fields that must be present in mapping and documents. Only top level fields are supported.
29-
* @param includePluginTypes whether types defined in plugins should be added to mapping
3029
*/
3130
public record DataGeneratorSpecification(
3231
DataSource dataSource,
3332
int maxFieldCountPerLevel,
3433
int maxObjectDepth,
3534
int nestedFieldsLimit,
3635
boolean fullyDynamicMapping,
37-
List<PredefinedField> predefinedFields,
38-
boolean includePluginTypes
36+
List<PredefinedField> predefinedFields
3937
) {
4038

4139
public static Builder builder() {
@@ -53,7 +51,6 @@ public static class Builder {
5351
private int nestedFieldsLimit;
5452
private boolean fullyDynamicMapping;
5553
private List<PredefinedField> predefinedFields;
56-
private boolean includePluginTypes;
5754

5855
public Builder() {
5956
this.dataSourceHandlers = new ArrayList<>();
@@ -64,7 +61,6 @@ public Builder() {
6461
this.nestedFieldsLimit = 50;
6562
fullyDynamicMapping = false;
6663
this.predefinedFields = new ArrayList<>();
67-
this.includePluginTypes = true;
6864
}
6965

7066
public Builder withDataSourceHandlers(Collection<DataSourceHandler> handlers) {
@@ -97,20 +93,14 @@ public Builder withPredefinedFields(List<PredefinedField> predefinedFields) {
9793
return this;
9894
}
9995

100-
public Builder withIncludePluginTypes(boolean includePluginTypes) {
101-
this.includePluginTypes = includePluginTypes;
102-
return this;
103-
}
104-
10596
public DataGeneratorSpecification build() {
10697
return new DataGeneratorSpecification(
10798
new DataSource(dataSourceHandlers),
10899
maxFieldCountPerLevel,
109100
maxObjectDepth,
110101
nestedFieldsLimit,
111102
fullyDynamicMapping,
112-
predefinedFields,
113-
includePluginTypes
103+
predefinedFields
114104
);
115105
}
116106
}

‎test/framework/src/main/java/org/elasticsearch/datageneration/MappingGenerator.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -108,8 +108,7 @@ private void generateMapping(
108108
fieldName,
109109
leaf.type(),
110110
context.eligibleCopyToDestinations(),
111-
context.parentDynamicMapping(),
112-
specification.includePluginTypes()
111+
context.parentDynamicMapping()
113112
)
114113
)
115114
.mappingGenerator();

‎test/framework/src/main/java/org/elasticsearch/datageneration/datasource/DataSourceRequest.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -203,8 +203,7 @@ record LeafMappingParametersGenerator(
203203
String fieldName,
204204
String fieldType,
205205
Set<String> eligibleCopyToFields,
206-
DynamicMapping dynamicMapping,
207-
boolean includePluginTypesInMultiFields
206+
DynamicMapping dynamicMapping
208207
) implements DataSourceRequest<DataSourceResponse.LeafMappingParametersGenerator> {
209208
public DataSourceResponse.LeafMappingParametersGenerator accept(DataSourceHandler handler) {
210209
return handler.handle(this);

‎test/framework/src/main/java/org/elasticsearch/datageneration/datasource/DefaultMappingParametersHandler.java‎

Lines changed: 9 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@
2222
import java.time.ZoneOffset;
2323
import java.time.format.DateTimeFormatter;
2424
import java.util.HashMap;
25-
import java.util.List;
2625
import java.util.Locale;
2726
import java.util.Map;
2827
import java.util.function.Supplier;
@@ -38,18 +37,18 @@ public DataSourceResponse.LeafMappingParametersGenerator handle(DataSourceReques
3837
}
3938

4039
return new DataSourceResponse.LeafMappingParametersGenerator(switch (fieldType) {
41-
case KEYWORD -> keywordMapping(false, request);
40+
case KEYWORD -> keywordMapping(request);
4241
case LONG, INTEGER, SHORT, BYTE, DOUBLE, FLOAT, HALF_FLOAT, UNSIGNED_LONG -> numberMapping(fieldType);
4342
case SCALED_FLOAT -> scaledFloatMapping();
4443
case COUNTED_KEYWORD -> countedKeywordMapping();
4544
case BOOLEAN -> booleanMapping();
4645
case DATE -> dateMapping();
4746
case GEO_POINT -> geoPointMapping();
48-
case TEXT -> textMapping(false, request);
47+
case TEXT -> textMapping();
4948
case IP -> ipMapping();
5049
case CONSTANT_KEYWORD -> constantKeywordMapping();
51-
case WILDCARD -> wildcardMapping(false, request);
52-
case MATCH_ONLY_TEXT -> matchOnlyTextMapping(false, request);
50+
case WILDCARD -> wildcardMapping();
51+
case MATCH_ONLY_TEXT -> matchOnlyTextMapping();
5352
});
5453
}
5554

@@ -79,7 +78,7 @@ private Supplier<Map<String, Object>> numberMapping(FieldType fieldType) {
7978
};
8079
}
8180

82-
private Supplier<Map<String, Object>> keywordMapping(boolean hasParent, DataSourceRequest.LeafMappingParametersGenerator request) {
81+
private Supplier<Map<String, Object>> keywordMapping(DataSourceRequest.LeafMappingParametersGenerator request) {
8382
return () -> {
8483
var mapping = commonMappingParameters();
8584

@@ -104,9 +103,6 @@ private Supplier<Map<String, Object>> keywordMapping(boolean hasParent, DataSour
104103
if (ESTestCase.randomDouble() <= 0.2) {
105104
mapping.put("null_value", ESTestCase.randomAlphaOfLengthBetween(0, 10));
106105
}
107-
if (hasParent == false && ESTestCase.randomBoolean()) {
108-
mapping.put("fields", stringSubField(request));
109-
}
110106

111107
return mapping;
112108
};
@@ -201,17 +197,13 @@ private Supplier<Map<String, Object>> geoPointMapping() {
201197
};
202198
}
203199

204-
private Supplier<Map<String, Object>> textMapping(boolean hasParent, DataSourceRequest.LeafMappingParametersGenerator request) {
200+
private Supplier<Map<String, Object>> textMapping() {
205201
return () -> {
206202
var mapping = new HashMap<String, Object>();
207203

208204
mapping.put("store", ESTestCase.randomBoolean());
209205
mapping.put("index", ESTestCase.randomBoolean());
210206

211-
if (hasParent == false && ESTestCase.randomBoolean()) {
212-
mapping.put("fields", stringSubField(request));
213-
}
214-
215207
return mapping;
216208
};
217209
}
@@ -244,7 +236,7 @@ private Supplier<Map<String, Object>> constantKeywordMapping() {
244236
};
245237
}
246238

247-
private Supplier<Map<String, Object>> wildcardMapping(boolean hasParent, DataSourceRequest.LeafMappingParametersGenerator request) {
239+
private Supplier<Map<String, Object>> wildcardMapping() {
248240
return () -> {
249241
var mapping = new HashMap<String, Object>();
250242

@@ -254,47 +246,13 @@ private Supplier<Map<String, Object>> wildcardMapping(boolean hasParent, DataSou
254246
if (ESTestCase.randomDouble() <= 0.2) {
255247
mapping.put("null_value", ESTestCase.randomAlphaOfLengthBetween(0, 10));
256248
}
257-
if (hasParent == false && ESTestCase.randomBoolean()) {
258-
mapping.put("fields", stringSubField(request));
259-
}
260-
261-
return mapping;
262-
};
263-
}
264249

265-
private Supplier<Map<String, Object>> matchOnlyTextMapping(
266-
boolean hasParent,
267-
DataSourceRequest.LeafMappingParametersGenerator request
268-
) {
269-
return () -> {
270-
var mapping = new HashMap<String, Object>();
271-
if (hasParent == false && ESTestCase.randomBoolean()) {
272-
mapping.put("fields", stringSubField(request));
273-
}
274250
return mapping;
275251
};
276252
}
277253

278-
private Map<String, Object> stringSubField(DataSourceRequest.LeafMappingParametersGenerator request) {
279-
FieldType parent = FieldType.tryParse(request.fieldType());
280-
List<FieldType> childTypes = request.includePluginTypesInMultiFields()
281-
? List.of(FieldType.TEXT, FieldType.KEYWORD, FieldType.WILDCARD, FieldType.MATCH_ONLY_TEXT)
282-
: List.of(FieldType.TEXT, FieldType.KEYWORD);
283-
var childType = ESTestCase.randomValueOtherThan(parent, () -> ESTestCase.randomFrom(childTypes));
284-
var child = switch (childType) {
285-
case TEXT -> textMapping(true, request).get();
286-
case KEYWORD -> {
287-
var mapping = keywordMapping(true, request).get();
288-
mapping.remove("copy_to");
289-
yield mapping;
290-
}
291-
case MATCH_ONLY_TEXT -> matchOnlyTextMapping(true, request).get();
292-
case WILDCARD -> wildcardMapping(true, request).get();
293-
default -> throw new AssertionError("unreachable");
294-
};
295-
296-
child.put("type", childType.toString());
297-
return Map.of("subfield_" + childType, child);
254+
private Supplier<Map<String, Object>> matchOnlyTextMapping() {
255+
return HashMap::new;
298256
}
299257

300258
public static HashMap<String, Object> commonMappingParameters() {

‎test/framework/src/main/java/org/elasticsearch/datageneration/datasource/MultifieldAddonHandler.java‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.datageneration.datasource;
11+
12+
import org.elasticsearch.datageneration.FieldType;
13+
import org.elasticsearch.test.ESTestCase;
14+
15+
import java.util.List;
16+
import java.util.Map;
17+
import java.util.Set;
18+
import java.util.stream.Collectors;
19+
20+
public class MultifieldAddonHandler implements DataSourceHandler {
21+
22+
private static final String PLACEHOLDER_NAME = "_an_improbably_placeholder_name";
23+
private static final float DEFAULT_CHANCE_OF_CHILD_FIELD = 0.5f;
24+
private final Map<FieldType, List<FieldType>> subfieldTypes;
25+
private final float chanceOfChildField;
26+
27+
private static final Set<FieldType> STRING_TYPES = Set.of(FieldType.TEXT, FieldType.KEYWORD, FieldType.MATCH_ONLY_TEXT, FieldType.WILDCARD);
28+
public static MultifieldAddonHandler ALL_STRING_TYPES = new MultifieldAddonHandler(STRING_TYPES.stream().collect(Collectors.toMap(t -> t, t -> STRING_TYPES.stream().filter(s -> s != t).toList())));
29+
30+
public MultifieldAddonHandler(Map<FieldType, List<FieldType>> subfieldTypes, float chanceOfChildField) {
31+
this.subfieldTypes = subfieldTypes;
32+
this.chanceOfChildField = chanceOfChildField;
33+
}
34+
35+
public MultifieldAddonHandler(Map<FieldType, List<FieldType>> subfieldTypes) {
36+
this(subfieldTypes, DEFAULT_CHANCE_OF_CHILD_FIELD);
37+
}
38+
39+
@Override
40+
public DataSourceResponse.LeafMappingParametersGenerator handle(DataSourceRequest.LeafMappingParametersGenerator request) {
41+
42+
// Need to delegate creation of the same type of field to other handlers. So skip request
43+
// if it's for the placeholder name used when creating the child and parent fields.
44+
if (request.fieldName().equals(PLACEHOLDER_NAME)) {
45+
return null;
46+
}
47+
48+
FieldType parentType = FieldType.tryParse(request.fieldType());
49+
List<FieldType> childTypes = subfieldTypes.get(parentType);
50+
if (childTypes == null) {
51+
return null;
52+
}
53+
54+
return new DataSourceResponse.LeafMappingParametersGenerator(() -> {
55+
assert parentType != null;
56+
var parent = getMappingForType(parentType, request);
57+
if (ESTestCase.randomFloat() > chanceOfChildField) {
58+
return parent;
59+
}
60+
61+
var childType = ESTestCase.randomFrom(childTypes);
62+
var child = getChildMappingForType(childType, request);
63+
64+
child.put("type", childType.toString());
65+
String childName = "subfield_" + childType;
66+
parent.put("fields", Map.of(childName, child));
67+
return parent;
68+
});
69+
}
70+
71+
private static Map<String, Object> getChildMappingForType(FieldType type, DataSourceRequest.LeafMappingParametersGenerator request) {
72+
Map<String, Object> mapping = getMappingForType(type, request);
73+
if (type == FieldType.KEYWORD) {
74+
mapping.remove("copy_to");
75+
}
76+
return mapping;
77+
}
78+
79+
private static Map<String, Object> getMappingForType(FieldType type, DataSourceRequest.LeafMappingParametersGenerator request) {
80+
return request.dataSource().get(
81+
new DataSourceRequest.LeafMappingParametersGenerator(
82+
request.dataSource(),
83+
PLACEHOLDER_NAME,
84+
type.toString(),
85+
request.eligibleCopyToFields(),
86+
request.dynamicMapping()
87+
)
88+
).mappingGenerator().get();
89+
}
90+
}

0 commit comments

Comments
 (0)