Skip to content

Commit c1f2f80

Browse files
lktslkts
authored
Correctly inject subobjects parameter in logsdb tests (#113643)
1 parent 0c69de1 commit c1f2f80

File tree

9 files changed

+114
-77
lines changed

9 files changed

+114
-77
lines changed

‎modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/logsdb/qa/DataGenerationHelper.java‎

Lines changed: 3 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -12,25 +12,18 @@
1212
import org.elasticsearch.common.settings.Settings;
1313
import org.elasticsearch.core.CheckedConsumer;
1414
import org.elasticsearch.index.mapper.Mapper;
15-
import org.elasticsearch.index.mapper.ObjectMapper;
1615
import org.elasticsearch.logsdb.datageneration.DataGenerator;
1716
import org.elasticsearch.logsdb.datageneration.DataGeneratorSpecification;
1817
import org.elasticsearch.logsdb.datageneration.FieldDataGenerator;
19-
import org.elasticsearch.logsdb.datageneration.datasource.DataSourceHandler;
20-
import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest;
21-
import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse;
2218
import org.elasticsearch.logsdb.datageneration.fields.PredefinedField;
2319
import org.elasticsearch.test.ESTestCase;
2420
import org.elasticsearch.xcontent.XContentBuilder;
2521

2622
import java.io.IOException;
27-
import java.util.HashMap;
2823
import java.util.List;
29-
import java.util.Map;
3024
import java.util.function.Consumer;
3125

3226
public class DataGenerationHelper {
33-
private final ObjectMapper.Subobjects subobjects;
3427
private final boolean keepArraySource;
3528

3629
private final DataGenerator dataGenerator;
@@ -40,44 +33,10 @@ public DataGenerationHelper() {
4033
}
4134

4235
public DataGenerationHelper(Consumer<DataGeneratorSpecification.Builder> builderConfigurator) {
43-
// TODO enable subobjects: auto
44-
// It is disabled because it currently does not have auto flattening and that results in asserts being triggered when using copy_to.
45-
this.subobjects = ESTestCase.randomValueOtherThan(
46-
ObjectMapper.Subobjects.AUTO,
47-
() -> ESTestCase.randomFrom(ObjectMapper.Subobjects.values())
48-
);
4936
this.keepArraySource = ESTestCase.randomBoolean();
5037

51-
var specificationBuilder = DataGeneratorSpecification.builder().withFullyDynamicMapping(ESTestCase.randomBoolean());
52-
if (subobjects != ObjectMapper.Subobjects.ENABLED) {
53-
specificationBuilder = specificationBuilder.withNestedFieldsLimit(0);
54-
}
55-
56-
specificationBuilder.withDataSourceHandlers(List.of(new DataSourceHandler() {
57-
@Override
58-
public DataSourceResponse.ObjectMappingParametersGenerator handle(DataSourceRequest.ObjectMappingParametersGenerator request) {
59-
if (subobjects == ObjectMapper.Subobjects.ENABLED) {
60-
// Use default behavior
61-
return null;
62-
}
63-
64-
assert request.isNested() == false;
65-
66-
// "enabled: false" is not compatible with subobjects: false
67-
// "dynamic: false/strict/runtime" is not compatible with subobjects: false
68-
return new DataSourceResponse.ObjectMappingParametersGenerator(() -> {
69-
var parameters = new HashMap<String, Object>();
70-
parameters.put("subobjects", subobjects.toString());
71-
if (ESTestCase.randomBoolean()) {
72-
parameters.put("dynamic", "true");
73-
}
74-
if (ESTestCase.randomBoolean()) {
75-
parameters.put("enabled", "true");
76-
}
77-
return parameters;
78-
});
79-
}
80-
}))
38+
var specificationBuilder = DataGeneratorSpecification.builder()
39+
.withFullyDynamicMapping(ESTestCase.randomBoolean())
8140
.withPredefinedFields(
8241
List.of(
8342
// Customized because it always needs doc_values for aggregations.
@@ -136,11 +95,7 @@ void logsDbMapping(XContentBuilder builder) throws IOException {
13695
}
13796

13897
void standardMapping(XContentBuilder builder) throws IOException {
139-
if (subobjects != ObjectMapper.Subobjects.ENABLED) {
140-
dataGenerator.writeMapping(builder, Map.of("subobjects", subobjects.toString()));
141-
} else {
142-
dataGenerator.writeMapping(builder);
143-
}
98+
dataGenerator.writeMapping(builder);
14499
}
145100

146101
void logsDbSettings(Settings.Builder builder) {

‎server/src/main/java/org/elasticsearch/index/mapper/ObjectMapper.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ public enum Subobjects {
6363
this.printedValue = printedValue;
6464
}
6565

66-
static Subobjects from(Object node) {
66+
public static Subobjects from(Object node) {
6767
if (node instanceof Boolean value) {
6868
return value ? Subobjects.ENABLED : Subobjects.DISABLED;
6969
}

‎test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DataSourceRequest.java‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,10 @@
99

1010
package org.elasticsearch.logsdb.datageneration.datasource;
1111

12+
import org.elasticsearch.index.mapper.ObjectMapper;
1213
import org.elasticsearch.logsdb.datageneration.DataGeneratorSpecification;
1314
import org.elasticsearch.logsdb.datageneration.FieldType;
1415
import org.elasticsearch.logsdb.datageneration.fields.DynamicMapping;
15-
import org.elasticsearch.test.ESTestCase;
1616

1717
import java.util.Set;
1818

@@ -116,15 +116,11 @@ public DataSourceResponse.LeafMappingParametersGenerator accept(DataSourceHandle
116116
}
117117
}
118118

119-
record ObjectMappingParametersGenerator(boolean isRoot, boolean isNested)
119+
record ObjectMappingParametersGenerator(boolean isRoot, boolean isNested, ObjectMapper.Subobjects parentSubobjects)
120120
implements
121121
DataSourceRequest<DataSourceResponse.ObjectMappingParametersGenerator> {
122122
public DataSourceResponse.ObjectMappingParametersGenerator accept(DataSourceHandler handler) {
123123
return handler.handle(this);
124124
}
125-
126-
public String syntheticSourceKeepValue() {
127-
return isRoot() ? ESTestCase.randomFrom("none", "arrays") : ESTestCase.randomFrom("none", "arrays", "all");
128-
}
129125
}
130126
}

‎test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/datasource/DefaultMappingParametersHandler.java‎

Lines changed: 32 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
package org.elasticsearch.logsdb.datageneration.datasource;
1111

1212
import org.elasticsearch.index.mapper.Mapper;
13+
import org.elasticsearch.index.mapper.ObjectMapper;
1314
import org.elasticsearch.logsdb.datageneration.fields.DynamicMapping;
1415
import org.elasticsearch.test.ESTestCase;
1516

@@ -78,8 +79,11 @@ private Supplier<Map<String, Object>> scaledFloatMapping(Map<String, Object> inj
7879
@Override
7980
public DataSourceResponse.ObjectMappingParametersGenerator handle(DataSourceRequest.ObjectMappingParametersGenerator request) {
8081
if (request.isNested()) {
82+
assert request.parentSubobjects() != ObjectMapper.Subobjects.DISABLED;
83+
8184
return new DataSourceResponse.ObjectMappingParametersGenerator(() -> {
8285
var parameters = new HashMap<String, Object>();
86+
8387
if (ESTestCase.randomBoolean()) {
8488
parameters.put("dynamic", ESTestCase.randomFrom("true", "false", "strict"));
8589
}
@@ -93,14 +97,41 @@ public DataSourceResponse.ObjectMappingParametersGenerator handle(DataSourceRequ
9397

9498
return new DataSourceResponse.ObjectMappingParametersGenerator(() -> {
9599
var parameters = new HashMap<String, Object>();
100+
101+
if (request.parentSubobjects() == ObjectMapper.Subobjects.DISABLED) {
102+
// "enabled: false" is not compatible with subobjects: false
103+
// changing "dynamic" from parent context is not compatible with subobjects: false
104+
// changing subobjects value is not compatible with subobjects: false
105+
if (ESTestCase.randomBoolean()) {
106+
parameters.put("enabled", "true");
107+
}
108+
109+
return parameters;
110+
}
111+
96112
if (ESTestCase.randomBoolean()) {
97113
parameters.put("dynamic", ESTestCase.randomFrom("true", "false", "strict", "runtime"));
98114
}
99115
if (ESTestCase.randomBoolean()) {
100116
parameters.put("enabled", ESTestCase.randomFrom("true", "false"));
101117
}
118+
// Changing subobjects from subobjects: false is not supported, but we can f.e. go from "true" to "false".
119+
// TODO enable subobjects: auto
120+
// It is disabled because it currently does not have auto flattening and that results in asserts being triggered when using
121+
// copy_to.
122+
if (ESTestCase.randomBoolean()) {
123+
parameters.put(
124+
"subobjects",
125+
ESTestCase.randomValueOtherThan(
126+
ObjectMapper.Subobjects.AUTO,
127+
() -> ESTestCase.randomFrom(ObjectMapper.Subobjects.values())
128+
).toString()
129+
);
130+
}
131+
102132
if (ESTestCase.randomBoolean()) {
103-
parameters.put(Mapper.SYNTHETIC_SOURCE_KEEP_PARAM, request.syntheticSourceKeepValue());
133+
var value = request.isRoot() ? ESTestCase.randomFrom("none", "arrays") : ESTestCase.randomFrom("none", "arrays", "all");
134+
parameters.put(Mapper.SYNTHETIC_SOURCE_KEEP_PARAM, value);
104135
}
105136

106137
return parameters;

‎test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/Context.java‎

Lines changed: 39 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99

1010
package org.elasticsearch.logsdb.datageneration.fields;
1111

12+
import org.elasticsearch.index.mapper.ObjectMapper;
1213
import org.elasticsearch.logsdb.datageneration.DataGeneratorSpecification;
1314
import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest;
1415
import org.elasticsearch.logsdb.datageneration.datasource.DataSourceResponse;
@@ -31,9 +32,14 @@ class Context {
3132
private final AtomicInteger nestedFieldsCount;
3233
private final Set<String> eligibleCopyToDestinations;
3334
private final DynamicMapping parentDynamicMapping;
35+
private final ObjectMapper.Subobjects currentSubobjectsConfig;
3436

35-
Context(DataGeneratorSpecification specification, DynamicMapping parentDynamicMapping) {
36-
this(specification, "", 0, new AtomicInteger(0), new HashSet<>(), parentDynamicMapping);
37+
Context(
38+
DataGeneratorSpecification specification,
39+
DynamicMapping parentDynamicMapping,
40+
ObjectMapper.Subobjects currentSubobjectsConfig
41+
) {
42+
this(specification, "", 0, new AtomicInteger(0), new HashSet<>(), parentDynamicMapping, currentSubobjectsConfig);
3743
}
3844

3945
private Context(
@@ -42,7 +48,8 @@ private Context(
4248
int objectDepth,
4349
AtomicInteger nestedFieldsCount,
4450
Set<String> eligibleCopyToDestinations,
45-
DynamicMapping parentDynamicMapping
51+
DynamicMapping parentDynamicMapping,
52+
ObjectMapper.Subobjects currentSubobjectsConfig
4653
) {
4754
this.specification = specification;
4855
this.childFieldGenerator = specification.dataSource().get(new DataSourceRequest.ChildFieldGenerator(specification));
@@ -52,6 +59,7 @@ private Context(
5259
this.nestedFieldsCount = nestedFieldsCount;
5360
this.eligibleCopyToDestinations = eligibleCopyToDestinations;
5461
this.parentDynamicMapping = parentDynamicMapping;
62+
this.currentSubobjectsConfig = currentSubobjectsConfig;
5563
}
5664

5765
public DataGeneratorSpecification specification() {
@@ -66,21 +74,30 @@ public DataSourceResponse.FieldTypeGenerator fieldTypeGenerator(DynamicMapping d
6674
return specification.dataSource().get(new DataSourceRequest.FieldTypeGenerator(dynamicMapping));
6775
}
6876

69-
public Context subObject(String name, DynamicMapping dynamicMapping) {
77+
public Context subObject(String name, DynamicMapping dynamicMapping, ObjectMapper.Subobjects subobjects) {
7078
return new Context(
7179
specification,
7280
pathToField(name),
7381
objectDepth + 1,
7482
nestedFieldsCount,
7583
eligibleCopyToDestinations,
76-
dynamicMapping
84+
dynamicMapping,
85+
subobjects
7786
);
7887
}
7988

80-
public Context nestedObject(String name, DynamicMapping dynamicMapping) {
89+
public Context nestedObject(String name, DynamicMapping dynamicMapping, ObjectMapper.Subobjects subobjects) {
8190
nestedFieldsCount.incrementAndGet();
8291
// copy_to can't be used across nested documents so all currently eligible fields are not eligible inside nested document.
83-
return new Context(specification, pathToField(name), objectDepth + 1, nestedFieldsCount, new HashSet<>(), dynamicMapping);
92+
return new Context(
93+
specification,
94+
pathToField(name),
95+
objectDepth + 1,
96+
nestedFieldsCount,
97+
new HashSet<>(),
98+
dynamicMapping,
99+
subobjects
100+
);
84101
}
85102

86103
public boolean shouldAddDynamicObjectField(DynamicMapping dynamicMapping) {
@@ -99,10 +116,11 @@ public boolean shouldAddObjectField() {
99116
return childFieldGenerator.generateRegularSubObject();
100117
}
101118

102-
public boolean shouldAddNestedField() {
119+
public boolean shouldAddNestedField(ObjectMapper.Subobjects subobjects) {
103120
if (objectDepth >= specification.maxObjectDepth()
104121
|| nestedFieldsCount.get() >= specification.nestedFieldsLimit()
105-
|| parentDynamicMapping == DynamicMapping.FORCED) {
122+
|| parentDynamicMapping == DynamicMapping.FORCED
123+
|| subobjects == ObjectMapper.Subobjects.DISABLED) {
106124
return false;
107125
}
108126

@@ -131,6 +149,14 @@ public DynamicMapping determineDynamicMapping(Map<String, Object> mappingParamet
131149
return dynamicParameter.equals("strict") ? DynamicMapping.FORBIDDEN : DynamicMapping.SUPPORTED;
132150
}
133151

152+
public ObjectMapper.Subobjects determineSubobjects(Map<String, Object> mappingParameters) {
153+
if (currentSubobjectsConfig == ObjectMapper.Subobjects.DISABLED) {
154+
return ObjectMapper.Subobjects.DISABLED;
155+
}
156+
157+
return ObjectMapper.Subobjects.from(mappingParameters.getOrDefault("subobjects", "true"));
158+
}
159+
134160
public Set<String> getEligibleCopyToDestinations() {
135161
return eligibleCopyToDestinations;
136162
}
@@ -142,4 +168,8 @@ public void markFieldAsEligibleForCopyTo(String field) {
142168
private String pathToField(String field) {
143169
return path.isEmpty() ? field : path + "." + field;
144170
}
171+
172+
public ObjectMapper.Subobjects getCurrentSubobjectsConfig() {
173+
return currentSubobjectsConfig;
174+
}
145175
}

‎test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/GenericSubObjectFieldDataGenerator.java‎

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
package org.elasticsearch.logsdb.datageneration.fields;
1111

1212
import org.elasticsearch.core.CheckedConsumer;
13+
import org.elasticsearch.index.mapper.ObjectMapper;
1314
import org.elasticsearch.logsdb.datageneration.FieldDataGenerator;
1415
import org.elasticsearch.logsdb.datageneration.FieldType;
1516
import org.elasticsearch.logsdb.datageneration.datasource.DataSourceRequest;
@@ -31,7 +32,7 @@ public class GenericSubObjectFieldDataGenerator {
3132
this.context = context;
3233
}
3334

34-
List<ChildField> generateChildFields(DynamicMapping dynamicMapping) {
35+
List<ChildField> generateChildFields(DynamicMapping dynamicMapping, ObjectMapper.Subobjects subobjects) {
3536
var existingFieldNames = new HashSet<String>();
3637
// no child fields is legal
3738
var childFieldsCount = context.childFieldGenerator().generateChildFieldCount();
@@ -42,12 +43,24 @@ List<ChildField> generateChildFields(DynamicMapping dynamicMapping) {
4243

4344
if (context.shouldAddDynamicObjectField(dynamicMapping)) {
4445
result.add(
45-
new ChildField(fieldName, new ObjectFieldDataGenerator(context.subObject(fieldName, DynamicMapping.FORCED)), true)
46+
new ChildField(
47+
fieldName,
48+
new ObjectFieldDataGenerator(context.subObject(fieldName, DynamicMapping.FORCED, subobjects)),
49+
true
50+
)
4651
);
4752
} else if (context.shouldAddObjectField()) {
48-
result.add(new ChildField(fieldName, new ObjectFieldDataGenerator(context.subObject(fieldName, dynamicMapping)), false));
49-
} else if (context.shouldAddNestedField()) {
50-
result.add(new ChildField(fieldName, new NestedFieldDataGenerator(context.nestedObject(fieldName, dynamicMapping)), false));
53+
result.add(
54+
new ChildField(fieldName, new ObjectFieldDataGenerator(context.subObject(fieldName, dynamicMapping, subobjects)), false)
55+
);
56+
} else if (context.shouldAddNestedField(subobjects)) {
57+
result.add(
58+
new ChildField(
59+
fieldName,
60+
new NestedFieldDataGenerator(context.nestedObject(fieldName, dynamicMapping, subobjects)),
61+
false
62+
)
63+
);
5164
} else {
5265
var fieldTypeInfo = context.fieldTypeGenerator(dynamicMapping).generator().get();
5366

‎test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/NestedFieldDataGenerator.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,13 +28,14 @@ public class NestedFieldDataGenerator implements FieldDataGenerator {
2828

2929
this.mappingParameters = context.specification()
3030
.dataSource()
31-
.get(new DataSourceRequest.ObjectMappingParametersGenerator(false, true))
31+
.get(new DataSourceRequest.ObjectMappingParametersGenerator(false, true, context.getCurrentSubobjectsConfig()))
3232
.mappingGenerator()
3333
.get();
3434
var dynamicMapping = context.determineDynamicMapping(mappingParameters);
35+
var subobjects = context.determineSubobjects(mappingParameters);
3536

3637
var genericGenerator = new GenericSubObjectFieldDataGenerator(context);
37-
this.childFields = genericGenerator.generateChildFields(dynamicMapping);
38+
this.childFields = genericGenerator.generateChildFields(dynamicMapping, subobjects);
3839
}
3940

4041
@Override

‎test/framework/src/main/java/org/elasticsearch/logsdb/datageneration/fields/ObjectFieldDataGenerator.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,13 +28,14 @@ public class ObjectFieldDataGenerator implements FieldDataGenerator {
2828

2929
this.mappingParameters = context.specification()
3030
.dataSource()
31-
.get(new DataSourceRequest.ObjectMappingParametersGenerator(false, false))
31+
.get(new DataSourceRequest.ObjectMappingParametersGenerator(false, false, context.getCurrentSubobjectsConfig()))
3232
.mappingGenerator()
3333
.get();
3434
var dynamicMapping = context.determineDynamicMapping(mappingParameters);
35+
var subobjects = context.determineSubobjects(mappingParameters);
3536

3637
var genericGenerator = new GenericSubObjectFieldDataGenerator(context);
37-
this.childFields = genericGenerator.generateChildFields(dynamicMapping);
38+
this.childFields = genericGenerator.generateChildFields(dynamicMapping, subobjects);
3839
}
3940

4041
@Override

0 commit comments

Comments
 (0)