Skip to content

Commit e736f1c

Browse files
pabloempabloem
authored
Merge branch 'main' into pem-unwanted-test-tsrateit
2 parents 5b6d7dc + ae7bfd6 commit e736f1c

File tree

115 files changed

+2181
-656
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

115 files changed

+2181
-656
lines changed

‎docs/changelog/133166.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 133166
2+
summary: Improve Expanding Lookup Join performance by pushing a filter to the right
3+
side of the lookup join
4+
area: ES|QL
5+
type: enhancement
6+
issues: []

‎libs/core/src/main/java/org/elasticsearch/core/TimeValue.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -340,7 +340,7 @@ private static String formatDecimal(double value, int fractionPieces) {
340340
}
341341

342342
public String getStringRep() {
343-
if (duration < 0) {
343+
if (duration < 0 && TimeUnit.MILLISECONDS == timeUnit) {
344344
return Long.toString(duration);
345345
}
346346
return switch (timeUnit) {

‎libs/core/src/test/java/org/elasticsearch/common/unit/TimeValueTests.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,7 @@ public void testRoundTrip() {
109109
assertThat(TimeValue.parseTimeValue(s, null, "test").getStringRep(), equalTo(s));
110110
final TimeValue t = new TimeValue(randomIntBetween(1, 128), randomFrom(TimeUnit.values()));
111111
assertThat(TimeValue.parseTimeValue(t.getStringRep(), null, "test"), equalTo(t));
112+
assertThat(TimeValue.timeValueSeconds(-1), equalTo(TimeValue.parseTimeValue(TimeValue.timeValueSeconds(-1).getStringRep(), "foo")));
112113
}
113114

114115
private static final String FRACTIONAL_TIME_VALUES_ARE_NOT_SUPPORTED = "fractional time values are not supported";

‎modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java‎

Lines changed: 27 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -2429,24 +2429,20 @@ public void testGetEffectiveMappings() throws Exception {
24292429
{
24302430
ComponentTemplate ct1 = new ComponentTemplate(new Template(null, new CompressedXContent("""
24312431
{
2432-
"_doc":{
2433-
"dynamic":"strict",
2434-
"properties":{
2435-
"field1":{
2436-
"type":"text"
2437-
}
2432+
"dynamic":"strict",
2433+
"properties":{
2434+
"field1":{
2435+
"type":"text"
24382436
}
24392437
}
24402438
}
24412439
"""), null), 3L, null);
24422440
ComponentTemplate ct2 = new ComponentTemplate(new Template(null, new CompressedXContent("""
24432441
{
2444-
"_doc":{
2445-
"dynamic":"strict",
2446-
"properties":{
2447-
"field2":{
2448-
"type":"text"
2449-
}
2442+
"dynamic":"strict",
2443+
"properties":{
2444+
"field2":{
2445+
"type":"text"
24502446
}
24512447
}
24522448
}
@@ -2464,12 +2460,10 @@ public void testGetEffectiveMappings() throws Exception {
24642460
.indexPatterns(List.of("effective-*"))
24652461
.template(Template.builder().mappings(CompressedXContent.fromJSON("""
24662462
{
2467-
"_doc":{
2468-
"dynamic":"strict",
2469-
"properties":{
2470-
"field3":{
2471-
"type":"text"
2472-
}
2463+
"dynamic":"strict",
2464+
"properties":{
2465+
"field3":{
2466+
"type":"text"
24732467
}
24742468
}
24752469
}
@@ -2530,25 +2524,22 @@ public void testGetEffectiveMappings() throws Exception {
25302524
Map<String, Object> effectiveMappingMap = XContentHelper.convertToMap(effectiveMappings.uncompressed(), true, XContentType.JSON)
25312525
.v2();
25322526
Map<String, Object> expectedEffectiveMappingMap = Map.of(
2533-
"_doc",
2527+
"dynamic",
2528+
"strict",
2529+
"_data_stream_timestamp",
2530+
Map.of("enabled", true),
2531+
"properties",
25342532
Map.of(
2535-
"dynamic",
2536-
"strict",
2537-
"_data_stream_timestamp",
2538-
Map.of("enabled", true),
2539-
"properties",
2540-
Map.of(
2541-
"@timestamp",
2542-
Map.of("type", "date"),
2543-
"field1",
2544-
Map.of("type", "keyword"),
2545-
"field2",
2546-
Map.of("type", "text"),
2547-
"field3",
2548-
Map.of("type", "text"),
2549-
"field4",
2550-
Map.of("type", "keyword")
2551-
)
2533+
"@timestamp",
2534+
Map.of("type", "date"),
2535+
"field1",
2536+
Map.of("type", "keyword"),
2537+
"field2",
2538+
Map.of("type", "text"),
2539+
"field3",
2540+
Map.of("type", "text"),
2541+
"field4",
2542+
Map.of("type", "keyword")
25522543
)
25532544
);
25542545
assertThat(effectiveMappingMap, equalTo(expectedEffectiveMappingMap));

‎modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TransportUpdateDataStreamMappingsActionIT.java‎

Lines changed: 26 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -55,38 +55,32 @@ public void testGetAndUpdateMappings() throws IOException {
5555
createDataStream(dataStreamName);
5656

5757
Map<String, Object> originalMappings = Map.of(
58-
"_doc",
59-
Map.of(
60-
"dynamic",
61-
"strict",
62-
"_data_stream_timestamp",
63-
Map.of("enabled", true),
64-
"properties",
65-
Map.of("@timestamp", Map.of("type", "date"), "foo1", Map.of("type", "text"), "foo2", Map.of("type", "text"))
66-
)
58+
"dynamic",
59+
"strict",
60+
"_data_stream_timestamp",
61+
Map.of("enabled", true),
62+
"properties",
63+
Map.of("@timestamp", Map.of("type", "date"), "foo1", Map.of("type", "text"), "foo2", Map.of("type", "text"))
6764
);
6865
Map<String, Object> mappingOverrides = Map.of(
6966
"properties",
7067
Map.of("foo2", Map.of("type", "keyword"), "foo3", Map.of("type", "text"))
7168
);
7269
Map<String, Object> expectedEffectiveMappings = Map.of(
73-
"_doc",
70+
"dynamic",
71+
"strict",
72+
"_data_stream_timestamp",
73+
Map.of("enabled", true),
74+
"properties",
7475
Map.of(
75-
"dynamic",
76-
"strict",
77-
"_data_stream_timestamp",
78-
Map.of("enabled", true),
79-
"properties",
80-
Map.of(
81-
"@timestamp",
82-
Map.of("type", "date"),
83-
"foo1",
84-
Map.of("type", "text"),
85-
"foo2",
86-
Map.of("type", "keyword"),
87-
"foo3",
88-
Map.of("type", "text")
89-
)
76+
"@timestamp",
77+
Map.of("type", "date"),
78+
"foo1",
79+
Map.of("type", "text"),
80+
"foo2",
81+
Map.of("type", "keyword"),
82+
"foo3",
83+
Map.of("type", "text")
9084
)
9185
);
9286
assertExpectedMappings(dataStreamName, Map.of(), originalMappings);
@@ -137,15 +131,13 @@ public void testGetAndUpdateMappings() throws IOException {
137131
private void createDataStream(String dataStreamName) throws IOException {
138132
String mappingString = """
139133
{
140-
"_doc":{
141-
"dynamic":"strict",
142-
"properties":{
143-
"foo1":{
144-
"type":"text"
145-
},
146-
"foo2":{
147-
"type":"text"
148-
}
134+
"dynamic":"strict",
135+
"properties":{
136+
"foo1":{
137+
"type":"text"
138+
},
139+
"foo2":{
140+
"type":"text"
149141
}
150142
}
151143
}

‎modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/250_data_stream_mappings.yml‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ setup:
3434
name: my-data-stream-1
3535
- match: { data_streams.0.name: my-data-stream-1 }
3636
- match: { data_streams.0.mappings: {} }
37-
- length: { data_streams.0.effective_mappings._doc.properties: 2 }
37+
- length: { data_streams.0.effective_mappings.properties: 2 }
3838

3939
- do:
4040
indices.get_data_stream:
@@ -56,7 +56,7 @@ setup:
5656
- match: { data_streams.0.name: my-data-stream-1 }
5757
- match: { data_streams.0.applied_to_data_stream: true }
5858
- match: { data_streams.0.mappings.properties.name.type: "keyword" }
59-
- match: { data_streams.0.effective_mappings._doc.properties.name.type: "keyword" }
59+
- match: { data_streams.0.effective_mappings.properties.name.type: "keyword" }
6060

6161
- do:
6262
indices.rollover:
@@ -71,9 +71,9 @@ setup:
7171
indices.get_data_stream_mappings:
7272
name: my-data-stream-1
7373
- match: { data_streams.0.name: my-data-stream-1 }
74-
- length: { data_streams.0.effective_mappings._doc.properties: 3 }
74+
- length: { data_streams.0.effective_mappings.properties: 3 }
7575
- match: { data_streams.0.mappings.properties.name.type: "keyword" }
76-
- match: { data_streams.0.effective_mappings._doc.properties.name.type: "keyword" }
76+
- match: { data_streams.0.effective_mappings.properties.name.type: "keyword" }
7777

7878
- do:
7979
indices.get_data_stream:
@@ -149,9 +149,9 @@ setup:
149149
- match: { data_streams.0.mappings.properties.field1.type: "text" }
150150
- match: { data_streams.0.mappings.properties.field2: null }
151151
- match: { data_streams.0.mappings.properties.field3.type: "text" }
152-
- match: { data_streams.0.effective_mappings._doc.properties.field1.type: "text" }
153-
- match: { data_streams.0.effective_mappings._doc.properties.field2.type: "keyword" }
154-
- match: { data_streams.0.effective_mappings._doc.properties.field3.type: "text" }
152+
- match: { data_streams.0.effective_mappings.properties.field1.type: "text" }
153+
- match: { data_streams.0.effective_mappings.properties.field2.type: "keyword" }
154+
- match: { data_streams.0.effective_mappings.properties.field3.type: "text" }
155155

156156
- do:
157157
cluster.put_component_template:
@@ -180,10 +180,10 @@ setup:
180180
indices.get_data_stream_mappings:
181181
name: my-component-only-data-stream-1
182182
- match: { data_streams.0.name: my-component-only-data-stream-1 }
183-
- length: { data_streams.0.effective_mappings._doc.properties: 5 }
183+
- length: { data_streams.0.effective_mappings.properties: 5 }
184184
- match: { data_streams.0.mappings.properties.field1.type: "text" }
185-
- match: { data_streams.0.effective_mappings._doc.properties.field3.type: "text" }
186-
- match: { data_streams.0.effective_mappings._doc.properties.field4.type: "keyword" }
185+
- match: { data_streams.0.effective_mappings.properties.field3.type: "text" }
186+
- match: { data_streams.0.effective_mappings.properties.field4.type: "keyword" }
187187

188188
- do:
189189
indices.get_data_stream:
@@ -212,7 +212,7 @@ setup:
212212
- match: { data_streams.0.name: my-component-only-data-stream-1 }
213213
- match: { data_streams.0.applied_to_data_stream: true }
214214
- match: { data_streams.0.mappings null }
215-
- match: { data_streams.0.effective_mappings._doc.properties.field1.type: "keyword" }
216-
- match: { data_streams.0.effective_mappings._doc.properties.field2.type: "keyword" }
217-
- match: { data_streams.0.effective_mappings._doc.properties.field3: null }
218-
- match: { data_streams.0.effective_mappings._doc.properties.field4.type: "keyword" }
215+
- match: { data_streams.0.effective_mappings.properties.field1.type: "keyword" }
216+
- match: { data_streams.0.effective_mappings.properties.field2.type: "keyword" }
217+
- match: { data_streams.0.effective_mappings.properties.field3: null }
218+
- match: { data_streams.0.effective_mappings.properties.field4.type: "keyword" }

‎muted-tests.yml‎

Lines changed: 0 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -495,24 +495,9 @@ tests:
495495
- class: org.elasticsearch.xpack.esql.action.CrossClusterAsyncQueryStopIT
496496
method: testStopQueryLocal
497497
issue: https://github.com/elastic/elasticsearch/issues/133481
498-
- class: org.elasticsearch.compute.data.BasicBlockTests
499-
method: testIntBlock
500-
issue: https://github.com/elastic/elasticsearch/issues/133596
501-
- class: org.elasticsearch.compute.data.BasicBlockTests
502-
method: testDoubleBlock
503-
issue: https://github.com/elastic/elasticsearch/issues/133606
504-
- class: org.elasticsearch.compute.data.BasicBlockTests
505-
method: testBooleanBlock
506-
issue: https://github.com/elastic/elasticsearch/issues/133608
507-
- class: org.elasticsearch.compute.data.BasicBlockTests
508-
method: testLongBlock
509-
issue: https://github.com/elastic/elasticsearch/issues/133618
510498
- class: org.elasticsearch.xpack.esql.inference.rerank.RerankOperatorTests
511499
method: testSimpleCircuitBreaking
512500
issue: https://github.com/elastic/elasticsearch/issues/133619
513-
- class: org.elasticsearch.compute.data.BasicBlockTests
514-
method: testFloatBlock
515-
issue: https://github.com/elastic/elasticsearch/issues/133621
516501
- class: org.elasticsearch.xpack.esql.qa.single_node.GenerativeIT
517502
method: test
518503
issue: https://github.com/elastic/elasticsearch/issues/133077

‎server/src/main/java/org/elasticsearch/TransportVersions.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -357,6 +357,7 @@ static TransportVersion def(int id) {
357357
public static final TransportVersion STREAMS_ENDPOINT_PARAM_RESTRICTIONS = def(9_148_0_00);
358358
public static final TransportVersion RESOLVE_INDEX_MODE_FILTER = def(9_149_0_00);
359359
public static final TransportVersion SEMANTIC_QUERY_MULTIPLE_INFERENCE_IDS = def(9_150_0_00);
360+
public static final TransportVersion ESQL_LOOKUP_JOIN_PRE_JOIN_FILTER = def(9_151_0_00);
360361

361362
/*
362363
* STOP! READ THIS FIRST! No, really,

‎server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java‎

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,6 @@
4646
import org.elasticsearch.index.IndexMode;
4747
import org.elasticsearch.index.IndexSettings;
4848
import org.elasticsearch.index.mapper.DateFieldMapper;
49-
import org.elasticsearch.index.mapper.DocumentMapper;
5049
import org.elasticsearch.index.mapper.MapperService;
5150
import org.elasticsearch.indices.IndicesService;
5251
import org.elasticsearch.indices.SystemIndices;
@@ -500,13 +499,20 @@ public static CompressedXContent getEffectiveMappings(
500499
writeIndex.getName()
501500
);
502501
return indicesService.withTempIndexService(projectMetadata.index(writeIndex), indexService -> {
503-
MapperService mapperService = indexService.mapperService();
504-
DocumentMapper documentMapper = mapperService.merge(
505-
MapperService.SINGLE_MAPPING_NAME,
506-
mappings,
507-
MapperService.MergeReason.INDEX_TEMPLATE
508-
);
509-
return documentMapper.mappingSource();
502+
CompressedXContent mergedMapping = indexService.mapperService()
503+
.merge(MapperService.SINGLE_MAPPING_NAME, mappings, MapperService.MergeReason.INDEX_TEMPLATE)
504+
.mappingSource();
505+
/*
506+
* If the merged mapping contains the old "_doc" type placeholder, we remove it to make things more straightforward for the
507+
* client:
508+
*/
509+
Map<String, Object> mergedMappingMap = XContentHelper.convertToMap(mergedMapping.uncompressed(), true, XContentType.JSON).v2();
510+
if (mergedMappingMap.containsKey(MapperService.SINGLE_MAPPING_NAME)) {
511+
mergedMapping = ComposableIndexTemplate.convertMappingMapToXContent(
512+
(Map<String, ?>) mergedMappingMap.get(MapperService.SINGLE_MAPPING_NAME)
513+
);
514+
}
515+
return mergedMapping;
510516
});
511517
}
512518

‎server/src/main/java/org/elasticsearch/index/IndexingSlowLog.java‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -35,29 +35,29 @@ public final class IndexingSlowLog implements IndexingOperationListener {
3535
public static final String INDEX_INDEXING_SLOWLOG_PREFIX = "index.indexing.slowlog";
3636
public static final Setting<TimeValue> INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_WARN_SETTING = Setting.timeSetting(
3737
INDEX_INDEXING_SLOWLOG_PREFIX + ".threshold.index.warn",
38-
TimeValue.timeValueNanos(-1),
39-
TimeValue.timeValueMillis(-1),
38+
TimeValue.MINUS_ONE,
39+
TimeValue.MINUS_ONE,
4040
Property.Dynamic,
4141
Property.IndexScope
4242
);
4343
public static final Setting<TimeValue> INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_INFO_SETTING = Setting.timeSetting(
4444
INDEX_INDEXING_SLOWLOG_PREFIX + ".threshold.index.info",
45-
TimeValue.timeValueNanos(-1),
46-
TimeValue.timeValueMillis(-1),
45+
TimeValue.MINUS_ONE,
46+
TimeValue.MINUS_ONE,
4747
Property.Dynamic,
4848
Property.IndexScope
4949
);
5050
public static final Setting<TimeValue> INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_DEBUG_SETTING = Setting.timeSetting(
5151
INDEX_INDEXING_SLOWLOG_PREFIX + ".threshold.index.debug",
52-
TimeValue.timeValueNanos(-1),
53-
TimeValue.timeValueMillis(-1),
52+
TimeValue.MINUS_ONE,
53+
TimeValue.MINUS_ONE,
5454
Property.Dynamic,
5555
Property.IndexScope
5656
);
5757
public static final Setting<TimeValue> INDEX_INDEXING_SLOWLOG_THRESHOLD_INDEX_TRACE_SETTING = Setting.timeSetting(
5858
INDEX_INDEXING_SLOWLOG_PREFIX + ".threshold.index.trace",
59-
TimeValue.timeValueNanos(-1),
60-
TimeValue.timeValueMillis(-1),
59+
TimeValue.MINUS_ONE,
60+
TimeValue.MINUS_ONE,
6161
Property.Dynamic,
6262
Property.IndexScope
6363
);

0 commit comments

Comments
 (0)