Skip to content

Commit e7ba191

Browse files
julian-elasticjulian-elastic
committed
Merge branch 'main' into indexLikeBackport_9_20
2 parents d1c4f50 + 8bcb0c4 commit e7ba191

File tree

164 files changed

+3054
-1081
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

164 files changed

+3054
-1081
lines changed

‎README.asciidoc‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -275,7 +275,7 @@ For the complete Elasticsearch documentation visit
275275
https://www.elastic.co/guide/en/elasticsearch/reference/current/index.html[elastic.co].
276276

277277
For information about our documentation processes, see the
278-
xref:docs/README.asciidoc[docs README].
278+
xref:https://github.com/elastic/elasticsearch/blob/main/docs/README.md[docs README].
279279

280280
[[examples]]
281281
== Examples and guides

‎docs/changelog/129013.yaml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
pr: 129013
2+
summary: "Add remote index support to LOOKUP JOIN"
3+
area: ES|QL
4+
type: feature
5+
issues: [ ]
6+
highlight:
7+
title: Add remote index support to LOOKUP JOIN
8+
body: |-
9+
Queries containing LOOKUP JOIN now can be preformed on cross-cluster indices, for example:
10+
[source,yaml]
11+
----------------------------
12+
FROM logs-*, remote:logs-* | LOOKUP JOIN clients on ip | SORT timestamp | LIMIT 100
13+
----------------------------

‎docs/changelog/130593.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 130593
2+
summary: Add new `CachePopulationReason`
3+
area: Store
4+
type: enhancement
5+
issues: []

‎docs/changelog/130776.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 130776
2+
summary: Fix msearch request parsing when index expression is null
3+
area: Search
4+
type: bug
5+
issues:
6+
- 129631

‎docs/changelog/130924.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 130924
2+
summary: Check field data type before casting when applying geo distance sort
3+
area: Search
4+
type: bug
5+
issues:
6+
- 129500

‎docs/reference/elasticsearch/index.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Elasticsearch and index management
1+
# Elasticsearch
22

33
This section contains reference information for {{es}} and index management features.
44

@@ -7,7 +7,7 @@ To learn more about {{es}} features and how to get started, refer to the [{{es}}
77
For more details about query and scripting languages, check these sections:
88
* [Query languages](../query-languages/index.md)
99
* [Scripting languages](../scripting-languages/index.md)
10-
10+
1111
{{es}} also provides the following REST APIs:
1212

1313
* [{{es}} API](https://www.elastic.co/docs/api/doc/elasticsearch)

‎docs/reference/elasticsearch/rest-apis/index.md‎

Lines changed: 664 additions & 0 deletions
Large diffs are not rendered by default.

‎docs/reference/search-connectors/elastic-managed-connectors.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
---
22
applies_to:
3-
ess: discontinued 9.0.0
3+
ess: removed 9.0.0
44
mapped_pages:
55
- https://www.elastic.co/guide/en/elasticsearch/reference/current/es-native-connectors.html
66
---

‎modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java‎

Lines changed: 164 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
import org.elasticsearch.ElasticsearchStatusException;
1313
import org.elasticsearch.ExceptionsHelper;
1414
import org.elasticsearch.action.ActionFuture;
15+
import org.elasticsearch.action.ActionType;
1516
import org.elasticsearch.action.DocWriteRequest;
1617
import org.elasticsearch.action.DocWriteResponse;
1718
import org.elasticsearch.action.RequestBuilder;
@@ -41,6 +42,7 @@
4142
import org.elasticsearch.action.admin.indices.stats.ShardStats;
4243
import org.elasticsearch.action.admin.indices.template.delete.TransportDeleteComposableIndexTemplateAction;
4344
import org.elasticsearch.action.admin.indices.template.get.GetComposableIndexTemplateAction;
45+
import org.elasticsearch.action.admin.indices.template.put.PutComponentTemplateAction;
4446
import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
4547
import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction;
4648
import org.elasticsearch.action.admin.indices.validate.query.ValidateQueryRequestBuilder;
@@ -52,6 +54,7 @@
5254
import org.elasticsearch.action.datastreams.GetDataStreamAction;
5355
import org.elasticsearch.action.datastreams.GetDataStreamAction.Response.DataStreamInfo;
5456
import org.elasticsearch.action.datastreams.ModifyDataStreamsAction;
57+
import org.elasticsearch.action.datastreams.UpdateDataStreamMappingsAction;
5558
import org.elasticsearch.action.delete.DeleteRequest;
5659
import org.elasticsearch.action.index.IndexRequest;
5760
import org.elasticsearch.action.search.MultiSearchRequestBuilder;
@@ -62,6 +65,7 @@
6265
import org.elasticsearch.cluster.ClusterStateUpdateTask;
6366
import org.elasticsearch.cluster.health.ClusterHealthStatus;
6467
import org.elasticsearch.cluster.metadata.AliasMetadata;
68+
import org.elasticsearch.cluster.metadata.ComponentTemplate;
6569
import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
6670
import org.elasticsearch.cluster.metadata.DataStream;
6771
import org.elasticsearch.cluster.metadata.DataStreamAction;
@@ -72,20 +76,24 @@
7276
import org.elasticsearch.cluster.metadata.IndexMetadataStats;
7377
import org.elasticsearch.cluster.metadata.IndexWriteLoad;
7478
import org.elasticsearch.cluster.metadata.Metadata;
79+
import org.elasticsearch.cluster.metadata.ProjectId;
80+
import org.elasticsearch.cluster.metadata.ProjectMetadata;
7581
import org.elasticsearch.cluster.metadata.Template;
7682
import org.elasticsearch.cluster.routing.IndexRoutingTable;
7783
import org.elasticsearch.cluster.routing.ShardRouting;
7884
import org.elasticsearch.cluster.service.ClusterService;
7985
import org.elasticsearch.common.Strings;
8086
import org.elasticsearch.common.compress.CompressedXContent;
8187
import org.elasticsearch.common.settings.Settings;
88+
import org.elasticsearch.common.xcontent.XContentHelper;
8289
import org.elasticsearch.core.Nullable;
8390
import org.elasticsearch.index.Index;
8491
import org.elasticsearch.index.IndexNotFoundException;
8592
import org.elasticsearch.index.mapper.DataStreamTimestampFieldMapper;
8693
import org.elasticsearch.index.mapper.DateFieldMapper;
8794
import org.elasticsearch.index.query.TermQueryBuilder;
8895
import org.elasticsearch.index.shard.IndexingStats;
96+
import org.elasticsearch.indices.IndicesService;
8997
import org.elasticsearch.indices.InvalidAliasNameException;
9098
import org.elasticsearch.indices.InvalidIndexNameException;
9199
import org.elasticsearch.plugins.Plugin;
@@ -2411,6 +2419,162 @@ public void testShardSizeIsForecastedDuringRollover() throws Exception {
24112419
assertThat(forecastedShardSizeInBytes.getAsLong(), is(equalTo(expectedTotalSizeInBytes / shardCount)));
24122420
}
24132421

2422+
public void testGetEffectiveMappings() throws Exception {
2423+
/*
2424+
* This test creates a composable template with a mapping and with two component templates with mappings. It then makes sure that
2425+
* DataStream.getEffectiveMappings returns a mapping that merges the template's mapping, the component templates' mappings, and the
2426+
* mapping override given. It then makes sure we get the same result calling the non-static version of getEffectiveMappings.
2427+
*/
2428+
ComposableIndexTemplate composableIndexTemplate;
2429+
{
2430+
ComponentTemplate ct1 = new ComponentTemplate(new Template(null, new CompressedXContent("""
2431+
{
2432+
"_doc":{
2433+
"dynamic":"strict",
2434+
"properties":{
2435+
"field1":{
2436+
"type":"text"
2437+
}
2438+
}
2439+
}
2440+
}
2441+
"""), null), 3L, null);
2442+
ComponentTemplate ct2 = new ComponentTemplate(new Template(null, new CompressedXContent("""
2443+
{
2444+
"_doc":{
2445+
"dynamic":"strict",
2446+
"properties":{
2447+
"field2":{
2448+
"type":"text"
2449+
}
2450+
}
2451+
}
2452+
}
2453+
"""), null), 3L, null);
2454+
client().execute(PutComponentTemplateAction.INSTANCE, new PutComponentTemplateAction.Request("ct1").componentTemplate(ct1))
2455+
.get();
2456+
client().execute(PutComponentTemplateAction.INSTANCE, new PutComponentTemplateAction.Request("ct2").componentTemplate(ct2))
2457+
.get();
2458+
2459+
List<String> componentTemplates = List.of("ct1", "ct2");
2460+
String templateName = "effective-mapping-template";
2461+
TransportPutComposableIndexTemplateAction.Request request = new TransportPutComposableIndexTemplateAction.Request(templateName);
2462+
request.indexTemplate(
2463+
ComposableIndexTemplate.builder()
2464+
.indexPatterns(List.of("effective-*"))
2465+
.template(Template.builder().mappings(CompressedXContent.fromJSON("""
2466+
{
2467+
"_doc":{
2468+
"dynamic":"strict",
2469+
"properties":{
2470+
"field3":{
2471+
"type":"text"
2472+
}
2473+
}
2474+
}
2475+
}
2476+
""")))
2477+
.componentTemplates(componentTemplates)
2478+
.dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate())
2479+
.build()
2480+
);
2481+
client().execute(TransportPutComposableIndexTemplateAction.TYPE, request).actionGet();
2482+
GetComposableIndexTemplateAction.Response getTemplateResponse = client().execute(
2483+
GetComposableIndexTemplateAction.INSTANCE,
2484+
new GetComposableIndexTemplateAction.Request(TEST_REQUEST_TIMEOUT, templateName)
2485+
).actionGet();
2486+
composableIndexTemplate = getTemplateResponse.indexTemplates().values().iterator().next();
2487+
}
2488+
// The mappingOverrides changes the type of one field, and adds another field:
2489+
CompressedXContent mappingOverrides = new CompressedXContent("""
2490+
{
2491+
"properties":{
2492+
"field1":{
2493+
"type":"keyword"
2494+
},
2495+
"field4":{
2496+
"type":"keyword"
2497+
}
2498+
}
2499+
}
2500+
""");
2501+
2502+
String dataStreamName = "effective-mappings-test";
2503+
Index writeIndex;
2504+
{
2505+
CreateDataStreamAction.Request createDataStreamRequest = new CreateDataStreamAction.Request(
2506+
TEST_REQUEST_TIMEOUT,
2507+
TEST_REQUEST_TIMEOUT,
2508+
dataStreamName
2509+
);
2510+
client().execute(CreateDataStreamAction.INSTANCE, createDataStreamRequest).get();
2511+
writeIndex = getDataStream(dataStreamName).getWriteIndex();
2512+
}
2513+
2514+
ProjectMetadata projectMetadata = client().admin()
2515+
.cluster()
2516+
.state(new ClusterStateRequest(TEST_REQUEST_TIMEOUT))
2517+
.get()
2518+
.getState()
2519+
.metadata()
2520+
.getProject(ProjectId.DEFAULT);
2521+
IndicesService indicesService = internalCluster().getInstance(IndicesService.class);
2522+
CompressedXContent effectiveMappings = DataStream.getEffectiveMappings(
2523+
projectMetadata,
2524+
composableIndexTemplate,
2525+
mappingOverrides,
2526+
writeIndex,
2527+
indicesService
2528+
);
2529+
assertNotNull(effectiveMappings);
2530+
Map<String, Object> effectiveMappingMap = XContentHelper.convertToMap(effectiveMappings.uncompressed(), true, XContentType.JSON)
2531+
.v2();
2532+
Map<String, Object> expectedEffectiveMappingMap = Map.of(
2533+
"_doc",
2534+
Map.of(
2535+
"dynamic",
2536+
"strict",
2537+
"_data_stream_timestamp",
2538+
Map.of("enabled", true),
2539+
"properties",
2540+
Map.of(
2541+
"@timestamp",
2542+
Map.of("type", "date"),
2543+
"field1",
2544+
Map.of("type", "keyword"),
2545+
"field2",
2546+
Map.of("type", "text"),
2547+
"field3",
2548+
Map.of("type", "text"),
2549+
"field4",
2550+
Map.of("type", "keyword")
2551+
)
2552+
)
2553+
);
2554+
assertThat(effectiveMappingMap, equalTo(expectedEffectiveMappingMap));
2555+
2556+
// Add the same mappingOverrides to the data stream:
2557+
client().execute(
2558+
new ActionType<UpdateDataStreamMappingsAction.Response>(UpdateDataStreamMappingsAction.NAME),
2559+
new UpdateDataStreamMappingsAction.Request(mappingOverrides, false, TEST_REQUEST_TIMEOUT, TEST_REQUEST_TIMEOUT).indices(
2560+
dataStreamName
2561+
)
2562+
).actionGet();
2563+
assertThat(getDataStream(dataStreamName).getEffectiveMappings(projectMetadata, indicesService), equalTo(effectiveMappings));
2564+
}
2565+
2566+
private DataStream getDataStream(String dataStreamName) throws ExecutionException, InterruptedException {
2567+
return client().admin()
2568+
.cluster()
2569+
.state(new ClusterStateRequest(TEST_REQUEST_TIMEOUT))
2570+
.get()
2571+
.getState()
2572+
.getMetadata()
2573+
.getProject(ProjectId.DEFAULT)
2574+
.dataStreams()
2575+
.get(dataStreamName);
2576+
}
2577+
24142578
private void indexDocsAndEnsureThereIsCapturedWriteLoad(String dataStreamName) throws Exception {
24152579
assertBusy(() -> {
24162580
for (int i = 0; i < 10; i++) {

‎modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/TransportUpdateDataStreamMappingsActionIT.java‎

Lines changed: 27 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -55,20 +55,39 @@ public void testGetAndUpdateMappings() throws IOException {
5555
createDataStream(dataStreamName);
5656

5757
Map<String, Object> originalMappings = Map.of(
58-
"dynamic",
59-
"strict",
60-
"properties",
61-
Map.of("foo1", Map.of("type", "text"), "foo2", Map.of("type", "text"))
58+
"_doc",
59+
Map.of(
60+
"dynamic",
61+
"strict",
62+
"_data_stream_timestamp",
63+
Map.of("enabled", true),
64+
"properties",
65+
Map.of("@timestamp", Map.of("type", "date"), "foo1", Map.of("type", "text"), "foo2", Map.of("type", "text"))
66+
)
6267
);
6368
Map<String, Object> mappingOverrides = Map.of(
6469
"properties",
6570
Map.of("foo2", Map.of("type", "keyword"), "foo3", Map.of("type", "text"))
6671
);
6772
Map<String, Object> expectedEffectiveMappings = Map.of(
68-
"dynamic",
69-
"strict",
70-
"properties",
71-
Map.of("foo1", Map.of("type", "text"), "foo2", Map.of("type", "keyword"), "foo3", Map.of("type", "text"))
73+
"_doc",
74+
Map.of(
75+
"dynamic",
76+
"strict",
77+
"_data_stream_timestamp",
78+
Map.of("enabled", true),
79+
"properties",
80+
Map.of(
81+
"@timestamp",
82+
Map.of("type", "date"),
83+
"foo1",
84+
Map.of("type", "text"),
85+
"foo2",
86+
Map.of("type", "keyword"),
87+
"foo3",
88+
Map.of("type", "text")
89+
)
90+
)
7291
);
7392
assertExpectedMappings(dataStreamName, Map.of(), originalMappings);
7493
updateMappings(dataStreamName, mappingOverrides, expectedEffectiveMappings, true);

0 commit comments

Comments
 (0)