Skip to content

Commit 250c40b

Browse files
benwtrentbenwtrent
authored
Merge branch 'main' into bugfix/address-source-confirmed-npe
2 parents c9954ab + e054e1c commit 250c40b

File tree

45 files changed

+1744
-236
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

45 files changed

+1744
-236
lines changed

‎docs/changelog/126091.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 126091
2+
summary: Allow balancing weights to be set per tier
3+
area: Allocation
4+
type: enhancement
5+
issues: []

‎modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -258,15 +258,15 @@ static GetDataStreamAction.Response innerOperation(
258258
} else {
259259
indexTemplate = MetadataIndexTemplateService.findV2Template(state.metadata(), dataStream.getName(), false);
260260
if (indexTemplate != null) {
261-
Settings settings = MetadataIndexTemplateService.resolveSettings(state.metadata(), indexTemplate);
261+
Settings settings = dataStream.getEffectiveSettings(state.metadata());
262262
ilmPolicyName = settings.get(IndexMetadata.LIFECYCLE_NAME);
263263
if (indexMode == null && state.metadata().templatesV2().get(indexTemplate) != null) {
264264
indexMode = resolveMode(
265265
state,
266266
indexSettingProviders,
267267
dataStream,
268268
settings,
269-
state.metadata().templatesV2().get(indexTemplate)
269+
dataStream.getEffectiveIndexTemplate(state.metadata())
270270
);
271271
}
272272
indexTemplatePreferIlmValue = PREFER_ILM_SETTING.get(settings);

‎modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java‎

Lines changed: 128 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,14 +11,18 @@
1111
import org.elasticsearch.action.datastreams.GetDataStreamAction;
1212
import org.elasticsearch.cluster.ClusterName;
1313
import org.elasticsearch.cluster.ClusterState;
14+
import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
1415
import org.elasticsearch.cluster.metadata.DataStream;
1516
import org.elasticsearch.cluster.metadata.DataStreamFailureStoreSettings;
1617
import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention;
1718
import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionSettings;
1819
import org.elasticsearch.cluster.metadata.DataStreamTestHelper;
20+
import org.elasticsearch.cluster.metadata.IndexMetadata;
1921
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
2022
import org.elasticsearch.cluster.metadata.Metadata;
23+
import org.elasticsearch.cluster.metadata.ProjectId;
2124
import org.elasticsearch.cluster.metadata.ProjectMetadata;
25+
import org.elasticsearch.cluster.metadata.Template;
2226
import org.elasticsearch.common.settings.ClusterSettings;
2327
import org.elasticsearch.common.settings.Settings;
2428
import org.elasticsearch.core.TimeValue;
@@ -27,15 +31,20 @@
2731
import org.elasticsearch.index.IndexMode;
2832
import org.elasticsearch.index.IndexNotFoundException;
2933
import org.elasticsearch.index.IndexSettingProviders;
34+
import org.elasticsearch.index.IndexSettings;
3035
import org.elasticsearch.indices.SystemIndices;
3136
import org.elasticsearch.indices.TestIndexNameExpressionResolver;
3237
import org.elasticsearch.test.ESTestCase;
3338

3439
import java.time.Instant;
3540
import java.time.temporal.ChronoUnit;
41+
import java.util.ArrayList;
3642
import java.util.List;
3743
import java.util.Set;
44+
import java.util.stream.Collectors;
3845

46+
import static org.elasticsearch.cluster.metadata.DataStream.getDefaultBackingIndexName;
47+
import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.createIndexMetadata;
3948
import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.getClusterStateWithDataStreams;
4049
import static org.elasticsearch.test.LambdaMatchers.transformedItemsMatch;
4150
import static org.elasticsearch.test.LambdaMatchers.transformedMatch;
@@ -317,9 +326,9 @@ public void testGetTimeSeriesMixedDataStream() {
317326
null
318327
);
319328

320-
var name1 = DataStream.getDefaultBackingIndexName("ds-1", 1, instant.toEpochMilli());
321-
var name2 = DataStream.getDefaultBackingIndexName("ds-1", 2, instant.toEpochMilli());
322-
var name3 = DataStream.getDefaultBackingIndexName("ds-1", 3, twoHoursAgo.toEpochMilli());
329+
var name1 = getDefaultBackingIndexName("ds-1", 1, instant.toEpochMilli());
330+
var name2 = getDefaultBackingIndexName("ds-1", 2, instant.toEpochMilli());
331+
var name3 = getDefaultBackingIndexName("ds-1", 3, twoHoursAgo.toEpochMilli());
323332
assertThat(
324333
response.getDataStreams(),
325334
contains(
@@ -527,4 +536,120 @@ public void testProvidersAffectMode() {
527536
equalTo("standard")
528537
);
529538
}
539+
540+
public void testGetEffectiveSettingsTemplateOnlySettings() {
541+
// Set a lifecycle only in the template, and make sure that is in the response:
542+
ProjectId projectId = randomProjectIdOrDefault();
543+
GetDataStreamAction.Request req = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] {});
544+
final String templatePolicy = "templatePolicy";
545+
final String templateIndexMode = IndexMode.LOOKUP.getName();
546+
final String dataStreamPolicy = "dataStreamPolicy";
547+
final String dataStreamIndexMode = IndexMode.LOGSDB.getName();
548+
549+
ClusterState state = getClusterStateWithDataStreamWithSettings(
550+
projectId,
551+
Settings.builder()
552+
.put(IndexMetadata.LIFECYCLE_NAME, templatePolicy)
553+
.put(IndexSettings.MODE.getKey(), templateIndexMode)
554+
.build(),
555+
Settings.EMPTY
556+
);
557+
558+
GetDataStreamAction.Response response = TransportGetDataStreamsAction.innerOperation(
559+
state.projectState(projectId),
560+
req,
561+
resolver,
562+
systemIndices,
563+
ClusterSettings.createBuiltInClusterSettings(),
564+
dataStreamGlobalRetentionSettings,
565+
emptyDataStreamFailureStoreSettings,
566+
new IndexSettingProviders(Set.of()),
567+
null
568+
);
569+
assertNotNull(response.getDataStreams());
570+
assertThat(response.getDataStreams().size(), equalTo(1));
571+
assertThat(response.getDataStreams().get(0).getIlmPolicy(), equalTo(templatePolicy));
572+
assertThat(response.getDataStreams().get(0).getIndexModeName(), equalTo(templateIndexMode));
573+
}
574+
575+
public void testGetEffectiveSettings() {
576+
ProjectId projectId = randomProjectIdOrDefault();
577+
GetDataStreamAction.Request req = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] {});
578+
final String templatePolicy = "templatePolicy";
579+
final String templateIndexMode = IndexMode.LOOKUP.getName();
580+
final String dataStreamPolicy = "dataStreamPolicy";
581+
final String dataStreamIndexMode = IndexMode.LOGSDB.getName();
582+
// Now set a lifecycle in both the template and the data stream, and make sure the response has the data stream one:
583+
ClusterState state = getClusterStateWithDataStreamWithSettings(
584+
projectId,
585+
Settings.builder()
586+
.put(IndexMetadata.LIFECYCLE_NAME, templatePolicy)
587+
.put(IndexSettings.MODE.getKey(), templateIndexMode)
588+
.build(),
589+
Settings.builder()
590+
.put(IndexMetadata.LIFECYCLE_NAME, dataStreamPolicy)
591+
.put(IndexSettings.MODE.getKey(), dataStreamIndexMode)
592+
.build()
593+
);
594+
GetDataStreamAction.Response response = TransportGetDataStreamsAction.innerOperation(
595+
state.projectState(projectId),
596+
req,
597+
resolver,
598+
systemIndices,
599+
ClusterSettings.createBuiltInClusterSettings(),
600+
dataStreamGlobalRetentionSettings,
601+
emptyDataStreamFailureStoreSettings,
602+
new IndexSettingProviders(Set.of()),
603+
null
604+
);
605+
assertNotNull(response.getDataStreams());
606+
assertThat(response.getDataStreams().size(), equalTo(1));
607+
assertThat(response.getDataStreams().get(0).getIlmPolicy(), equalTo(dataStreamPolicy));
608+
assertThat(response.getDataStreams().get(0).getIndexModeName(), equalTo(dataStreamIndexMode));
609+
}
610+
611+
private static ClusterState getClusterStateWithDataStreamWithSettings(
612+
ProjectId projectId,
613+
Settings templateSettings,
614+
Settings dataStreamSettings
615+
) {
616+
String dataStreamName = "data-stream-1";
617+
int numberOfBackingIndices = randomIntBetween(1, 5);
618+
long currentTime = System.currentTimeMillis();
619+
int replicas = 0;
620+
boolean replicated = false;
621+
ProjectMetadata.Builder builder = ProjectMetadata.builder(projectId);
622+
builder.put(
623+
"template_1",
624+
ComposableIndexTemplate.builder()
625+
.indexPatterns(List.of("*"))
626+
.template(Template.builder().settings(templateSettings))
627+
.dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate())
628+
.build()
629+
);
630+
631+
List<IndexMetadata> backingIndices = new ArrayList<>();
632+
for (int backingIndexNumber = 1; backingIndexNumber <= numberOfBackingIndices; backingIndexNumber++) {
633+
backingIndices.add(
634+
createIndexMetadata(
635+
getDefaultBackingIndexName(dataStreamName, backingIndexNumber, currentTime),
636+
true,
637+
templateSettings,
638+
replicas
639+
)
640+
);
641+
}
642+
List<IndexMetadata> allIndices = new ArrayList<>(backingIndices);
643+
644+
DataStream ds = DataStream.builder(
645+
dataStreamName,
646+
backingIndices.stream().map(IndexMetadata::getIndex).collect(Collectors.toList())
647+
).setGeneration(numberOfBackingIndices).setSettings(dataStreamSettings).setReplicated(replicated).build();
648+
builder.put(ds);
649+
650+
for (IndexMetadata index : allIndices) {
651+
builder.put(index, false);
652+
}
653+
return ClusterState.builder(new ClusterName("_name")).putProjectMetadata(builder.build()).build();
654+
}
530655
}

‎modules/repository-s3/src/main/java/org/elasticsearch/repositories/s3/S3BlobContainer.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -394,7 +394,7 @@ public void copyBlob(
394394
SocketAccess.doPrivilegedVoid(() -> clientReference.client().copyObject(copyObjectRequest));
395395
}
396396
}
397-
} catch (final Exception e) {
397+
} catch (final SdkException e) {
398398
if (e instanceof SdkServiceException sse && sse.statusCode() == RestStatus.NOT_FOUND.getStatus()) {
399399
throw new NoSuchFileException(
400400
"Copy source [" + s3SourceBlobContainer.buildKey(sourceBlobName) + "] not found: " + sse.getMessage()
@@ -660,7 +660,7 @@ private void executeMultipart(
660660
SocketAccess.doPrivilegedVoid(() -> clientReference.client().completeMultipartUpload(completeMultipartUploadRequest));
661661
}
662662
cleanupOnFailureActions.clear();
663-
} catch (final Exception e) {
663+
} catch (final SdkException e) {
664664
if (e instanceof SdkServiceException sse && sse.statusCode() == RestStatus.NOT_FOUND.getStatus()) {
665665
throw new NoSuchFileException(blobName, null, e.getMessage());
666666
}

‎muted-tests.yml‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -408,9 +408,6 @@ tests:
408408
- class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT
409409
method: test {rerank.Reranker before a limit ASYNC}
410410
issue: https://github.com/elastic/elasticsearch/issues/127051
411-
- class: org.elasticsearch.xpack.esql.qa.single_node.EsqlSpecIT
412-
method: test {rrf.SimpleRrf ASYNC}
413-
issue: https://github.com/elastic/elasticsearch/issues/127063
414411
- class: org.elasticsearch.packaging.test.DockerTests
415412
method: test026InstallBundledRepositoryPlugins
416413
issue: https://github.com/elastic/elasticsearch/issues/127081
@@ -426,9 +423,6 @@ tests:
426423
- class: org.elasticsearch.xpack.esql.qa.single_node.GenerativeIT
427424
method: test
428425
issue: https://github.com/elastic/elasticsearch/issues/127157
429-
- class: org.elasticsearch.xpack.esql.qa.single_node.EsqlSpecIT
430-
method: test {fork.ForkWithWhereSortDescAndLimit SYNC}
431-
issue: https://github.com/elastic/elasticsearch/issues/127326
432426
- class: org.elasticsearch.geometry.utils.SpatialEnvelopeVisitorTests
433427
method: testVisitGeoPointsWrapping
434428
issue: https://github.com/elastic/elasticsearch/issues/123425
@@ -441,9 +435,15 @@ tests:
441435
- class: org.elasticsearch.xpack.esql.heap_attack.HeapAttackIT
442436
method: testLookupExplosionNoFetch
443437
issue: https://github.com/elastic/elasticsearch/issues/127365
444-
- class: org.elasticsearch.repositories.blobstore.testkit.analyze.S3RepositoryAnalysisRestIT
445-
method: testRepositoryAnalysis
446-
issue: https://github.com/elastic/elasticsearch/issues/127399
438+
- class: org.elasticsearch.action.admin.cluster.state.TransportClusterStateActionDisruptionIT
439+
method: testNonLocalRequestAlwaysFindsMasterAndWaitsForMetadata
440+
issue: https://github.com/elastic/elasticsearch/issues/127422
441+
- class: org.elasticsearch.xpack.esql.qa.single_node.PushQueriesIT
442+
method: testPushCaseInsensitiveEqualityOnDefaults
443+
issue: https://github.com/elastic/elasticsearch/issues/127431
444+
- class: org.elasticsearch.action.admin.cluster.state.TransportClusterStateActionDisruptionIT
445+
method: testLocalRequestAlwaysSucceeds
446+
issue: https://github.com/elastic/elasticsearch/issues/127423
447447

448448
# Examples:
449449
#

‎server/src/internalClusterTest/java/org/elasticsearch/cluster/metadata/MetadataCreateIndexServiceIT.java‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.cluster.metadata;
11+
12+
import org.elasticsearch.action.ActionListener;
13+
import org.elasticsearch.action.admin.indices.create.CreateIndexClusterStateUpdateRequest;
14+
import org.elasticsearch.action.admin.indices.get.GetIndexRequest;
15+
import org.elasticsearch.action.admin.indices.get.GetIndexResponse;
16+
import org.elasticsearch.action.support.master.ShardsAcknowledgedResponse;
17+
import org.elasticsearch.common.settings.Settings;
18+
import org.elasticsearch.core.TimeValue;
19+
import org.elasticsearch.test.ESIntegTestCase;
20+
21+
import java.util.Locale;
22+
import java.util.concurrent.CountDownLatch;
23+
import java.util.concurrent.TimeUnit;
24+
25+
import static org.hamcrest.Matchers.equalTo;
26+
27+
public class MetadataCreateIndexServiceIT extends ESIntegTestCase {
28+
29+
public void testRequestTemplateIsRespected() throws InterruptedException {
30+
/*
31+
* This test passes a template in the CreateIndexClusterStateUpdateRequest, and makes sure that the settings from that template
32+
* are used when creating the index.
33+
*/
34+
MetadataCreateIndexService metadataCreateIndexService = internalCluster().getCurrentMasterNodeInstance(
35+
MetadataCreateIndexService.class
36+
);
37+
final String indexName = randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
38+
final int numberOfReplicas = randomIntBetween(1, 7);
39+
CreateIndexClusterStateUpdateRequest request = new CreateIndexClusterStateUpdateRequest(
40+
"testRequestTemplateIsRespected",
41+
ProjectId.DEFAULT,
42+
indexName,
43+
randomAlphaOfLength(20)
44+
);
45+
request.setMatchingTemplate(
46+
ComposableIndexTemplate.builder()
47+
.template(Template.builder().settings(Settings.builder().put("index.number_of_replicas", numberOfReplicas)))
48+
.build()
49+
);
50+
final CountDownLatch listenerCalledLatch = new CountDownLatch(1);
51+
ActionListener<ShardsAcknowledgedResponse> listener = new ActionListener<>() {
52+
@Override
53+
public void onResponse(ShardsAcknowledgedResponse shardsAcknowledgedResponse) {
54+
listenerCalledLatch.countDown();
55+
}
56+
57+
@Override
58+
public void onFailure(Exception e) {
59+
logger.error(e);
60+
listenerCalledLatch.countDown();
61+
}
62+
};
63+
64+
metadataCreateIndexService.createIndex(
65+
TimeValue.THIRTY_SECONDS,
66+
TimeValue.THIRTY_SECONDS,
67+
TimeValue.THIRTY_SECONDS,
68+
request,
69+
listener
70+
);
71+
listenerCalledLatch.await(10, TimeUnit.SECONDS);
72+
GetIndexResponse response = admin().indices()
73+
.getIndex(new GetIndexRequest(TimeValue.THIRTY_SECONDS).indices(indexName))
74+
.actionGet();
75+
Settings settings = response.getSettings().get(indexName);
76+
assertThat(settings.get("index.number_of_replicas"), equalTo(Integer.toString(numberOfReplicas)));
77+
}
78+
}

‎server/src/main/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverService.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,6 @@
6666

6767
import static org.elasticsearch.cluster.metadata.IndexAbstraction.Type.ALIAS;
6868
import static org.elasticsearch.cluster.metadata.IndexAbstraction.Type.DATA_STREAM;
69-
import static org.elasticsearch.cluster.metadata.MetadataCreateDataStreamService.lookupTemplateForDataStream;
7069
import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findV1Templates;
7170
import static org.elasticsearch.cluster.metadata.MetadataIndexTemplateService.findV2Template;
7271
import static org.elasticsearch.cluster.routing.allocation.allocator.AllocationActionListener.rerouteCompletionIsNotRequired;
@@ -325,7 +324,7 @@ private RolloverResult rolloverDataStream(
325324
final SystemDataStreamDescriptor systemDataStreamDescriptor;
326325
if (dataStream.isSystem() == false) {
327326
systemDataStreamDescriptor = null;
328-
templateV2 = lookupTemplateForDataStream(dataStreamName, metadata);
327+
templateV2 = dataStream.getEffectiveIndexTemplate(projectState.metadata());
329328
} else {
330329
systemDataStreamDescriptor = systemIndices.findMatchingDataStreamDescriptor(dataStreamName);
331330
if (systemDataStreamDescriptor == null) {

‎server/src/main/java/org/elasticsearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -267,7 +267,7 @@ public static Template resolveTemplate(
267267
List<CompressedXContent> mappings = MetadataCreateIndexService.collectV2Mappings(
268268
null, // empty request mapping as the user can't specify any explicit mappings via the simulate api
269269
simulatedProject,
270-
matchingTemplate,
270+
template,
271271
xContentRegistry,
272272
simulatedIndexName
273273
);

0 commit comments

Comments
 (0)