Skip to content

Commit b001d4d

Browse files
original-brownbearoriginal-brownbear
authored
Merge branch 'main' into 0.5-fetch-peak
2 parents 400277c + 45d321d commit b001d4d

File tree

105 files changed

+3179
-1456
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

105 files changed

+3179
-1456
lines changed

‎.backportrc.json‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,9 @@
11
{
22
"upstream" : "elastic/elasticsearch",
3-
"targetBranchChoices" : [ "main", "8.x", "9.0", "8.18", "8.17", "8.16", "8.15", "8.14", "8.13", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ],
3+
"targetBranchChoices" : [ "main", "9.0", "8.19", "8.18", "8.17", "8.16", "8.15", "8.14", "8.13", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ],
44
"targetPRLabels" : [ "backport" ],
55
"branchLabelMapping" : {
66
"^v9.1.0$" : "main",
7-
"^v8.19.0$" : "8.x",
87
"^v(\\d+).(\\d+).\\d+(?:-(?:alpha|beta|rc)\\d+)?$" : "$1.$2"
98
}
109
}

‎.buildkite/pipelines/periodic.template.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -237,7 +237,7 @@ steps:
237237
image: family/elasticsearch-ubuntu-2004
238238
machineType: n2-standard-8
239239
buildDirectory: /dev/shm/bk
240-
if: build.branch == "main" || build.branch == "8.x" || build.branch == "7.17"
240+
if: build.branch == "main" || build.branch == "8.19" || build.branch == "7.17"
241241
- label: check-branch-consistency
242242
command: .ci/scripts/run-gradle.sh branchConsistency
243243
timeout_in_minutes: 15

‎.buildkite/pipelines/periodic.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -656,7 +656,7 @@ steps:
656656
image: family/elasticsearch-ubuntu-2004
657657
machineType: n2-standard-8
658658
buildDirectory: /dev/shm/bk
659-
if: build.branch == "main" || build.branch == "8.x" || build.branch == "7.17"
659+
if: build.branch == "main" || build.branch == "8.19" || build.branch == "7.17"
660660
- label: check-branch-consistency
661661
command: .ci/scripts/run-gradle.sh branchConsistency
662662
timeout_in_minutes: 15

‎.ci/scripts/resolve-dra-manifest.sh‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -24,11 +24,6 @@ if [ "$LATEST_VERSION" != "$ES_VERSION" ]; then
2424
echo "Latest build for '$ARTIFACT' is version $LATEST_VERSION but expected version $ES_VERSION." 1>&2
2525
NEW_BRANCH=$(echo $ES_VERSION | sed -E "s/([0-9]+\.[0-9]+)\.[0-9]/\1/g")
2626

27-
# Temporary
28-
if [[ "$ES_VERSION" == "8.16.0" ]]; then
29-
NEW_BRANCH="8.x"
30-
fi
31-
3227
echo "Using branch $NEW_BRANCH instead of $BRANCH." 1>&2
3328
LATEST_BUILD=$(fetch_build $WORKFLOW $ARTIFACT $NEW_BRANCH)
3429
fi

‎branches.json‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,13 +8,13 @@
88
"branch": "9.0"
99
},
1010
{
11-
"branch": "8.18"
11+
"branch": "8.19"
1212
},
1313
{
14-
"branch": "8.17"
14+
"branch": "8.18"
1515
},
1616
{
17-
"branch": "8.x"
17+
"branch": "8.17"
1818
},
1919
{
2020
"branch": "7.17"

‎docs/changelog/126091.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 126091
2+
summary: Allow balancing weights to be set per tier
3+
area: Allocation
4+
type: enhancement
5+
issues: []

‎docs/changelog/127225.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 127225
2+
summary: Fix count optimization with pushable union types
3+
area: ES|QL
4+
type: bug
5+
issues:
6+
- 127200

‎docs/changelog/127259.yaml‎

Lines changed: 0 additions & 5 deletions
This file was deleted.

‎modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -258,15 +258,15 @@ static GetDataStreamAction.Response innerOperation(
258258
} else {
259259
indexTemplate = MetadataIndexTemplateService.findV2Template(state.metadata(), dataStream.getName(), false);
260260
if (indexTemplate != null) {
261-
Settings settings = MetadataIndexTemplateService.resolveSettings(state.metadata(), indexTemplate);
261+
Settings settings = dataStream.getEffectiveSettings(state.metadata());
262262
ilmPolicyName = settings.get(IndexMetadata.LIFECYCLE_NAME);
263263
if (indexMode == null && state.metadata().templatesV2().get(indexTemplate) != null) {
264264
indexMode = resolveMode(
265265
state,
266266
indexSettingProviders,
267267
dataStream,
268268
settings,
269-
state.metadata().templatesV2().get(indexTemplate)
269+
dataStream.getEffectiveIndexTemplate(state.metadata())
270270
);
271271
}
272272
indexTemplatePreferIlmValue = PREFER_ILM_SETTING.get(settings);

‎modules/data-streams/src/test/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsActionTests.java‎

Lines changed: 128 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,14 +11,18 @@
1111
import org.elasticsearch.action.datastreams.GetDataStreamAction;
1212
import org.elasticsearch.cluster.ClusterName;
1313
import org.elasticsearch.cluster.ClusterState;
14+
import org.elasticsearch.cluster.metadata.ComposableIndexTemplate;
1415
import org.elasticsearch.cluster.metadata.DataStream;
1516
import org.elasticsearch.cluster.metadata.DataStreamFailureStoreSettings;
1617
import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention;
1718
import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionSettings;
1819
import org.elasticsearch.cluster.metadata.DataStreamTestHelper;
20+
import org.elasticsearch.cluster.metadata.IndexMetadata;
1921
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
2022
import org.elasticsearch.cluster.metadata.Metadata;
23+
import org.elasticsearch.cluster.metadata.ProjectId;
2124
import org.elasticsearch.cluster.metadata.ProjectMetadata;
25+
import org.elasticsearch.cluster.metadata.Template;
2226
import org.elasticsearch.common.settings.ClusterSettings;
2327
import org.elasticsearch.common.settings.Settings;
2428
import org.elasticsearch.core.TimeValue;
@@ -27,15 +31,20 @@
2731
import org.elasticsearch.index.IndexMode;
2832
import org.elasticsearch.index.IndexNotFoundException;
2933
import org.elasticsearch.index.IndexSettingProviders;
34+
import org.elasticsearch.index.IndexSettings;
3035
import org.elasticsearch.indices.SystemIndices;
3136
import org.elasticsearch.indices.TestIndexNameExpressionResolver;
3237
import org.elasticsearch.test.ESTestCase;
3338

3439
import java.time.Instant;
3540
import java.time.temporal.ChronoUnit;
41+
import java.util.ArrayList;
3642
import java.util.List;
3743
import java.util.Set;
44+
import java.util.stream.Collectors;
3845

46+
import static org.elasticsearch.cluster.metadata.DataStream.getDefaultBackingIndexName;
47+
import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.createIndexMetadata;
3948
import static org.elasticsearch.cluster.metadata.DataStreamTestHelper.getClusterStateWithDataStreams;
4049
import static org.elasticsearch.test.LambdaMatchers.transformedItemsMatch;
4150
import static org.elasticsearch.test.LambdaMatchers.transformedMatch;
@@ -317,9 +326,9 @@ public void testGetTimeSeriesMixedDataStream() {
317326
null
318327
);
319328

320-
var name1 = DataStream.getDefaultBackingIndexName("ds-1", 1, instant.toEpochMilli());
321-
var name2 = DataStream.getDefaultBackingIndexName("ds-1", 2, instant.toEpochMilli());
322-
var name3 = DataStream.getDefaultBackingIndexName("ds-1", 3, twoHoursAgo.toEpochMilli());
329+
var name1 = getDefaultBackingIndexName("ds-1", 1, instant.toEpochMilli());
330+
var name2 = getDefaultBackingIndexName("ds-1", 2, instant.toEpochMilli());
331+
var name3 = getDefaultBackingIndexName("ds-1", 3, twoHoursAgo.toEpochMilli());
323332
assertThat(
324333
response.getDataStreams(),
325334
contains(
@@ -527,4 +536,120 @@ public void testProvidersAffectMode() {
527536
equalTo("standard")
528537
);
529538
}
539+
540+
public void testGetEffectiveSettingsTemplateOnlySettings() {
541+
// Set a lifecycle only in the template, and make sure that is in the response:
542+
ProjectId projectId = randomProjectIdOrDefault();
543+
GetDataStreamAction.Request req = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] {});
544+
final String templatePolicy = "templatePolicy";
545+
final String templateIndexMode = IndexMode.LOOKUP.getName();
546+
final String dataStreamPolicy = "dataStreamPolicy";
547+
final String dataStreamIndexMode = IndexMode.LOGSDB.getName();
548+
549+
ClusterState state = getClusterStateWithDataStreamWithSettings(
550+
projectId,
551+
Settings.builder()
552+
.put(IndexMetadata.LIFECYCLE_NAME, templatePolicy)
553+
.put(IndexSettings.MODE.getKey(), templateIndexMode)
554+
.build(),
555+
Settings.EMPTY
556+
);
557+
558+
GetDataStreamAction.Response response = TransportGetDataStreamsAction.innerOperation(
559+
state.projectState(projectId),
560+
req,
561+
resolver,
562+
systemIndices,
563+
ClusterSettings.createBuiltInClusterSettings(),
564+
dataStreamGlobalRetentionSettings,
565+
emptyDataStreamFailureStoreSettings,
566+
new IndexSettingProviders(Set.of()),
567+
null
568+
);
569+
assertNotNull(response.getDataStreams());
570+
assertThat(response.getDataStreams().size(), equalTo(1));
571+
assertThat(response.getDataStreams().get(0).getIlmPolicy(), equalTo(templatePolicy));
572+
assertThat(response.getDataStreams().get(0).getIndexModeName(), equalTo(templateIndexMode));
573+
}
574+
575+
public void testGetEffectiveSettings() {
576+
ProjectId projectId = randomProjectIdOrDefault();
577+
GetDataStreamAction.Request req = new GetDataStreamAction.Request(TEST_REQUEST_TIMEOUT, new String[] {});
578+
final String templatePolicy = "templatePolicy";
579+
final String templateIndexMode = IndexMode.LOOKUP.getName();
580+
final String dataStreamPolicy = "dataStreamPolicy";
581+
final String dataStreamIndexMode = IndexMode.LOGSDB.getName();
582+
// Now set a lifecycle in both the template and the data stream, and make sure the response has the data stream one:
583+
ClusterState state = getClusterStateWithDataStreamWithSettings(
584+
projectId,
585+
Settings.builder()
586+
.put(IndexMetadata.LIFECYCLE_NAME, templatePolicy)
587+
.put(IndexSettings.MODE.getKey(), templateIndexMode)
588+
.build(),
589+
Settings.builder()
590+
.put(IndexMetadata.LIFECYCLE_NAME, dataStreamPolicy)
591+
.put(IndexSettings.MODE.getKey(), dataStreamIndexMode)
592+
.build()
593+
);
594+
GetDataStreamAction.Response response = TransportGetDataStreamsAction.innerOperation(
595+
state.projectState(projectId),
596+
req,
597+
resolver,
598+
systemIndices,
599+
ClusterSettings.createBuiltInClusterSettings(),
600+
dataStreamGlobalRetentionSettings,
601+
emptyDataStreamFailureStoreSettings,
602+
new IndexSettingProviders(Set.of()),
603+
null
604+
);
605+
assertNotNull(response.getDataStreams());
606+
assertThat(response.getDataStreams().size(), equalTo(1));
607+
assertThat(response.getDataStreams().get(0).getIlmPolicy(), equalTo(dataStreamPolicy));
608+
assertThat(response.getDataStreams().get(0).getIndexModeName(), equalTo(dataStreamIndexMode));
609+
}
610+
611+
private static ClusterState getClusterStateWithDataStreamWithSettings(
612+
ProjectId projectId,
613+
Settings templateSettings,
614+
Settings dataStreamSettings
615+
) {
616+
String dataStreamName = "data-stream-1";
617+
int numberOfBackingIndices = randomIntBetween(1, 5);
618+
long currentTime = System.currentTimeMillis();
619+
int replicas = 0;
620+
boolean replicated = false;
621+
ProjectMetadata.Builder builder = ProjectMetadata.builder(projectId);
622+
builder.put(
623+
"template_1",
624+
ComposableIndexTemplate.builder()
625+
.indexPatterns(List.of("*"))
626+
.template(Template.builder().settings(templateSettings))
627+
.dataStreamTemplate(new ComposableIndexTemplate.DataStreamTemplate())
628+
.build()
629+
);
630+
631+
List<IndexMetadata> backingIndices = new ArrayList<>();
632+
for (int backingIndexNumber = 1; backingIndexNumber <= numberOfBackingIndices; backingIndexNumber++) {
633+
backingIndices.add(
634+
createIndexMetadata(
635+
getDefaultBackingIndexName(dataStreamName, backingIndexNumber, currentTime),
636+
true,
637+
templateSettings,
638+
replicas
639+
)
640+
);
641+
}
642+
List<IndexMetadata> allIndices = new ArrayList<>(backingIndices);
643+
644+
DataStream ds = DataStream.builder(
645+
dataStreamName,
646+
backingIndices.stream().map(IndexMetadata::getIndex).collect(Collectors.toList())
647+
).setGeneration(numberOfBackingIndices).setSettings(dataStreamSettings).setReplicated(replicated).build();
648+
builder.put(ds);
649+
650+
for (IndexMetadata index : allIndices) {
651+
builder.put(index, false);
652+
}
653+
return ClusterState.builder(new ClusterName("_name")).putProjectMetadata(builder.build()).build();
654+
}
530655
}

0 commit comments

Comments
 (0)