Skip to content

Commit 2c55cd3

Browse files
eyalkoreneyalkoren
committed
Merge remote-tracking branch 'upstream/main' into support-dotted-fields-in-ingest-processors
2 parents e9de9ce + 1ad6dbc commit 2c55cd3

File tree

28 files changed

+191
-109
lines changed

28 files changed

+191
-109
lines changed

‎.ci/scripts/resolve-dra-manifest.sh‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,8 @@ strip_version() {
66
}
77

88
fetch_build() {
9-
curl -sS https://artifacts-$1.elastic.co/$2/latest/$3.json \
9+
>&2 echo "Checking for build id: https://artifacts-$1.elastic.co/$2/latest/$3.json"
10+
curl -sSf https://artifacts-$1.elastic.co/$2/latest/$3.json \
1011
| jq -r '.build_id'
1112
}
1213

@@ -15,7 +16,15 @@ BRANCH="${BRANCH:-$2}"
1516
ES_VERSION="${ES_VERSION:-$3}"
1617
WORKFLOW=${WORKFLOW:-$4}
1718

18-
LATEST_BUILD=$(fetch_build $WORKFLOW $ARTIFACT $BRANCH)
19+
if [[ "$WORKFLOW" == "staging" ]]; then
20+
LATEST_BUILD=$(fetch_build $WORKFLOW $ARTIFACT $ES_VERSION)
21+
elif [[ "$WORKFLOW" == "snapshot" ]]; then
22+
LATEST_BUILD=$(fetch_build $WORKFLOW $ARTIFACT $BRANCH)
23+
else
24+
echo "Unknown workflow: $WORKFLOW"
25+
exit 1
26+
fi
27+
1928
LATEST_VERSION=$(strip_version $LATEST_BUILD)
2029

2130
# If the latest artifact version doesn't match what we expect, try the corresponding version branch.

‎docs/changelog/130083.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 130083
2+
summary: Fix timeout bug in DBQ deletion of unused and orphan ML data
3+
area: Machine Learning
4+
type: bug
5+
issues: []

‎muted-tests.yml‎

Lines changed: 2 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -473,9 +473,6 @@ tests:
473473
- class: org.elasticsearch.packaging.test.DockerTests
474474
method: test081SymlinksAreFollowedWithEnvironmentVariableFiles
475475
issue: https://github.com/elastic/elasticsearch/issues/128867
476-
- class: org.elasticsearch.index.engine.ThreadPoolMergeExecutorServiceDiskSpaceTests
477-
method: testAvailableDiskSpaceMonitorWhenFileSystemStatErrors
478-
issue: https://github.com/elastic/elasticsearch/issues/129149
479476
- class: org.elasticsearch.xpack.esql.qa.single_node.GenerativeForkIT
480477
method: test {lookup-join.EnrichLookupStatsBug ASYNC}
481478
issue: https://github.com/elastic/elasticsearch/issues/129228
@@ -545,9 +542,6 @@ tests:
545542
- class: org.elasticsearch.qa.verify_version_constants.VerifyVersionConstantsIT
546543
method: testLuceneVersionConstant
547544
issue: https://github.com/elastic/elasticsearch/issues/125638
548-
- class: org.elasticsearch.xpack.esql.qa.single_node.GenerativeIT
549-
method: test
550-
issue: https://github.com/elastic/elasticsearch/issues/129819
551545
- class: org.elasticsearch.index.store.FsDirectoryFactoryTests
552546
method: testPreload
553547
issue: https://github.com/elastic/elasticsearch/issues/129852
@@ -569,15 +563,14 @@ tests:
569563
issue: https://github.com/elastic/elasticsearch/issues/130010
570564
- class: org.elasticsearch.xpack.esql.qa.mixed.MixedClusterEsqlSpecIT
571565
issue: https://github.com/elastic/elasticsearch/issues/128224
572-
- class: org.elasticsearch.xpack.autoscaling.storage.ReactiveStorageIT
573-
method: testScaleDuringSplitOrClone
574-
issue: https://github.com/elastic/elasticsearch/issues/130044
575566
- class: org.elasticsearch.cluster.metadata.ComposableIndexTemplateTests
576567
method: testMergeEmptyMappingsIntoTemplateWithNonEmptySettings
577568
issue: https://github.com/elastic/elasticsearch/issues/130050
578569
- class: org.elasticsearch.xpack.esql.qa.multi_node.GenerativeIT
579570
method: test
580571
issue: https://github.com/elastic/elasticsearch/issues/130067
572+
- class: geoip.GeoIpMultiProjectIT
573+
issue: https://github.com/elastic/elasticsearch/issues/130073
581574

582575
# Examples:
583576
#

‎qa/mixed-cluster/build.gradle‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ apply plugin: 'elasticsearch.rest-resources'
2323

2424
dependencies {
2525
restTestConfig project(path: ':modules:aggregations', configuration: 'restTests')
26+
restTestConfig project(path: ':modules:mapper-extras', configuration: 'restTests')
2627
}
2728

2829
restResources {

‎server/src/main/java/org/elasticsearch/TransportVersions.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -208,6 +208,7 @@ static TransportVersion def(int id) {
208208
public static final TransportVersion SPARSE_VECTOR_FIELD_PRUNING_OPTIONS_8_19 = def(8_841_0_58);
209209
public static final TransportVersion ML_INFERENCE_ELASTIC_DENSE_TEXT_EMBEDDINGS_ADDED_8_19 = def(8_841_0_59);
210210
public static final TransportVersion ML_INFERENCE_COHERE_API_VERSION_8_19 = def(8_841_0_60);
211+
public static final TransportVersion ESQL_DOCUMENTS_FOUND_AND_VALUES_LOADED_8_19 = def(8_841_0_61);
211212
public static final TransportVersion V_9_0_0 = def(9_000_0_09);
212213
public static final TransportVersion INITIAL_ELASTICSEARCH_9_0_1 = def(9_000_0_10);
213214
public static final TransportVersion INITIAL_ELASTICSEARCH_9_0_2 = def(9_000_0_11);

‎server/src/main/java/org/elasticsearch/index/engine/Engine.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2199,7 +2199,7 @@ public void close() throws IOException {
21992199
awaitPendingClose();
22002200
}
22012201

2202-
private void awaitPendingClose() {
2202+
protected final void awaitPendingClose() {
22032203
try {
22042204
closedLatch.await();
22052205
} catch (InterruptedException e) {

‎server/src/test/java/org/elasticsearch/index/engine/ThreadPoolMergeExecutorServiceDiskSpaceTests.java‎

Lines changed: 18 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -324,10 +324,19 @@ public void testDiskSpaceMonitorStartsAsDisabled() throws Exception {
324324
}
325325

326326
public void testAvailableDiskSpaceMonitorWhenFileSystemStatErrors() throws Exception {
327-
aFileStore.usableSpace = randomLongBetween(1L, 100L);
328-
aFileStore.totalSpace = randomLongBetween(1L, 100L);
329-
bFileStore.usableSpace = randomLongBetween(1L, 100L);
330-
bFileStore.totalSpace = randomLongBetween(1L, 100L);
327+
long aUsableSpace;
328+
long bUsableSpace;
329+
do {
330+
aFileStore.usableSpace = randomLongBetween(1L, 1000L);
331+
aFileStore.totalSpace = randomLongBetween(1L, 1000L);
332+
bFileStore.usableSpace = randomLongBetween(1L, 1000L);
333+
bFileStore.totalSpace = randomLongBetween(1L, 1000L);
334+
// the default 5% (same as flood stage level)
335+
aUsableSpace = Math.max(aFileStore.usableSpace - aFileStore.totalSpace / 20, 0L);
336+
bUsableSpace = Math.max(bFileStore.usableSpace - bFileStore.totalSpace / 20, 0L);
337+
} while (aUsableSpace == bUsableSpace); // they must be different in order to distinguish the available disk space updates
338+
long finalBUsableSpace = bUsableSpace;
339+
long finalAUsableSpace = aUsableSpace;
331340
boolean aErrorsFirst = randomBoolean();
332341
if (aErrorsFirst) {
333342
// the "a" file system will error when collecting stats
@@ -355,18 +364,10 @@ public void testAvailableDiskSpaceMonitorWhenFileSystemStatErrors() throws Excep
355364
assertThat(availableDiskSpaceUpdates.size(), is(1));
356365
if (aErrorsFirst) {
357366
// uses the stats from "b"
358-
assertThat(
359-
availableDiskSpaceUpdates.getLast().getBytes(),
360-
// the default 5% (same as flood stage level)
361-
is(Math.max(bFileStore.usableSpace - bFileStore.totalSpace / 20, 0L))
362-
);
367+
assertThat(availableDiskSpaceUpdates.getLast().getBytes(), is(finalBUsableSpace));
363368
} else {
364369
// uses the stats from "a"
365-
assertThat(
366-
availableDiskSpaceUpdates.getLast().getBytes(),
367-
// the default 5% (same as flood stage level)
368-
is(Math.max(aFileStore.usableSpace - aFileStore.totalSpace / 20, 0L))
369-
);
370+
assertThat(availableDiskSpaceUpdates.getLast().getBytes(), is(finalAUsableSpace));
370371
}
371372
}
372373
});
@@ -393,21 +394,14 @@ public void testAvailableDiskSpaceMonitorWhenFileSystemStatErrors() throws Excep
393394
}
394395
assertBusy(() -> {
395396
synchronized (availableDiskSpaceUpdates) {
397+
// the updates are different values
396398
assertThat(availableDiskSpaceUpdates.size(), is(3));
397399
if (aErrorsFirst) {
398400
// uses the stats from "a"
399-
assertThat(
400-
availableDiskSpaceUpdates.getLast().getBytes(),
401-
// the default 5% (same as flood stage level)
402-
is(Math.max(aFileStore.usableSpace - aFileStore.totalSpace / 20, 0L))
403-
);
401+
assertThat(availableDiskSpaceUpdates.getLast().getBytes(), is(finalAUsableSpace));
404402
} else {
405403
// uses the stats from "b"
406-
assertThat(
407-
availableDiskSpaceUpdates.getLast().getBytes(),
408-
// the default 5% (same as flood stage level)
409-
is(Math.max(bFileStore.usableSpace - bFileStore.totalSpace / 20, 0L))
410-
);
404+
assertThat(availableDiskSpaceUpdates.getLast().getBytes(), is(finalBUsableSpace));
411405
}
412406
}
413407
});

‎x-pack/plugin/autoscaling/src/internalClusterTest/java/org/elasticsearch/xpack/autoscaling/storage/AutoscalingStorageIntegTestCase.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
import org.elasticsearch.cluster.routing.allocation.DiskThresholdSettings;
1515
import org.elasticsearch.common.settings.Settings;
1616
import org.elasticsearch.datastreams.DataStreamsPlugin;
17+
import org.elasticsearch.index.engine.ThreadPoolMergeExecutorService;
1718
import org.elasticsearch.plugins.Plugin;
1819
import org.elasticsearch.xpack.autoscaling.LocalStateAutoscaling;
1920
import org.elasticsearch.xpack.autoscaling.action.GetAutoscalingCapacityAction;
@@ -39,7 +40,10 @@ protected Settings nodeSettings(final int nodeOrdinal, final Settings otherSetti
3940
builder.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), LOW_WATERMARK_BYTES + "b")
4041
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), HIGH_WATERMARK_BYTES + "b")
4142
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_DISK_FLOOD_STAGE_WATERMARK_SETTING.getKey(), "0b")
42-
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_REROUTE_INTERVAL_SETTING.getKey(), "0ms");
43+
.put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_REROUTE_INTERVAL_SETTING.getKey(), "0ms")
44+
// the periodicity for the checker for the available disk space as well as the merge tasks' aborting status
45+
// the default of 5 seconds might timeout some tests
46+
.put(ThreadPoolMergeExecutorService.INDICES_MERGE_DISK_CHECK_INTERVAL_SETTING.getKey(), "100ms");
4347
return builder.build();
4448
}
4549

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/LifecycleOperationMetadata.java‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -80,16 +80,21 @@ public static OperationMode currentILMMode(final ProjectMetadata projectMetadata
8080
);
8181
}
8282

83+
@Deprecated(forRemoval = true)
84+
public static OperationMode currentSLMMode(final ClusterState state) {
85+
return currentSLMMode(state.metadata().getProject());
86+
}
87+
8388
/**
8489
* Returns the current ILM mode based on the given cluster state. It first checks the newer
8590
* storage mechanism ({@link LifecycleOperationMetadata#getSLMOperationMode()}) before falling
8691
* back to {@link SnapshotLifecycleMetadata#getOperationMode()}. If neither exist, the default
8792
* value for an empty state is used.
8893
*/
8994
@SuppressWarnings("deprecated")
90-
public static OperationMode currentSLMMode(final ClusterState state) {
91-
SnapshotLifecycleMetadata oldMetadata = state.metadata().getProject().custom(SnapshotLifecycleMetadata.TYPE);
92-
LifecycleOperationMetadata currentMetadata = state.metadata().getProject().custom(LifecycleOperationMetadata.TYPE);
95+
public static OperationMode currentSLMMode(ProjectMetadata project) {
96+
SnapshotLifecycleMetadata oldMetadata = project.custom(SnapshotLifecycleMetadata.TYPE);
97+
LifecycleOperationMetadata currentMetadata = project.custom(LifecycleOperationMetadata.TYPE);
9398
return Optional.ofNullable(currentMetadata)
9499
.map(LifecycleOperationMetadata::getSLMOperationMode)
95100
.orElse(

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/OperationModeUpdateTask.java‎

Lines changed: 33 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,10 @@
1414
import org.elasticsearch.cluster.AckedClusterStateUpdateTask;
1515
import org.elasticsearch.cluster.ClusterState;
1616
import org.elasticsearch.cluster.ClusterStateUpdateTask;
17+
import org.elasticsearch.cluster.metadata.ProjectId;
18+
import org.elasticsearch.cluster.metadata.ProjectMetadata;
1719
import org.elasticsearch.common.Priority;
20+
import org.elasticsearch.core.FixForMultiProject;
1821
import org.elasticsearch.core.Nullable;
1922
import org.elasticsearch.core.Strings;
2023

@@ -29,6 +32,8 @@
2932
*/
3033
public class OperationModeUpdateTask extends ClusterStateUpdateTask {
3134
private static final Logger logger = LogManager.getLogger(OperationModeUpdateTask.class);
35+
36+
private final ProjectId projectId;
3237
@Nullable
3338
private final OperationMode ilmMode;
3439
@Nullable
@@ -47,18 +52,21 @@ public ClusterState execute(ClusterState currentState) {
4752
};
4853
}
4954

50-
private OperationModeUpdateTask(Priority priority, OperationMode ilmMode, OperationMode slmMode) {
55+
private OperationModeUpdateTask(Priority priority, ProjectId projectId, OperationMode ilmMode, OperationMode slmMode) {
5156
super(priority);
57+
this.projectId = projectId;
5258
this.ilmMode = ilmMode;
5359
this.slmMode = slmMode;
5460
}
5561

56-
public static OperationModeUpdateTask ilmMode(OperationMode mode) {
57-
return new OperationModeUpdateTask(getPriority(mode), mode, null);
62+
public static OperationModeUpdateTask ilmMode(ProjectId projectId, OperationMode mode) {
63+
return new OperationModeUpdateTask(getPriority(mode), projectId, mode, null);
5864
}
5965

6066
public static OperationModeUpdateTask slmMode(OperationMode mode) {
61-
return new OperationModeUpdateTask(getPriority(mode), null, mode);
67+
@FixForMultiProject // Use non-default ID when SLM has been made project-aware
68+
final var projectId = ProjectId.DEFAULT;
69+
return new OperationModeUpdateTask(getPriority(mode), projectId, null, mode);
6270
}
6371

6472
private static Priority getPriority(OperationMode mode) {
@@ -79,60 +87,61 @@ public OperationMode getSLMOperationMode() {
7987

8088
@Override
8189
public ClusterState execute(ClusterState currentState) {
82-
ClusterState newState = currentState;
83-
newState = updateILMState(newState);
84-
newState = updateSLMState(newState);
85-
return newState;
90+
ProjectMetadata oldProject = currentState.metadata().getProject(projectId);
91+
ProjectMetadata newProject = updateILMState(oldProject);
92+
newProject = updateSLMState(newProject);
93+
if (newProject == oldProject) {
94+
return currentState;
95+
}
96+
return ClusterState.builder(currentState).putProjectMetadata(newProject).build();
8697
}
8798

88-
private ClusterState updateILMState(final ClusterState currentState) {
99+
private ProjectMetadata updateILMState(final ProjectMetadata currentProject) {
89100
if (ilmMode == null) {
90-
return currentState;
101+
return currentProject;
91102
}
92103

93-
final var project = currentState.metadata().getProject();
94-
final OperationMode currentMode = currentILMMode(project);
104+
final OperationMode currentMode = currentILMMode(currentProject);
95105
if (currentMode.equals(ilmMode)) {
96106
// No need for a new state
97-
return currentState;
107+
return currentProject;
98108
}
99109

100110
final OperationMode newMode;
101111
if (currentMode.isValidChange(ilmMode)) {
102112
newMode = ilmMode;
103113
} else {
104114
// The transition is invalid, return the current state
105-
return currentState;
115+
return currentProject;
106116
}
107117

108118
logger.info("updating ILM operation mode to {}", newMode);
109-
final var updatedMetadata = new LifecycleOperationMetadata(newMode, currentSLMMode(currentState));
110-
return currentState.copyAndUpdateProject(project.id(), b -> b.putCustom(LifecycleOperationMetadata.TYPE, updatedMetadata));
119+
final var updatedMetadata = new LifecycleOperationMetadata(newMode, currentSLMMode(currentProject));
120+
return currentProject.copyAndUpdate(b -> b.putCustom(LifecycleOperationMetadata.TYPE, updatedMetadata));
111121
}
112122

113-
private ClusterState updateSLMState(final ClusterState currentState) {
123+
private ProjectMetadata updateSLMState(final ProjectMetadata currentProject) {
114124
if (slmMode == null) {
115-
return currentState;
125+
return currentProject;
116126
}
117127

118-
final var project = currentState.metadata().getProject();
119-
final OperationMode currentMode = currentSLMMode(currentState);
128+
final OperationMode currentMode = currentSLMMode(currentProject);
120129
if (currentMode.equals(slmMode)) {
121130
// No need for a new state
122-
return currentState;
131+
return currentProject;
123132
}
124133

125134
final OperationMode newMode;
126135
if (currentMode.isValidChange(slmMode)) {
127136
newMode = slmMode;
128137
} else {
129138
// The transition is invalid, return the current state
130-
return currentState;
139+
return currentProject;
131140
}
132141

133142
logger.info("updating SLM operation mode to {}", newMode);
134-
final var updatedMetadata = new LifecycleOperationMetadata(currentILMMode(project), newMode);
135-
return currentState.copyAndUpdateProject(project.id(), b -> b.putCustom(LifecycleOperationMetadata.TYPE, updatedMetadata));
143+
final var updatedMetadata = new LifecycleOperationMetadata(currentILMMode(currentProject), newMode);
144+
return currentProject.copyAndUpdate(b -> b.putCustom(LifecycleOperationMetadata.TYPE, updatedMetadata));
136145
}
137146

138147
@Override

0 commit comments

Comments
 (0)