Skip to content

Commit 2957699

Browse files
markjhoymarkjhoy
authored
Merge branch 'main' into markjhoy/add_sparse_vector_token_pruning_index_options
2 parents 31a3dfc + a0b8737 commit 2957699

File tree

52 files changed

+603
-558
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

52 files changed

+603
-558
lines changed

‎modules/ingest-otel/src/main/java/org/elasticsearch/ingest/otel/NormalizeForStreamPlugin.java‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99

1010
package org.elasticsearch.ingest.otel;
1111

12+
import org.elasticsearch.cluster.metadata.DataStream;
1213
import org.elasticsearch.ingest.Processor;
1314
import org.elasticsearch.plugins.IngestPlugin;
1415
import org.elasticsearch.plugins.Plugin;
@@ -19,6 +20,10 @@ public class NormalizeForStreamPlugin extends Plugin implements IngestPlugin {
1920

2021
@Override
2122
public Map<String, Processor.Factory> getProcessors(Processor.Parameters parameters) {
22-
return Map.of(NormalizeForStreamProcessor.TYPE, new NormalizeForStreamProcessor.Factory());
23+
if (DataStream.LOGS_STREAM_FEATURE_FLAG) {
24+
return Map.of(NormalizeForStreamProcessor.TYPE, new NormalizeForStreamProcessor.Factory());
25+
} else {
26+
return Map.of();
27+
}
2328
}
2429
}

‎modules/ingest-otel/src/yamlRestTest/java/org/elasticsearch/ingest/otel/IngestOtelClientYamlTestSuiteIT.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
1414

1515
import org.elasticsearch.test.cluster.ElasticsearchCluster;
16+
import org.elasticsearch.test.cluster.FeatureFlag;
1617
import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
1718
import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
1819
import org.junit.ClassRule;
@@ -24,7 +25,10 @@ public IngestOtelClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate tes
2425
}
2526

2627
@ClassRule
27-
public static ElasticsearchCluster cluster = ElasticsearchCluster.local().module("ingest-otel").build();
28+
public static ElasticsearchCluster cluster = ElasticsearchCluster.local()
29+
.module("ingest-otel")
30+
.feature(FeatureFlag.LOGS_STREAM)
31+
.build();
2832

2933
@Override
3034
protected String getTestRestCluster() {

‎muted-tests.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -583,6 +583,9 @@ tests:
583583
- class: org.elasticsearch.test.apmintegration.TracesApmIT
584584
method: testApmIntegration
585585
issue: https://github.com/elastic/elasticsearch/issues/129651
586+
- class: org.elasticsearch.xpack.esql.qa.multi_node.EsqlSpecIT
587+
method: test {scoring.TestMatchPhraseWithScoreBoost ASYNC}
588+
issue: https://github.com/elastic/elasticsearch/issues/129676
586589

587590
# Examples:
588591
#

‎test/test-clusters/src/main/java/org/elasticsearch/test/cluster/FeatureFlag.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,8 @@ public enum FeatureFlag {
2121
DOC_VALUES_SKIPPER("es.doc_values_skipper_feature_flag_enabled=true", Version.fromString("8.18.1"), null),
2222
USE_LUCENE101_POSTINGS_FORMAT("es.use_lucene101_postings_format_feature_flag_enabled=true", Version.fromString("9.1.0"), null),
2323
INFERENCE_CUSTOM_SERVICE_ENABLED("es.inference_custom_service_feature_flag_enabled=true", Version.fromString("8.19.0"), null),
24-
IVF_FORMAT("es.ivf_format_feature_flag_enabled=true", Version.fromString("9.1.0"), null);
24+
IVF_FORMAT("es.ivf_format_feature_flag_enabled=true", Version.fromString("9.1.0"), null),
25+
LOGS_STREAM("es.logs_stream_feature_flag_enabled=true", Version.fromString("9.1.0"), null);
2526

2627
public final String systemProperty;
2728
public final Version from;

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/AbstractUnfollowIndexStep.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,8 @@
88

99
import org.elasticsearch.action.ActionListener;
1010
import org.elasticsearch.client.internal.Client;
11-
import org.elasticsearch.cluster.ClusterState;
1211
import org.elasticsearch.cluster.ClusterStateObserver;
12+
import org.elasticsearch.cluster.ProjectState;
1313
import org.elasticsearch.cluster.metadata.IndexMetadata;
1414

1515
import java.util.Map;
@@ -25,7 +25,7 @@ abstract class AbstractUnfollowIndexStep extends AsyncActionStep {
2525
@Override
2626
public final void performAction(
2727
IndexMetadata indexMetadata,
28-
ClusterState currentClusterState,
28+
ProjectState currentState,
2929
ClusterStateObserver observer,
3030
ActionListener<Void> listener
3131
) {
@@ -36,8 +36,8 @@ public final void performAction(
3636
return;
3737
}
3838

39-
innerPerformAction(followerIndex, currentClusterState, listener);
39+
innerPerformAction(followerIndex, currentState, listener);
4040
}
4141

42-
abstract void innerPerformAction(String followerIndex, ClusterState currentClusterState, ActionListener<Void> listener);
42+
abstract void innerPerformAction(String followerIndex, ProjectState currentState, ActionListener<Void> listener);
4343
}

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/AsyncActionStep.java‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,11 @@
88

99
import org.elasticsearch.action.ActionListener;
1010
import org.elasticsearch.client.internal.Client;
11-
import org.elasticsearch.cluster.ClusterState;
1211
import org.elasticsearch.cluster.ClusterStateObserver;
12+
import org.elasticsearch.cluster.ProjectState;
1313
import org.elasticsearch.cluster.metadata.IndexMetadata;
14+
import org.elasticsearch.cluster.metadata.ProjectId;
15+
import org.elasticsearch.core.Nullable;
1416

1517
/**
1618
* Performs an action which must be performed asynchronously because it may take time to complete.
@@ -24,17 +26,23 @@ public AsyncActionStep(StepKey key, StepKey nextStepKey, Client client) {
2426
this.client = client;
2527
}
2628

27-
protected Client getClient() {
29+
// For testing only
30+
@Nullable
31+
Client getClientWithoutProject() {
2832
return client;
2933
}
3034

35+
protected Client getClient(ProjectId projectId) {
36+
return client.projectClient(projectId);
37+
}
38+
3139
public boolean indexSurvives() {
3240
return true;
3341
}
3442

3543
public abstract void performAction(
3644
IndexMetadata indexMetadata,
37-
ClusterState currentClusterState,
45+
ProjectState currentState,
3846
ClusterStateObserver observer,
3947
ActionListener<Void> listener
4048
);

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/AsyncBranchingStep.java‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -10,12 +10,13 @@
1010
import org.apache.lucene.util.SetOnce;
1111
import org.elasticsearch.action.ActionListener;
1212
import org.elasticsearch.client.internal.Client;
13-
import org.elasticsearch.cluster.ClusterState;
1413
import org.elasticsearch.cluster.ClusterStateObserver;
14+
import org.elasticsearch.cluster.ProjectState;
1515
import org.elasticsearch.cluster.metadata.IndexMetadata;
16+
import org.elasticsearch.cluster.metadata.ProjectId;
17+
import org.elasticsearch.common.TriConsumer;
1618

1719
import java.util.Objects;
18-
import java.util.function.BiConsumer;
1920

2021
/**
2122
* This step changes its {@link #getNextStepKey()} depending on the
@@ -26,14 +27,14 @@ public class AsyncBranchingStep extends AsyncActionStep {
2627

2728
private final StepKey nextStepKeyOnFalse;
2829
private final StepKey nextStepKeyOnTrue;
29-
private final BiConsumer<IndexMetadata, ActionListener<Boolean>> asyncPredicate;
30+
private final TriConsumer<ProjectId, IndexMetadata, ActionListener<Boolean>> asyncPredicate;
3031
private final SetOnce<Boolean> predicateValue;
3132

3233
public AsyncBranchingStep(
3334
StepKey key,
3435
StepKey nextStepKeyOnFalse,
3536
StepKey nextStepKeyOnTrue,
36-
BiConsumer<IndexMetadata, ActionListener<Boolean>> asyncPredicate,
37+
TriConsumer<ProjectId, IndexMetadata, ActionListener<Boolean>> asyncPredicate,
3738
Client client
3839
) {
3940
// super.nextStepKey is set to null since it is not used by this step
@@ -52,11 +53,11 @@ public boolean isRetryable() {
5253
@Override
5354
public void performAction(
5455
IndexMetadata indexMetadata,
55-
ClusterState currentClusterState,
56+
ProjectState currentState,
5657
ClusterStateObserver observer,
5758
ActionListener<Void> listener
5859
) {
59-
asyncPredicate.accept(indexMetadata, listener.safeMap(value -> {
60+
asyncPredicate.apply(currentState.projectId(), indexMetadata, listener.safeMap(value -> {
6061
predicateValue.set(value);
6162
return null;
6263
}));
@@ -87,7 +88,7 @@ final StepKey getNextStepKeyOnTrue() {
8788
/**
8889
* @return the next step if {@code predicate} is true
8990
*/
90-
final BiConsumer<IndexMetadata, ActionListener<Boolean>> getAsyncPredicate() {
91+
final TriConsumer<ProjectId, IndexMetadata, ActionListener<Boolean>> getAsyncPredicate() {
9192
return asyncPredicate;
9293
}
9394

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/AsyncRetryDuringSnapshotActionStep.java‎

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,11 @@
1414
import org.elasticsearch.cluster.ClusterState;
1515
import org.elasticsearch.cluster.ClusterStateObserver;
1616
import org.elasticsearch.cluster.NotMasterException;
17+
import org.elasticsearch.cluster.ProjectState;
1718
import org.elasticsearch.cluster.SnapshotsInProgress;
1819
import org.elasticsearch.cluster.metadata.IndexMetadata;
20+
import org.elasticsearch.cluster.metadata.ProjectId;
21+
import org.elasticsearch.cluster.metadata.ProjectMetadata;
1922
import org.elasticsearch.cluster.node.DiscoveryNode;
2023
import org.elasticsearch.core.TimeValue;
2124
import org.elasticsearch.index.Index;
@@ -39,44 +42,48 @@ public AsyncRetryDuringSnapshotActionStep(StepKey key, StepKey nextStepKey, Clie
3942
@Override
4043
public final void performAction(
4144
IndexMetadata indexMetadata,
42-
ClusterState currentClusterState,
45+
ProjectState currentState,
4346
ClusterStateObserver observer,
4447
ActionListener<Void> listener
4548
) {
4649
// Wrap the original listener to handle exceptions caused by ongoing snapshots
4750
SnapshotExceptionListener snapshotExceptionListener = new SnapshotExceptionListener(
51+
currentState.projectId(),
4852
indexMetadata.getIndex(),
4953
listener,
5054
observer,
51-
currentClusterState.nodes().getLocalNode()
55+
currentState.cluster().nodes().getLocalNode()
5256
);
53-
performDuringNoSnapshot(indexMetadata, currentClusterState, snapshotExceptionListener);
57+
performDuringNoSnapshot(indexMetadata, currentState.metadata(), snapshotExceptionListener);
5458
}
5559

5660
/**
5761
* Method to be performed during which no snapshots for the index are already underway.
5862
*/
59-
abstract void performDuringNoSnapshot(IndexMetadata indexMetadata, ClusterState currentClusterState, ActionListener<Void> listener);
63+
abstract void performDuringNoSnapshot(IndexMetadata indexMetadata, ProjectMetadata currentProject, ActionListener<Void> listener);
6064

6165
/**
6266
* SnapshotExceptionListener is an injected listener wrapper that checks to see if a particular
6367
* action failed due to a {@code SnapshotInProgressException}. If it did, then it registers a
6468
* ClusterStateObserver listener waiting for the next time the snapshot is not running,
65-
* re-running the step's {@link #performAction(IndexMetadata, ClusterState, ClusterStateObserver, ActionListener)}
69+
* re-running the step's {@link #performAction(IndexMetadata, ProjectState, ClusterStateObserver, ActionListener)}
6670
* method when the snapshot is no longer running.
6771
*/
6872
class SnapshotExceptionListener implements ActionListener<Void> {
73+
private final ProjectId projectId;
6974
private final Index index;
7075
private final ActionListener<Void> originalListener;
7176
private final ClusterStateObserver observer;
7277
private final DiscoveryNode localNode;
7378

7479
SnapshotExceptionListener(
80+
ProjectId projectId,
7581
Index index,
7682
ActionListener<Void> originalListener,
7783
ClusterStateObserver observer,
7884
DiscoveryNode localNode
7985
) {
86+
this.projectId = projectId;
8087
this.index = index;
8188
this.originalListener = originalListener;
8289
this.observer = observer;
@@ -106,13 +113,14 @@ public void onNewClusterState(ClusterState state) {
106113
}
107114
try {
108115
logger.debug("[{}] retrying ILM step after snapshot has completed", indexName);
109-
IndexMetadata idxMeta = state.metadata().getProject().index(index);
116+
final var projectState = state.projectState(projectId);
117+
IndexMetadata idxMeta = projectState.metadata().index(index);
110118
if (idxMeta == null) {
111119
// The index has since been deleted, mission accomplished!
112120
originalListener.onResponse(null);
113121
} else {
114122
// Re-invoke the performAction method with the new state
115-
performAction(idxMeta, state, observer, originalListener);
123+
performAction(idxMeta, projectState, observer, originalListener);
116124
}
117125
} catch (Exception e) {
118126
originalListener.onFailure(e);
@@ -133,7 +141,7 @@ public void onTimeout(TimeValue timeout) {
133141
// ILM actions should only run on master, lets bail on failover
134142
return true;
135143
}
136-
if (state.metadata().getProject().index(index) == null) {
144+
if (state.metadata().getProject(projectId).index(index) == null) {
137145
// The index has since been deleted, mission accomplished!
138146
return true;
139147
}

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/CleanupShrinkIndexStep.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,9 @@
1212
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
1313
import org.elasticsearch.action.support.master.AcknowledgedResponse;
1414
import org.elasticsearch.client.internal.Client;
15-
import org.elasticsearch.cluster.ClusterState;
1615
import org.elasticsearch.cluster.metadata.IndexMetadata;
1716
import org.elasticsearch.cluster.metadata.LifecycleExecutionState;
17+
import org.elasticsearch.cluster.metadata.ProjectMetadata;
1818
import org.elasticsearch.common.Strings;
1919
import org.elasticsearch.core.TimeValue;
2020
import org.elasticsearch.index.IndexNotFoundException;
@@ -36,11 +36,11 @@ public boolean isRetryable() {
3636
}
3737

3838
@Override
39-
void performDuringNoSnapshot(IndexMetadata indexMetadata, ClusterState currentClusterState, ActionListener<Void> listener) {
39+
void performDuringNoSnapshot(IndexMetadata indexMetadata, ProjectMetadata currentProject, ActionListener<Void> listener) {
4040
final String shrunkenIndexSource = IndexMetadata.INDEX_RESIZE_SOURCE_NAME.get(indexMetadata.getSettings());
4141
if (Strings.isNullOrEmpty(shrunkenIndexSource) == false) {
4242
// the current managed index is a shrunk index
43-
if (currentClusterState.metadata().getProject().index(shrunkenIndexSource) == null) {
43+
if (currentProject.index(shrunkenIndexSource) == null) {
4444
// if the source index does not exist, we'll skip deleting the
4545
// (managed) shrunk index as that will cause data loss
4646
String policyName = indexMetadata.getLifecyclePolicyName();
@@ -64,7 +64,7 @@ void performDuringNoSnapshot(IndexMetadata indexMetadata, ClusterState currentCl
6464
listener.onResponse(null);
6565
return;
6666
}
67-
getClient().admin()
67+
getClient(currentProject.id()).admin()
6868
.indices()
6969
.delete(new DeleteIndexRequest(shrinkIndexName).masterNodeTimeout(TimeValue.MAX_VALUE), new ActionListener<>() {
7070
@Override

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/CleanupSnapshotStep.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,9 @@
99
import org.elasticsearch.action.ActionListener;
1010
import org.elasticsearch.action.support.master.AcknowledgedResponse;
1111
import org.elasticsearch.client.internal.Client;
12-
import org.elasticsearch.cluster.ClusterState;
1312
import org.elasticsearch.cluster.metadata.IndexMetadata;
1413
import org.elasticsearch.cluster.metadata.LifecycleExecutionState;
14+
import org.elasticsearch.cluster.metadata.ProjectMetadata;
1515
import org.elasticsearch.common.Strings;
1616
import org.elasticsearch.core.TimeValue;
1717
import org.elasticsearch.repositories.RepositoryMissingException;
@@ -33,7 +33,7 @@ public boolean isRetryable() {
3333
}
3434

3535
@Override
36-
void performDuringNoSnapshot(IndexMetadata indexMetadata, ClusterState currentClusterState, ActionListener<Void> listener) {
36+
void performDuringNoSnapshot(IndexMetadata indexMetadata, ProjectMetadata currentProject, ActionListener<Void> listener) {
3737
final String indexName = indexMetadata.getIndex().getName();
3838

3939
LifecycleExecutionState lifecycleState = indexMetadata.getLifecycleExecutionState();
@@ -48,7 +48,7 @@ void performDuringNoSnapshot(IndexMetadata indexMetadata, ClusterState currentCl
4848
listener.onResponse(null);
4949
return;
5050
}
51-
getClient().admin()
51+
getClient(currentProject.id()).admin()
5252
.cluster()
5353
.prepareDeleteSnapshot(TimeValue.MAX_VALUE, repositoryName, snapshotName)
5454
.execute(new ActionListener<>() {

0 commit comments

Comments
 (0)