Skip to content

Commit 8bd4645

Browse files
nielsbaumannielsbauman
authored
Make XPackUsage APIs project-aware (#130358)
Updates the APIs to be able to handle multiple projects. This commit does not guarantee that every sub-API returns sensible results in a multi-project context. Some APIs/features potentially require further work to return sensible data for multiple projects. Since this API is for internal use only in serverless, we can check at a later time whether any features require additional work.
1 parent f88aaa3 commit 8bd4645

File tree

16 files changed

+122
-61
lines changed

16 files changed

+122
-61
lines changed

‎x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/CCRUsageTransportAction.java‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
import org.elasticsearch.action.support.ActionFilters;
1111
import org.elasticsearch.cluster.ClusterState;
1212
import org.elasticsearch.cluster.metadata.IndexMetadata;
13-
import org.elasticsearch.cluster.metadata.Metadata;
13+
import org.elasticsearch.cluster.project.ProjectResolver;
1414
import org.elasticsearch.cluster.service.ClusterService;
1515
import org.elasticsearch.common.settings.Settings;
1616
import org.elasticsearch.injection.guice.Inject;
@@ -32,6 +32,7 @@ public class CCRUsageTransportAction extends XPackUsageFeatureTransportAction {
3232

3333
private final Settings settings;
3434
private final XPackLicenseState licenseState;
35+
private final ProjectResolver projectResolver;
3536

3637
@Inject
3738
public CCRUsageTransportAction(
@@ -40,11 +41,13 @@ public CCRUsageTransportAction(
4041
ThreadPool threadPool,
4142
ActionFilters actionFilters,
4243
Settings settings,
43-
XPackLicenseState licenseState
44+
XPackLicenseState licenseState,
45+
ProjectResolver projectResolver
4446
) {
4547
super(XPackUsageFeatureAction.CCR.name(), transportService, clusterService, threadPool, actionFilters);
4648
this.settings = settings;
4749
this.licenseState = licenseState;
50+
this.projectResolver = projectResolver;
4851
}
4952

5053
@Override
@@ -54,19 +57,19 @@ protected void localClusterStateOperation(
5457
ClusterState state,
5558
ActionListener<XPackUsageFeatureResponse> listener
5659
) {
57-
Metadata metadata = state.metadata();
60+
final var project = projectResolver.getProjectMetadata(state);
5861

5962
int numberOfFollowerIndices = 0;
6063
long lastFollowerIndexCreationDate = 0L;
61-
for (IndexMetadata imd : metadata.getProject()) {
64+
for (IndexMetadata imd : project) {
6265
if (imd.getCustomData("ccr") != null) {
6366
numberOfFollowerIndices++;
6467
if (lastFollowerIndexCreationDate < imd.getCreationDate()) {
6568
lastFollowerIndexCreationDate = imd.getCreationDate();
6669
}
6770
}
6871
}
69-
AutoFollowMetadata autoFollowMetadata = metadata.getProject().custom(AutoFollowMetadata.TYPE);
72+
AutoFollowMetadata autoFollowMetadata = project.custom(AutoFollowMetadata.TYPE);
7073
int numberOfAutoFollowPatterns = autoFollowMetadata != null ? autoFollowMetadata.getPatterns().size() : 0;
7174

7275
Long lastFollowTimeInMillis;

‎x-pack/plugin/ccr/src/test/java/org/elasticsearch/xpack/ccr/CCRInfoTransportActionTests.java‎

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,8 @@
1111
import org.elasticsearch.cluster.ClusterName;
1212
import org.elasticsearch.cluster.ClusterState;
1313
import org.elasticsearch.cluster.metadata.IndexMetadata;
14-
import org.elasticsearch.cluster.metadata.Metadata;
14+
import org.elasticsearch.cluster.metadata.ProjectMetadata;
15+
import org.elasticsearch.cluster.project.TestProjectResolvers;
1516
import org.elasticsearch.cluster.service.ClusterService;
1617
import org.elasticsearch.common.settings.Settings;
1718
import org.elasticsearch.common.util.Maps;
@@ -92,7 +93,8 @@ public void testName() {
9293
}
9394

9495
public void testUsageStats() throws Exception {
95-
Metadata.Builder metadata = Metadata.builder();
96+
final var projectId = randomProjectIdOrDefault();
97+
ProjectMetadata.Builder project = ProjectMetadata.builder(projectId);
9698

9799
int numFollowerIndices = randomIntBetween(0, 32);
98100
for (int i = 0; i < numFollowerIndices; i++) {
@@ -102,7 +104,7 @@ public void testUsageStats() throws Exception {
102104
.numberOfReplicas(0)
103105
.creationDate(i)
104106
.putCustom(Ccr.CCR_CUSTOM_METADATA_KEY, new HashMap<>());
105-
metadata.put(followerIndex);
107+
project.put(followerIndex);
106108
}
107109

108110
// Add a regular index, to check that we do not take that one into account:
@@ -111,7 +113,7 @@ public void testUsageStats() throws Exception {
111113
.numberOfShards(1)
112114
.numberOfReplicas(0)
113115
.creationDate(numFollowerIndices);
114-
metadata.put(regularIndex);
116+
project.put(regularIndex);
115117

116118
int numAutoFollowPatterns = randomIntBetween(0, 32);
117119
Map<String, AutoFollowMetadata.AutoFollowPattern> patterns = Maps.newMapWithExpectedSize(numAutoFollowPatterns);
@@ -136,9 +138,9 @@ public void testUsageStats() throws Exception {
136138
);
137139
patterns.put("pattern" + i, pattern);
138140
}
139-
metadata.putCustom(AutoFollowMetadata.TYPE, new AutoFollowMetadata(patterns, Collections.emptyMap(), Collections.emptyMap()));
141+
project.putCustom(AutoFollowMetadata.TYPE, new AutoFollowMetadata(patterns, Collections.emptyMap(), Collections.emptyMap()));
140142

141-
ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).metadata(metadata).build();
143+
ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).putProjectMetadata(project).build();
142144
Mockito.when(clusterService.state()).thenReturn(clusterState);
143145

144146
ThreadPool threadPool = mock(ThreadPool.class);
@@ -149,7 +151,8 @@ public void testUsageStats() throws Exception {
149151
threadPool,
150152
mock(ActionFilters.class),
151153
Settings.EMPTY,
152-
licenseState
154+
licenseState,
155+
TestProjectResolvers.singleProject(projectId)
153156
);
154157
PlainActionFuture<XPackUsageFeatureResponse> future = new PlainActionFuture<>();
155158
usageAction.localClusterStateOperation(null, null, clusterState, future);

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/DataStreamLifecycleUsageTransportAction.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention;
1616
import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionSettings;
1717
import org.elasticsearch.cluster.metadata.DataStreamLifecycle;
18+
import org.elasticsearch.cluster.project.ProjectResolver;
1819
import org.elasticsearch.cluster.service.ClusterService;
1920
import org.elasticsearch.core.TimeValue;
2021
import org.elasticsearch.core.Tuple;
@@ -32,17 +33,20 @@
3233
public class DataStreamLifecycleUsageTransportAction extends XPackUsageFeatureTransportAction {
3334

3435
private final DataStreamGlobalRetentionSettings globalRetentionSettings;
36+
private final ProjectResolver projectResolver;
3537

3638
@Inject
3739
public DataStreamLifecycleUsageTransportAction(
3840
TransportService transportService,
3941
ClusterService clusterService,
4042
ThreadPool threadPool,
4143
ActionFilters actionFilters,
42-
DataStreamGlobalRetentionSettings globalRetentionSettings
44+
DataStreamGlobalRetentionSettings globalRetentionSettings,
45+
ProjectResolver projectResolver
4346
) {
4447
super(XPackUsageFeatureAction.DATA_STREAM_LIFECYCLE.name(), transportService, clusterService, threadPool, actionFilters);
4548
this.globalRetentionSettings = globalRetentionSettings;
49+
this.projectResolver = projectResolver;
4650
}
4751

4852
@Override
@@ -52,7 +56,7 @@ protected void localClusterStateOperation(
5256
ClusterState state,
5357
ActionListener<XPackUsageFeatureResponse> listener
5458
) {
55-
final Collection<DataStream> dataStreams = state.metadata().getProject().dataStreams().values();
59+
final Collection<DataStream> dataStreams = projectResolver.getProjectMetadata(state).dataStreams().values();
5660
DataStreamLifecycleFeatureSetUsage.LifecycleStats lifecycleStats = calculateStats(
5761
dataStreams,
5862
clusterService.getClusterSettings().get(DataStreamLifecycle.CLUSTER_LIFECYCLE_DEFAULT_ROLLOVER_SETTING),

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/DataStreamUsageTransportAction.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
import org.elasticsearch.cluster.metadata.DataStreamGlobalRetention;
1616
import org.elasticsearch.cluster.metadata.DataStreamGlobalRetentionSettings;
1717
import org.elasticsearch.cluster.metadata.DataStreamLifecycle;
18+
import org.elasticsearch.cluster.project.ProjectResolver;
1819
import org.elasticsearch.cluster.service.ClusterService;
1920
import org.elasticsearch.core.TimeValue;
2021
import org.elasticsearch.core.Tuple;
@@ -33,6 +34,7 @@ public class DataStreamUsageTransportAction extends XPackUsageFeatureTransportAc
3334

3435
private final DataStreamFailureStoreSettings dataStreamFailureStoreSettings;
3536
private final DataStreamGlobalRetentionSettings globalRetentionSettings;
37+
private final ProjectResolver projectResolver;
3638

3739
@Inject
3840
public DataStreamUsageTransportAction(
@@ -41,11 +43,13 @@ public DataStreamUsageTransportAction(
4143
ThreadPool threadPool,
4244
ActionFilters actionFilters,
4345
DataStreamFailureStoreSettings dataStreamFailureStoreSettings,
44-
DataStreamGlobalRetentionSettings globalRetentionSettings
46+
DataStreamGlobalRetentionSettings globalRetentionSettings,
47+
ProjectResolver projectResolver
4548
) {
4649
super(XPackUsageFeatureAction.DATA_STREAMS.name(), transportService, clusterService, threadPool, actionFilters);
4750
this.dataStreamFailureStoreSettings = dataStreamFailureStoreSettings;
4851
this.globalRetentionSettings = globalRetentionSettings;
52+
this.projectResolver = projectResolver;
4953
}
5054

5155
@Override
@@ -55,7 +59,7 @@ protected void localClusterStateOperation(
5559
ClusterState state,
5660
ActionListener<XPackUsageFeatureResponse> listener
5761
) {
58-
final Map<String, DataStream> dataStreams = state.metadata().getProject().dataStreams();
62+
final Map<String, DataStream> dataStreams = projectResolver.getProjectMetadata(state).dataStreams();
5963
long backingIndicesCounter = 0;
6064
long failureStoreExplicitlyEnabledCounter = 0;
6165
long failureStoreEffectivelyEnabledCounter = 0;

‎x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/datatiers/DataTiersUsageTransportAction.java‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,11 @@
1313
import org.elasticsearch.client.internal.Client;
1414
import org.elasticsearch.client.internal.ParentTaskAssigningClient;
1515
import org.elasticsearch.cluster.ClusterState;
16+
import org.elasticsearch.cluster.ProjectState;
1617
import org.elasticsearch.cluster.metadata.IndexMetadata;
1718
import org.elasticsearch.cluster.metadata.Metadata;
1819
import org.elasticsearch.cluster.node.DiscoveryNodeRole;
20+
import org.elasticsearch.cluster.project.ProjectResolver;
1921
import org.elasticsearch.cluster.routing.RoutingNode;
2022
import org.elasticsearch.cluster.routing.ShardRouting;
2123
import org.elasticsearch.cluster.routing.allocation.DataTier;
@@ -43,17 +45,20 @@
4345
public class DataTiersUsageTransportAction extends XPackUsageFeatureTransportAction {
4446

4547
private final Client client;
48+
private final ProjectResolver projectResolver;
4649

4750
@Inject
4851
public DataTiersUsageTransportAction(
4952
TransportService transportService,
5053
ClusterService clusterService,
5154
ThreadPool threadPool,
5255
ActionFilters actionFilters,
53-
Client client
56+
Client client,
57+
ProjectResolver projectResolver
5458
) {
5559
super(XPackUsageFeatureAction.DATA_TIERS.name(), transportService, clusterService, threadPool, actionFilters);
5660
this.client = client;
61+
this.projectResolver = projectResolver;
5762
}
5863

5964
@Override
@@ -69,11 +74,12 @@ protected void localClusterStateOperation(
6974
NodesDataTiersUsageTransportAction.TYPE,
7075
new NodesDataTiersUsageTransportAction.NodesRequest(),
7176
listener.delegateFailureAndWrap((delegate, response) -> {
77+
final var projectState = projectResolver.getProjectState(state);
7278
// Generate tier specific stats for the nodes and indices
7379
delegate.onResponse(
7480
new XPackUsageFeatureResponse(
7581
new DataTiersFeatureSetUsage(
76-
aggregateStats(response.getNodes(), getIndicesGroupedByTier(state, response.getNodes()))
82+
aggregateStats(response.getNodes(), getIndicesGroupedByTier(projectState, response.getNodes()))
7783
)
7884
)
7985
);
@@ -82,16 +88,16 @@ protected void localClusterStateOperation(
8288
}
8389

8490
// Visible for testing
85-
static Map<String, Set<String>> getIndicesGroupedByTier(ClusterState state, List<NodeDataTiersUsage> nodes) {
91+
static Map<String, Set<String>> getIndicesGroupedByTier(ProjectState state, List<NodeDataTiersUsage> nodes) {
8692
Set<String> indices = nodes.stream()
87-
.map(nodeResponse -> state.getRoutingNodes().node(nodeResponse.getNode().getId()))
93+
.map(nodeResponse -> state.cluster().getRoutingNodes().node(nodeResponse.getNode().getId()))
8894
.filter(Objects::nonNull)
8995
.flatMap(node -> StreamSupport.stream(node.spliterator(), false))
9096
.map(ShardRouting::getIndexName)
9197
.collect(Collectors.toSet());
9298
Map<String, Set<String>> indicesByTierPreference = new HashMap<>();
9399
for (String indexName : indices) {
94-
IndexMetadata indexMetadata = state.metadata().getProject().index(indexName);
100+
IndexMetadata indexMetadata = state.metadata().index(indexName);
95101
// If the index was deleted in the meantime, skip
96102
if (indexMetadata == null) {
97103
continue;

‎x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/datatiers/DataTiersUsageTransportActionTests.java‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
import org.elasticsearch.cluster.ClusterName;
1111
import org.elasticsearch.cluster.ClusterState;
1212
import org.elasticsearch.cluster.metadata.IndexMetadata;
13-
import org.elasticsearch.cluster.metadata.Metadata;
13+
import org.elasticsearch.cluster.metadata.ProjectMetadata;
1414
import org.elasticsearch.cluster.node.DiscoveryNode;
1515
import org.elasticsearch.cluster.node.DiscoveryNodeRole;
1616
import org.elasticsearch.cluster.node.DiscoveryNodes;
@@ -57,15 +57,14 @@ public void testTierIndices() {
5757
IndexMetadata nonTiered = indexMetadata("non-tier", 1, 0); // No tier
5858
IndexMetadata hotIndex3 = indexMetadata("hot-3", 1, 0, DataTier.DATA_HOT);
5959

60-
Metadata.Builder metadataBuilder = Metadata.builder()
60+
ProjectMetadata.Builder projectBuilder = ProjectMetadata.builder(randomProjectIdOrDefault())
6161
.put(hotIndex1, false)
6262
.put(hotIndex2, false)
6363
.put(warmIndex1, false)
6464
.put(coldIndex1, false)
6565
.put(coldIndex2, false)
6666
.put(nonTiered, false)
67-
.put(hotIndex3, false)
68-
.generateClusterUuidIfNeeded();
67+
.put(hotIndex3, false);
6968
RoutingTable.Builder routingTableBuilder = RoutingTable.builder();
7069
routingTableBuilder.add(getIndexRoutingTable(hotIndex1, dataNode));
7170
routingTableBuilder.add(getIndexRoutingTable(hotIndex2, dataNode));
@@ -76,11 +75,11 @@ public void testTierIndices() {
7675
routingTableBuilder.add(getIndexRoutingTable(nonTiered, dataNode));
7776
ClusterState clusterState = ClusterState.builder(new ClusterName("test"))
7877
.nodes(discoBuilder)
79-
.metadata(metadataBuilder)
80-
.routingTable(routingTableBuilder.build())
78+
.putProjectMetadata(projectBuilder)
79+
.putRoutingTable(projectBuilder.getId(), routingTableBuilder.build())
8180
.build();
8281
Map<String, Set<String>> result = DataTiersUsageTransportAction.getIndicesGroupedByTier(
83-
clusterState,
82+
clusterState.projectState(projectBuilder.getId()),
8483
List.of(new NodeDataTiersUsage(dataNode, Map.of(DataTier.DATA_WARM, createStats(5, 5, 0, 10))))
8584
);
8685
assertThat(result.keySet(), equalTo(Set.of(DataTier.DATA_HOT, DataTier.DATA_WARM, DataTier.DATA_COLD)));

‎x-pack/plugin/ilm/src/main/java/org/elasticsearch/xpack/ilm/IndexLifecycleUsageTransportAction.java‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
import org.elasticsearch.action.admin.indices.rollover.RolloverConditions;
1111
import org.elasticsearch.action.support.ActionFilters;
1212
import org.elasticsearch.cluster.ClusterState;
13-
import org.elasticsearch.cluster.metadata.Metadata;
13+
import org.elasticsearch.cluster.project.ProjectResolver;
1414
import org.elasticsearch.cluster.service.ClusterService;
1515
import org.elasticsearch.core.Tuple;
1616
import org.elasticsearch.injection.guice.Inject;
@@ -42,14 +42,18 @@
4242

4343
public class IndexLifecycleUsageTransportAction extends XPackUsageFeatureTransportAction {
4444

45+
private final ProjectResolver projectResolver;
46+
4547
@Inject
4648
public IndexLifecycleUsageTransportAction(
4749
TransportService transportService,
4850
ClusterService clusterService,
4951
ThreadPool threadPool,
50-
ActionFilters actionFilters
52+
ActionFilters actionFilters,
53+
ProjectResolver projectResolver
5154
) {
5255
super(XPackUsageFeatureAction.INDEX_LIFECYCLE.name(), transportService, clusterService, threadPool, actionFilters);
56+
this.projectResolver = projectResolver;
5357
}
5458

5559
@Override
@@ -59,12 +63,12 @@ protected void localClusterStateOperation(
5963
ClusterState state,
6064
ActionListener<XPackUsageFeatureResponse> listener
6165
) {
62-
Metadata metadata = state.metadata();
63-
IndexLifecycleMetadata lifecycleMetadata = metadata.getProject().custom(IndexLifecycleMetadata.TYPE);
66+
final var project = projectResolver.getProjectMetadata(state);
67+
IndexLifecycleMetadata lifecycleMetadata = project.custom(IndexLifecycleMetadata.TYPE);
6468
final IndexLifecycleFeatureSetUsage usage;
6569
if (lifecycleMetadata != null) {
6670
Map<String, Integer> policyUsage = new HashMap<>();
67-
metadata.getProject().indices().values().forEach(value -> {
71+
project.indices().values().forEach(value -> {
6872
String policyName = value.getLifecyclePolicyName();
6973
Integer indicesManaged = policyUsage.get(policyName);
7074
if (indicesManaged == null) {

0 commit comments

Comments
 (0)