diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java index 839ac9c7653e4..7c04c38eff63c 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamIT.java @@ -1925,7 +1925,7 @@ public ClusterState execute(ClusterState currentState) throws Exception { DataStream original = currentState.getMetadata().dataStreams().get(dataStreamName); DataStream broken = original.copy() .setBackingIndices( - original.getBackingIndices() + original.getDataComponent() .copy() .setIndices( List.of(new Index(original.getIndices().get(0).getName(), "broken"), original.getIndices().get(1)) diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java index 40bde501f0bfd..0ae7504bb9d7f 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/DataStreamsSnapshotsIT.java @@ -148,7 +148,7 @@ public void setup() throws Exception { dsBackingIndexName = dataStreamInfos.get(0).getDataStream().getIndices().get(0).getName(); otherDsBackingIndexName = dataStreamInfos.get(1).getDataStream().getIndices().get(0).getName(); fsBackingIndexName = dataStreamInfos.get(2).getDataStream().getIndices().get(0).getName(); - fsFailureIndexName = dataStreamInfos.get(2).getDataStream().getFailureIndices().getIndices().get(0).getName(); + fsFailureIndexName = dataStreamInfos.get(2).getDataStream().getFailureIndices().get(0).getName(); // Will be used in some tests, to test renaming while restoring a snapshot: ds2BackingIndexName = dsBackingIndexName.replace("-ds-", "-ds2-"); @@ -279,7 +279,7 @@ public void testSnapshotAndRestoreAllDataStreamsInPlace() throws Exception { assertThat(backingIndices.stream().map(Index::getName).collect(Collectors.toList()), contains(otherDsBackingIndexName)); backingIndices = dataStreamInfos.get(2).getDataStream().getIndices(); assertThat(backingIndices.stream().map(Index::getName).collect(Collectors.toList()), contains(fsBackingIndexName)); - List failureIndices = dataStreamInfos.get(2).getDataStream().getFailureIndices().getIndices(); + List failureIndices = dataStreamInfos.get(2).getDataStream().getFailureIndices(); assertThat(failureIndices.stream().map(Index::getName).collect(Collectors.toList()), contains(fsFailureIndexName)); } @@ -375,7 +375,7 @@ public void testFailureStoreSnapshotAndRestore() throws Exception { assertEquals(1, dataStreamInfos.size()); assertEquals(1, dataStreamInfos.get(0).getDataStream().getIndices().size()); assertEquals(fsBackingIndexName, dataStreamInfos.get(0).getDataStream().getIndices().get(0).getName()); - assertEquals(fsFailureIndexName, dataStreamInfos.get(0).getDataStream().getFailureIndices().getIndices().get(0).getName()); + assertEquals(fsFailureIndexName, dataStreamInfos.get(0).getDataStream().getFailureIndices().get(0).getName()); } { // With rename pattern @@ -394,7 +394,7 @@ public void testFailureStoreSnapshotAndRestore() throws Exception { assertEquals(1, dataStreamInfos.size()); assertEquals(1, dataStreamInfos.get(0).getDataStream().getIndices().size()); assertEquals(fs2BackingIndexName, dataStreamInfos.get(0).getDataStream().getIndices().get(0).getName()); - assertEquals(fs2FailureIndexName, dataStreamInfos.get(0).getDataStream().getFailureIndices().getIndices().get(0).getName()); + assertEquals(fs2FailureIndexName, dataStreamInfos.get(0).getDataStream().getFailureIndices().get(0).getName()); } } @@ -587,8 +587,8 @@ public void testSnapshotAndRestoreAll() throws Exception { assertEquals(otherDsBackingIndexName, dataStreamInfos.get(1).getDataStream().getIndices().get(0).getName()); assertEquals(1, dataStreamInfos.get(2).getDataStream().getIndices().size()); assertEquals(fsBackingIndexName, dataStreamInfos.get(2).getDataStream().getIndices().get(0).getName()); - assertEquals(1, dataStreamInfos.get(2).getDataStream().getFailureIndices().getIndices().size()); - assertEquals(fsFailureIndexName, dataStreamInfos.get(2).getDataStream().getFailureIndices().getIndices().get(0).getName()); + assertEquals(1, dataStreamInfos.get(2).getDataStream().getFailureIndices().size()); + assertEquals(fsFailureIndexName, dataStreamInfos.get(2).getDataStream().getFailureIndices().get(0).getName()); GetAliasesResponse getAliasesResponse = client.admin() .indices() @@ -659,7 +659,7 @@ public void testSnapshotAndRestoreIncludeAliasesFalse() throws Exception { assertEquals(1, dataStreamInfos.get(2).getDataStream().getIndices().size()); assertEquals(fsBackingIndexName, dataStreamInfos.get(2).getDataStream().getIndices().get(0).getName()); assertEquals(1, dataStreamInfos.get(2).getDataStream().getIndices().size()); - assertEquals(fsFailureIndexName, dataStreamInfos.get(2).getDataStream().getFailureIndices().getIndices().get(0).getName()); + assertEquals(fsFailureIndexName, dataStreamInfos.get(2).getDataStream().getFailureIndices().get(0).getName()); GetAliasesResponse getAliasesResponse = client.admin() .indices() @@ -1257,8 +1257,8 @@ public void testExcludeDSFromSnapshotWhenExcludingAnyOfItsIndices() { assertThat(restoreSnapshotResponse.failedShards(), is(0)); GetDataStreamAction.Response.DataStreamInfo dataStream = getDataStreamInfo(dataStreamName).getFirst(); - assertThat(dataStream.getDataStream().getBackingIndices().getIndices(), not(empty())); - assertThat(dataStream.getDataStream().getFailureIndices().getIndices(), empty()); + assertThat(dataStream.getDataStream().getDataComponent().getIndices(), not(empty())); + assertThat(dataStream.getDataStream().getFailureIndices(), empty()); } } diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/LazyRolloverDuringDisruptionIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/LazyRolloverDuringDisruptionIT.java index 00dfd5c65b126..448f5356ff073 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/LazyRolloverDuringDisruptionIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/LazyRolloverDuringDisruptionIT.java @@ -58,7 +58,7 @@ public void testRolloverIsExecutedOnce() throws ExecutionException, InterruptedE // Verify that the data stream is marked for rollover and that it has currently one index DataStream dataStream = getDataStream(dataStreamName); assertThat(dataStream.rolloverOnWrite(), equalTo(true)); - assertThat(dataStream.getBackingIndices().getIndices().size(), equalTo(1)); + assertThat(dataStream.getDataComponent().getIndices().size(), equalTo(1)); // Introduce a disruption to the master node that should delay the rollover execution final var barrier = new CyclicBarrier(2); @@ -107,7 +107,7 @@ public void onFailure(Exception e) { // Verify that the rollover has happened once dataStream = getDataStream(dataStreamName); assertThat(dataStream.rolloverOnWrite(), equalTo(false)); - assertThat(dataStream.getBackingIndices().getIndices().size(), equalTo(2)); + assertThat(dataStream.getDataComponent().getIndices().size(), equalTo(2)); } private DataStream getDataStream(String dataStreamName) { diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java index cf8380a5c859c..0bc4157eb2e48 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java @@ -285,7 +285,7 @@ private void assertDataStreamBackingIndicesModes(final String dataStreamName, fi final GetDataStreamAction.Response getDataStreamResponse = client().execute(GetDataStreamAction.INSTANCE, getDataStreamRequest) .actionGet(); final DataStream dataStream = getDataStreamResponse.getDataStreams().get(0).getDataStream(); - final DataStream.DataStreamIndices backingIndices = dataStream.getBackingIndices(); + final DataStream.DataStreamIndices backingIndices = dataStream.getDataComponent(); final Iterator indexModesIterator = modes.iterator(); assertThat(backingIndices.getIndices().size(), Matchers.equalTo(modes.size())); for (final Index index : backingIndices.getIndices()) { diff --git a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceIT.java b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceIT.java index 19067d85a6805..1c00e3ad380dc 100644 --- a/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceIT.java +++ b/modules/data-streams/src/internalClusterTest/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceIT.java @@ -1084,7 +1084,7 @@ public void testLifecycleAppliedToFailureStore() throws Exception { assertThat(getDataStreamResponse.getDataStreams().get(0).getDataStream().getName(), equalTo(dataStreamName)); List backingIndices = getDataStreamResponse.getDataStreams().get(0).getDataStream().getIndices(); assertThat(backingIndices.size(), equalTo(1)); - List failureIndices = getDataStreamResponse.getDataStreams().get(0).getDataStream().getFailureIndices().getIndices(); + List failureIndices = getDataStreamResponse.getDataStreams().get(0).getDataStream().getFailureIndices(); assertThat(failureIndices.size(), equalTo(2)); }); @@ -1129,7 +1129,7 @@ public void testLifecycleAppliedToFailureStore() throws Exception { assertThat(getDataStreamResponse.getDataStreams().get(0).getDataStream().getName(), equalTo(dataStreamName)); List backingIndices = getDataStreamResponse.getDataStreams().get(0).getDataStream().getIndices(); assertThat(backingIndices.size(), equalTo(1)); - List failureIndices = getDataStreamResponse.getDataStreams().get(0).getDataStream().getFailureIndices().getIndices(); + List failureIndices = getDataStreamResponse.getDataStreams().get(0).getDataStream().getFailureIndices(); assertThat(failureIndices.size(), equalTo(1)); assertThat(failureIndices.get(0).getName(), equalTo(secondGenerationIndex)); }); @@ -1156,14 +1156,7 @@ private static List getFailureIndices(String dataStreamName) { .actionGet(); assertThat(getDataStreamResponse.getDataStreams().size(), equalTo(1)); assertThat(getDataStreamResponse.getDataStreams().get(0).getDataStream().getName(), equalTo(dataStreamName)); - return getDataStreamResponse.getDataStreams() - .get(0) - .getDataStream() - .getFailureIndices() - .getIndices() - .stream() - .map(Index::getName) - .toList(); + return getDataStreamResponse.getDataStreams().get(0).getDataStream().getFailureIndices().stream().map(Index::getName).toList(); } static void indexDocs(String dataStream, int numDocs) { diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DataStreamsStatsTransportAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DataStreamsStatsTransportAction.java index cc5e00d8283ad..c6bee86e20fcd 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DataStreamsStatsTransportAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DataStreamsStatsTransportAction.java @@ -171,11 +171,11 @@ protected DataStreamsStatsAction.DataStreamShardStats readShardResult(StreamInpu if (indexAbstraction.getType() == IndexAbstraction.Type.DATA_STREAM) { DataStream dataStream = (DataStream) indexAbstraction; AggregatedStats stats = aggregatedDataStreamsStats.computeIfAbsent(dataStream.getName(), s -> new AggregatedStats()); - dataStream.getBackingIndices().getIndices().stream().map(Index::getName).forEach(index -> { + dataStream.getIndices().stream().map(Index::getName).forEach(index -> { stats.backingIndices.add(index); allBackingIndices.add(index); }); - dataStream.getFailureIndices().getIndices().stream().map(Index::getName).forEach(index -> { + dataStream.getFailureIndices().stream().map(Index::getName).forEach(index -> { stats.backingIndices.add(index); allBackingIndices.add(index); }); diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DeleteDataStreamTransportAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DeleteDataStreamTransportAction.java index c770b84ee98a3..769e4db419606 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DeleteDataStreamTransportAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/DeleteDataStreamTransportAction.java @@ -156,7 +156,7 @@ static ClusterState removeDataStream( DataStream dataStream = currentState.metadata().dataStreams().get(dataStreamName); assert dataStream != null; backingIndicesToRemove.addAll(dataStream.getIndices()); - backingIndicesToRemove.addAll(dataStream.getFailureIndices().getIndices()); + backingIndicesToRemove.addAll(dataStream.getFailureIndices()); } // first delete the data streams and then the indices: diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java index 2d310fef0be7e..25d679f91b83e 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/action/TransportGetDataStreamsAction.java @@ -207,8 +207,8 @@ static GetDataStreamAction.Response innerOperation( Map backingIndicesSettingsValues = new HashMap<>(); Metadata metadata = state.getMetadata(); collectIndexSettingsValues(dataStream, backingIndicesSettingsValues, metadata, dataStream.getIndices()); - if (DataStream.isFailureStoreFeatureFlagEnabled() && dataStream.getFailureIndices().getIndices().isEmpty() == false) { - collectIndexSettingsValues(dataStream, backingIndicesSettingsValues, metadata, dataStream.getFailureIndices().getIndices()); + if (DataStream.isFailureStoreFeatureFlagEnabled() && dataStream.getFailureIndices().isEmpty() == false) { + collectIndexSettingsValues(dataStream, backingIndicesSettingsValues, metadata, dataStream.getFailureIndices()); } GetDataStreamAction.Response.TimeSeries timeSeries = null; diff --git a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java index 7de3f180753f8..e993460795e16 100644 --- a/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java +++ b/modules/data-streams/src/main/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleService.java @@ -762,10 +762,8 @@ static List getTargetIndices( targetIndices.add(index); } } - if (withFailureStore - && DataStream.isFailureStoreFeatureFlagEnabled() - && dataStream.getFailureIndices().getIndices().isEmpty() == false) { - for (Index index : dataStream.getFailureIndices().getIndices()) { + if (withFailureStore && DataStream.isFailureStoreFeatureFlagEnabled() && dataStream.getFailureIndices().isEmpty() == false) { + for (Index index : dataStream.getFailureIndices()) { if (dataStream.isIndexManagedByDataStreamLifecycle(index, indexMetadataSupplier) && indicesToExcludeForRemainingRun.contains(index) == false) { targetIndices.add(index); @@ -820,7 +818,7 @@ private Set maybeExecuteRollover(ClusterState state, DataStream dataStrea @Nullable private Index maybeExecuteRollover(ClusterState state, DataStream dataStream, boolean rolloverFailureStore) { - Index currentRunWriteIndex = rolloverFailureStore ? dataStream.getFailureStoreWriteIndex() : dataStream.getWriteIndex(); + Index currentRunWriteIndex = rolloverFailureStore ? dataStream.getWriteFailureIndex() : dataStream.getWriteIndex(); if (currentRunWriteIndex == null) { return null; } diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/UpdateTimeSeriesRangeServiceTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/UpdateTimeSeriesRangeServiceTests.java index f05fe518fa56d..8378526e6bdae 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/UpdateTimeSeriesRangeServiceTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/UpdateTimeSeriesRangeServiceTests.java @@ -141,7 +141,7 @@ public void testUpdateTimeSeriesTemporalRange_NoUpdateBecauseReplicated() { ).getMetadata(); DataStream d = metadata.dataStreams().get(dataStreamName); metadata = Metadata.builder(metadata) - .put(d.copy().setReplicated(true).setBackingIndices(d.getBackingIndices().copy().setRolloverOnWrite(false).build()).build()) + .put(d.copy().setReplicated(true).setBackingIndices(d.getDataComponent().copy().setRolloverOnWrite(false).build()).build()) .build(); now = now.plus(1, ChronoUnit.HOURS); diff --git a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java index 0bb990e544892..55afb228da2c0 100644 --- a/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java +++ b/modules/data-streams/src/test/java/org/elasticsearch/datastreams/lifecycle/DataStreamLifecycleServiceTests.java @@ -230,7 +230,7 @@ public void testOperationsExecutedOnce() { .toList(); assertThat(deleteRequests.get(0).indices()[0], is(dataStream.getIndices().get(0).getName())); assertThat(deleteRequests.get(1).indices()[0], is(dataStream.getIndices().get(1).getName())); - assertThat(deleteRequests.get(2).indices()[0], is(dataStream.getFailureIndices().getIndices().get(0).getName())); + assertThat(deleteRequests.get(2).indices()[0], is(dataStream.getFailureIndices().get(0).getName())); // on the second run the rollover and delete requests should not execute anymore // i.e. the count should *remain* 1 for rollover and 2 for deletes @@ -1495,7 +1495,7 @@ public void testTargetIndices() { ).copy().setDataStreamOptions(dataStreamOptions).build(); // failure store is managed even when disabled builder.put(dataStream); Metadata metadata = builder.build(); - Set indicesToExclude = Set.of(dataStream.getIndices().get(0), dataStream.getFailureIndices().getIndices().get(0)); + Set indicesToExclude = Set.of(dataStream.getIndices().get(0), dataStream.getFailureIndices().get(0)); List targetBackingIndicesOnly = DataStreamLifecycleService.getTargetIndices( dataStream, indicesToExclude, @@ -1506,9 +1506,7 @@ public void testTargetIndices() { List targetIndices = DataStreamLifecycleService.getTargetIndices(dataStream, indicesToExclude, metadata::index, true); assertThat( targetIndices, - equalTo( - List.of(dataStream.getIndices().get(1), dataStream.getIndices().get(2), dataStream.getFailureIndices().getIndices().get(1)) - ) + equalTo(List.of(dataStream.getIndices().get(1), dataStream.getIndices().get(2), dataStream.getFailureIndices().get(1))) ); } @@ -1540,10 +1538,7 @@ public void testFailureStoreIsManagedEvenWhenDisabled() { rolloverFailureIndexRequest.getRolloverTarget(), is(IndexNameExpressionResolver.combineSelector(dataStreamName, IndexComponentSelector.FAILURES)) ); - assertThat( - ((DeleteIndexRequest) clientSeenRequests.get(2)).indices()[0], - is(dataStream.getFailureIndices().getIndices().get(0).getName()) - ); + assertThat(((DeleteIndexRequest) clientSeenRequests.get(2)).indices()[0], is(dataStream.getFailureIndices().get(0).getName())); } public void testMaybeExecuteRetentionSuccessfulDownsampledIndex() { diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java index 9bc088f944be0..d8e0d4a623ad4 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/create/AutoCreateAction.java @@ -268,9 +268,9 @@ ClusterState execute( final var dataStream = clusterState.metadata().dataStreams().get(request.index()); final var backingIndexName = dataStream.getIndices().get(0).getName(); - final var indexNames = dataStream.getFailureIndices().getIndices().isEmpty() + final var indexNames = dataStream.getFailureIndices().isEmpty() ? List.of(backingIndexName) - : List.of(backingIndexName, dataStream.getFailureIndices().getIndices().get(0).getName()); + : List.of(backingIndexName, dataStream.getFailureIndices().get(0).getName()); taskContext.success(getAckListener(indexNames, allocationActionMultiListener)); successfulRequests.put(request, indexNames); return clusterState; diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexAction.java index 4aa022aff1c80..0f1b77af0242e 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/resolve/ResolveIndexAction.java @@ -634,7 +634,7 @@ private static void enrichIndexAbstraction( ); } case ALIAS -> { - String[] indexNames = getAliasIndexStream(resolvedExpression, ia, indicesLookup).map(Index::getName) + String[] indexNames = getAliasIndexStream(resolvedExpression, ia, clusterState.metadata()).map(Index::getName) .toArray(String[]::new); Arrays.sort(indexNames); aliases.add(new ResolvedAlias(ia.getName(), indexNames)); @@ -644,11 +644,11 @@ private static void enrichIndexAbstraction( Stream dataStreamIndices = resolvedExpression.selector() == null ? dataStream.getIndices().stream() : switch (resolvedExpression.selector()) { - case DATA -> dataStream.getBackingIndices().getIndices().stream(); - case FAILURES -> dataStream.getFailureIndices().getIndices().stream(); + case DATA -> dataStream.getDataComponent().getIndices().stream(); + case FAILURES -> dataStream.getFailureIndices().stream(); case ALL_APPLICABLE -> Stream.concat( - dataStream.getBackingIndices().getIndices().stream(), - dataStream.getFailureIndices().getIndices().stream() + dataStream.getIndices().stream(), + dataStream.getFailureIndices().stream() ); }; String[] backingIndices = dataStreamIndices.map(Index::getName).toArray(String[]::new); @@ -659,11 +659,7 @@ private static void enrichIndexAbstraction( } } - private static Stream getAliasIndexStream( - ResolvedExpression resolvedExpression, - IndexAbstraction ia, - SortedMap indicesLookup - ) { + private static Stream getAliasIndexStream(ResolvedExpression resolvedExpression, IndexAbstraction ia, Metadata metadata) { Stream aliasIndices; if (resolvedExpression.selector() == null) { aliasIndices = ia.getIndices().stream(); @@ -672,30 +668,11 @@ private static Stream getAliasIndexStream( case DATA -> ia.getIndices().stream(); case FAILURES -> { assert ia.isDataStreamRelated() : "Illegal selector [failures] used on non data stream alias"; - yield ia.getIndices() - .stream() - .map(Index::getName) - .map(indicesLookup::get) - .map(IndexAbstraction::getParentDataStream) - .filter(Objects::nonNull) - .distinct() - .map(DataStream::getFailureIndices) - .flatMap(failureIndices -> failureIndices.getIndices().stream()); + yield ia.getFailureIndices(metadata).stream(); } case ALL_APPLICABLE -> { if (ia.isDataStreamRelated()) { - yield Stream.concat( - ia.getIndices().stream(), - ia.getIndices() - .stream() - .map(Index::getName) - .map(indicesLookup::get) - .map(IndexAbstraction::getParentDataStream) - .filter(Objects::nonNull) - .distinct() - .map(DataStream::getFailureIndices) - .flatMap(failureIndices -> failureIndices.getIndices().stream()) - ); + yield Stream.concat(ia.getIndices().stream(), ia.getFailureIndices(metadata).stream()); } else { yield ia.getIndices().stream(); } diff --git a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java index 4f0aa9c5bade4..ac47144c1f558 100644 --- a/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java +++ b/server/src/main/java/org/elasticsearch/action/admin/indices/rollover/TransportRolloverAction.java @@ -246,7 +246,7 @@ protected void masterOperation( } // When we're initializing a failure store, we skip the stats request because there is no source index to retrieve stats for. - if (targetFailureStore && ((DataStream) rolloverTargetAbstraction).getFailureIndices().getIndices().isEmpty()) { + if (targetFailureStore && ((DataStream) rolloverTargetAbstraction).getFailureIndices().isEmpty()) { initializeFailureStore(rolloverRequest, listener, trialSourceIndexName, trialRolloverIndexName); return; } diff --git a/server/src/main/java/org/elasticsearch/action/bulk/BulkOperation.java b/server/src/main/java/org/elasticsearch/action/bulk/BulkOperation.java index dd473869fb2d9..fcf303ff2c82e 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/BulkOperation.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/BulkOperation.java @@ -654,7 +654,7 @@ private boolean addDocumentToRedirectRequests(BulkItemRequest request, Exception * If so, we'll need to roll it over before we index the failed documents into the failure store. */ private void maybeMarkFailureStoreForRollover(DataStream dataStream) { - if (dataStream.getFailureIndices().isRolloverOnWrite() == false) { + if (dataStream.getFailureComponent().isRolloverOnWrite() == false) { return; } failureStoresToBeRolledOver.add(dataStream.getName()); diff --git a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java index 523381321ada7..b3e76e52fda26 100644 --- a/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java +++ b/server/src/main/java/org/elasticsearch/action/bulk/TransportBulkAction.java @@ -321,9 +321,9 @@ private void populateMissingTargets( // Determine which data streams and failure stores need to be rolled over. DataStream dataStream = state.metadata().dataStreams().get(request.index()); if (dataStream != null) { - if (writeToFailureStore == false && dataStream.getBackingIndices().isRolloverOnWrite()) { + if (writeToFailureStore == false && dataStream.getDataComponent().isRolloverOnWrite()) { dataStreamsToBeRolledOver.add(request.index()); - } else if (lazyRolloverFailureStoreFeature && writeToFailureStore && dataStream.getFailureIndices().isRolloverOnWrite()) { + } else if (lazyRolloverFailureStoreFeature && writeToFailureStore && dataStream.getFailureComponent().isRolloverOnWrite()) { failureStoresToBeRolledOver.add(request.index()); } } diff --git a/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java b/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java index c55957787aee7..5dd60a1122bf7 100644 --- a/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java +++ b/server/src/main/java/org/elasticsearch/action/datastreams/GetDataStreamAction.java @@ -418,10 +418,10 @@ public XContentBuilder toXContent( builder.field(FAILURE_STORE_ENABLED.getPreferredName(), failureStoreEffectivelyEnabled); builder.field( DataStream.ROLLOVER_ON_WRITE_FIELD.getPreferredName(), - dataStream.getFailureIndices().isRolloverOnWrite() + dataStream.getFailureComponent().isRolloverOnWrite() ); - indicesToXContent(builder, dataStream.getFailureIndices().getIndices()); - addAutoShardingEvent(builder, params, dataStream.getFailureIndices().getAutoShardingEvent()); + indicesToXContent(builder, dataStream.getFailureIndices()); + addAutoShardingEvent(builder, params, dataStream.getFailureComponent().getAutoShardingEvent()); builder.endObject(); } builder.endObject(); diff --git a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java index 4343451256920..bcb8a7fb78bf3 100644 --- a/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java +++ b/server/src/main/java/org/elasticsearch/action/index/IndexRequest.java @@ -885,12 +885,12 @@ public Index getConcreteWriteIndex(IndexAbstraction ia, Metadata metadata) { // Resolve write index and get parent data stream to handle the case of dealing with an alias String defaultWriteIndexName = ia.getWriteIndex().getName(); DataStream dataStream = metadata.getIndicesLookup().get(defaultWriteIndexName).getParentDataStream(); - if (dataStream.getFailureIndices().getIndices().size() < 1) { + if (dataStream.getFailureIndices().size() < 1) { throw new ElasticsearchException( "Attempting to write a document to a failure store but the target data stream does not have one enabled" ); } - return dataStream.getFailureIndices().getIndices().get(dataStream.getFailureIndices().getIndices().size() - 1); + return dataStream.getFailureIndices().get(dataStream.getFailureIndices().size() - 1); } else { // Resolve as normal return ia.getWriteIndex(this, metadata); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java index 286ecb3113337..1313060936f63 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/DataStream.java @@ -259,6 +259,15 @@ public List getIndices() { return backingIndices.indices; } + @Override + public List getFailureIndices(Metadata ignored) { + return failureIndices.indices; + } + + public List getFailureIndices() { + return failureIndices.indices; + } + public long getGeneration() { return generation; } @@ -268,11 +277,20 @@ public Index getWriteIndex() { return backingIndices.getWriteIndex(); } + /** + * @param metadata is not necessary for data streams + * @return the write failure index if the failure store is enabled and there is already at least one failure, null otherwise + */ + @Override + public Index getWriteFailureIndex(Metadata metadata) { + return getWriteFailureIndex(); + } + /** * @return the write failure index if the failure store is enabled and there is already at least one failure, null otherwise */ @Nullable - public Index getFailureStoreWriteIndex() { + public Index getWriteFailureIndex() { return failureIndices.indices.isEmpty() ? null : failureIndices.getWriteIndex(); } @@ -495,11 +513,11 @@ public DataStreamAutoShardingEvent getAutoShardingEvent() { return backingIndices.autoShardingEvent; } - public DataStreamIndices getBackingIndices() { + public DataStreamIndices getDataComponent() { return backingIndices; } - public DataStreamIndices getFailureIndices() { + public DataStreamIndices getFailureComponent() { return failureIndices; } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstraction.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstraction.java index 0585263c45e99..8429876f9f937 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstraction.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexAbstraction.java @@ -38,6 +38,16 @@ public interface IndexAbstraction { */ List getIndices(); + /** + * It retrieves the failure indices of an index abstraction given it supports the failure store. + * @param metadata certain abstractions require the matadata to lazily retrieve the failure indices. + * @return All concrete failure indices this index abstraction is referring to. If the failure store is + * not supported, it returns an empty list. + */ + default List getFailureIndices(@Nullable Metadata metadata) { + return List.of(); + } + /** * A write index is a dedicated concrete index, that accepts all the new documents that belong to an index abstraction. *

@@ -50,6 +60,18 @@ public interface IndexAbstraction { @Nullable Index getWriteIndex(); + /** + * A write failure index is a dedicated concrete index, that accepts all the new documents that belong to the failure store of + * an index abstraction. Only an index abstraction with true {@link #isDataStreamRelated()} supports a failure store. + * @param metadata certain index abstraction require the metadata to lazily retrieve the failure indices + * @return the write failure index of this index abstraction or null if this index abstraction doesn't have + * a write failure index or it does not support the failure store. + */ + @Nullable + default Index getWriteFailureIndex(Metadata metadata) { + return null; + } + default Index getWriteIndex(IndexRequest request, Metadata metadata) { return getWriteIndex(); } @@ -199,9 +221,10 @@ class Alias implements IndexAbstraction { private final boolean isHidden; private final boolean isSystem; private final boolean dataStreamAlias; + private final List dataStreams; public Alias(AliasMetadata aliasMetadata, List indexMetadatas) { - // note: don't capture a reference to any of these indexMetadatas here + // note: don't capture a reference to any of these indexMetadata here this.aliasName = aliasMetadata.getAlias(); this.referenceIndices = new ArrayList<>(indexMetadatas.size()); boolean isSystem = true; @@ -226,15 +249,22 @@ public Alias(AliasMetadata aliasMetadata, List indexMetadatas) { this.isHidden = aliasMetadata.isHidden() == null ? false : aliasMetadata.isHidden(); this.isSystem = isSystem; dataStreamAlias = false; + dataStreams = List.of(); } - public Alias(DataStreamAlias dataStreamAlias, List indicesOfAllDataStreams, Index writeIndexOfWriteDataStream) { + public Alias( + DataStreamAlias dataStreamAlias, + List indicesOfAllDataStreams, + Index writeIndexOfWriteDataStream, + List dataStreams + ) { this.aliasName = dataStreamAlias.getName(); this.referenceIndices = indicesOfAllDataStreams; this.writeIndex = writeIndexOfWriteDataStream; this.isHidden = false; this.isSystem = false; this.dataStreamAlias = true; + this.dataStreams = dataStreams; } @Override @@ -251,11 +281,38 @@ public List getIndices() { return referenceIndices; } + @Override + public List getFailureIndices(Metadata metadata) { + if (isDataStreamRelated() == false) { + return List.of(); + } + assert metadata != null : "metadata must not be null to be able to retrieve the failure indices"; + List failureIndices = new ArrayList<>(); + for (String dataStreamName : dataStreams) { + DataStream dataStream = metadata.dataStreams().get(dataStreamName); + if (dataStream != null && dataStream.getFailureIndices().isEmpty() == false) { + failureIndices.addAll(dataStream.getFailureIndices()); + } + } + return failureIndices; + } + @Nullable public Index getWriteIndex() { return writeIndex; } + @Nullable + @Override + public Index getWriteFailureIndex(Metadata metadata) { + if (isDataStreamRelated() == false || writeIndex == null) { + return null; + } + assert metadata != null : "metadata must not be null to be able to retrieve the failure indices"; + DataStream dataStream = metadata.getIndicesLookup().get(writeIndex.getName()).getParentDataStream(); + return dataStream == null ? null : dataStream.getWriteFailureIndex(); + } + @Override public Index getWriteIndex(IndexRequest request, Metadata metadata) { if (dataStreamAlias == false) { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java index 8b88f7609fa8c..9ad00b517d51c 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexNameExpressionResolver.java @@ -491,27 +491,28 @@ Index[] concreteIndices(Context context, String... indexExpressions) { for (ResolvedExpression expression : expressions) { final IndexAbstraction indexAbstraction = indicesLookup.get(expression.resource()); assert indexAbstraction != null; - if (indexAbstraction.getType() == Type.ALIAS && context.isResolveToWriteIndex()) { - Index writeIndex = indexAbstraction.getWriteIndex(); - if (writeIndex == null) { - throw new IllegalArgumentException( - "no write index is defined for alias [" - + indexAbstraction.getName() - + "]." - + " The write index may be explicitly disabled using is_write_index=false or the alias points to multiple" - + " indices without one being designated as a write index" - ); - } - if (indexAbstraction.isDataStreamRelated()) { - DataStream dataStream = indicesLookup.get(indexAbstraction.getWriteIndex().getName()).getParentDataStream(); - resolveWriteIndexForDataStreams(context, dataStream, concreteIndicesResult, expression.selector()); - } else { + if (context.isResolveToWriteIndex()) { + if (shouldIncludeRegularIndices(context.getOptions(), expression.selector())) { + Index writeIndex = indexAbstraction.getWriteIndex(); + if (writeIndex == null && indexAbstraction.getType() == Type.ALIAS) { + throw new IllegalArgumentException( + "no write index is defined for alias [" + + indexAbstraction.getName() + + "]." + + " The write index may be explicitly disabled using is_write_index=false or the alias points to multiple" + + " indices without one being designated as a write index" + ); + } if (addIndex(writeIndex, null, context)) { concreteIndicesResult.add(writeIndex); } } - } else if (indexAbstraction.getType() == Type.DATA_STREAM && context.isResolveToWriteIndex()) { - resolveWriteIndexForDataStreams(context, (DataStream) indexAbstraction, concreteIndicesResult, expression.selector()); + if (shouldIncludeFailureIndices(context.getOptions(), expression.selector())) { + Index failureStoreWriteIndex = indexAbstraction.getWriteFailureIndex(context.state.metadata()); + if (failureStoreWriteIndex != null && addIndex(failureStoreWriteIndex, null, context)) { + concreteIndicesResult.add(failureStoreWriteIndex); + } + } } else { if (context.getOptions().allowAliasesToMultipleIndices() == false && resolvesToMoreThanOneIndex(indexAbstraction, context, expression)) { @@ -530,23 +531,17 @@ && resolvesToMoreThanOneIndex(indexAbstraction, context, expression)) { ); } - if (indexAbstraction.getType() == Type.DATA_STREAM) { - resolveIndicesForDataStream(context, (DataStream) indexAbstraction, concreteIndicesResult, expression.selector()); - } else if (indexAbstraction.getType() == Type.ALIAS - && indexAbstraction.isDataStreamRelated() - && shouldIncludeFailureIndices(context.getOptions(), expression.selector())) { - for (DataStream dataStream : getAliasDataStreams(indexAbstraction, indicesLookup)) { - resolveIndicesForDataStream(context, dataStream, concreteIndicesResult, expression.selector()); - } - } else { - List indices = indexAbstraction.getIndices(); - for (int i = 0, n = indices.size(); i < n; i++) { - Index index = indices.get(i); - if (shouldTrackConcreteIndex(context, index)) { - concreteIndicesResult.add(index); - } + if (indexAbstraction.isDataStreamRelated()) { + resolveIndicesForDataStreamRelatedAbstraction(context, indexAbstraction, concreteIndicesResult, expression.selector()); + } else { + List indices = indexAbstraction.getIndices(); + for (int i = 0, n = indices.size(); i < n; i++) { + Index index = indices.get(i); + if (shouldTrackConcreteIndex(context, index)) { + concreteIndicesResult.add(index); } } + } } } @@ -558,30 +553,14 @@ && shouldIncludeFailureIndices(context.getOptions(), expression.selector())) { return resultArray; } - private static Set getAliasDataStreams(IndexAbstraction indexAbstraction, Map indicesLookup) { - // Collect the data streams involved with the alias - assert indexAbstraction.getType().equals(Type.ALIAS) && indexAbstraction.isDataStreamRelated() - : "Non data stream alias [" + indexAbstraction.getName() + "]"; - Set aliasDataStreams = new HashSet<>(); - List indices = indexAbstraction.getIndices(); - for (int i = 0, n = indices.size(); i < n; i++) { - Index index = indices.get(i); - DataStream parentDataStream = indicesLookup.get(index.getName()).getParentDataStream(); - if (parentDataStream != null) { - aliasDataStreams.add(parentDataStream); - } - } - return aliasDataStreams; - } - - private static void resolveIndicesForDataStream( + private static void resolveIndicesForDataStreamRelatedAbstraction( Context context, - DataStream dataStream, + IndexAbstraction indexAbstraction, Set concreteIndicesResult, IndexComponentSelector selector ) { if (shouldIncludeRegularIndices(context.getOptions(), selector)) { - List indices = dataStream.getIndices(); + List indices = indexAbstraction.getIndices(); for (int i = 0, n = indices.size(); i < n; i++) { Index index = indices.get(i); if (shouldTrackConcreteIndex(context, index)) { @@ -590,7 +569,7 @@ private static void resolveIndicesForDataStream( } } if (shouldIncludeFailureIndices(context.getOptions(), selector)) { - List failureIndices = dataStream.getFailureIndices().getIndices(); + List failureIndices = indexAbstraction.getFailureIndices(context.state.metadata()); for (int i = 0, n = failureIndices.size(); i < n; i++) { Index index = failureIndices.get(i); if (shouldTrackConcreteIndex(context, index)) { @@ -613,7 +592,7 @@ private static void resolveWriteIndexForDataStreams( } } if (shouldIncludeFailureIndices(context.getOptions(), selector)) { - Index failureStoreWriteIndex = dataStream.getFailureStoreWriteIndex(); + Index failureStoreWriteIndex = dataStream.getWriteFailureIndex(); if (failureStoreWriteIndex != null && addIndex(failureStoreWriteIndex, null, context)) { concreteIndicesResult.add(failureStoreWriteIndex); } @@ -652,7 +631,7 @@ private static boolean resolvesToMoreThanOneIndex(IndexAbstraction indexAbstract count += parentDataStream.getIndices().size(); } if (shouldIncludeFailureIndices(context.getOptions(), expression.selector())) { - count += parentDataStream.getFailureIndices().getIndices().size(); + count += parentDataStream.getFailureIndices().size(); } if (count > 1) { // Early out if we already have more than one index accounted @@ -669,7 +648,7 @@ private static boolean resolvesToMoreThanOneIndex(IndexAbstraction indexAbstract count += dataStream.getIndices().size(); } if (shouldIncludeFailureIndices(context.getOptions(), expression.selector())) { - count += dataStream.getFailureIndices().getIndices().size(); + count += dataStream.getFailureIndices().size(); } return count > 1; } @@ -973,7 +952,7 @@ public String[] indexAliases( IndexAbstraction ia = state.metadata().getIndicesLookup().get(index); DataStream dataStream = ia.getParentDataStream(); if (dataStream != null) { - if (dataStream.getFailureIndices().containsIndex(index)) { + if (dataStream.getFailureComponent().containsIndex(index)) { // Alias filters are not applied against indices in an abstraction's failure component. // They do not match the mapping of the data stream nor are the documents mapped for searching. return null; @@ -1100,10 +1079,10 @@ public Map> resolveSearchRouting(ClusterState state, @Nullab aliasIndices.addAll(indexAbstraction.getIndices()); } if (shouldIncludeFailureIndices(context.getOptions(), selector) && indexAbstraction.isDataStreamRelated()) { - Set dataStreams = getAliasDataStreams(indexAbstraction, context.state.metadata().getIndicesLookup()); - aliasIndices = aliasIndices == null ? new ArrayList<>(dataStreams.size()) : aliasIndices; - for (DataStream dataStream : dataStreams) { - aliasIndices.addAll(dataStream.getFailureIndices().getIndices()); + List failureIndices = indexAbstraction.getFailureIndices(context.state.metadata()); + if (failureIndices.isEmpty() == false) { + aliasIndices = aliasIndices == null ? new ArrayList<>(failureIndices.size()) : aliasIndices; + aliasIndices.addAll(failureIndices); } } aliasIndices = aliasIndices == null ? List.of() : aliasIndices; @@ -1150,8 +1129,8 @@ public Map> resolveSearchRouting(ClusterState state, @Nullab } } if (shouldIncludeFailureIndices(context.getOptions(), resolvedExpression.selector())) { - if (dataStream.getFailureIndices().getIndices() != null) { - for (Index failureIndex : dataStream.getFailureIndices().getIndices()) { + if (dataStream.getFailureIndices().isEmpty() == false) { + for (Index failureIndex : dataStream.getFailureIndices()) { String concreteIndex = failureIndex.getName(); routings = collectRoutings(routings, paramRouting, norouting, concreteIndex); } @@ -1730,22 +1709,10 @@ private static Set expandToOpenClosed( } } if (shouldIncludeFailureIndices(context.getOptions(), selector)) { - if (indexAbstraction.getType() == Type.ALIAS && indexAbstraction.isDataStreamRelated()) { - Set aliasDataStreams = getAliasDataStreams( - indexAbstraction, - context.state.metadata().getIndicesLookup() - ); - for (DataStream ds : aliasDataStreams) { - List failureIndices = ds.getFailureIndices().getIndices(); - for (int i = 0; i < failureIndices.size(); i++) { - Index index = failureIndices.get(i); - resources.add(new ResolvedExpression(index.getName(), IndexComponentSelector.DATA)); - } - } - } else if (indexAbstraction.getType() == Type.DATA_STREAM) { - DataStream dataStream = (DataStream) indexAbstraction; - for (int i = 0, n = dataStream.getFailureIndices().getIndices().size(); i < n; i++) { - Index index = dataStream.getFailureIndices().getIndices().get(i); + if (indexAbstraction.isDataStreamRelated()) { + List failureIndices = indexAbstraction.getFailureIndices(context.state.metadata()); + for (int i = 0, n = failureIndices.size(); i < n; i++) { + Index index = failureIndices.get(i); IndexMetadata indexMetadata = context.state.metadata().index(index); if (indexMetadata.getState() != excludeState) { resources.add(new ResolvedExpression(index.getName(), IndexComponentSelector.DATA)); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java index 6a4f24818003e..35e853cdd55a9 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java @@ -2492,7 +2492,7 @@ private static boolean assertContainsIndexIfDataStream(DataStream parent, IndexM assert parent == null || parent.getIndices().stream().anyMatch(index -> indexMetadata.getIndex().getName().equals(index.getName())) || (DataStream.isFailureStoreFeatureFlagEnabled() - && parent.getFailureIndices() + && parent.getFailureComponent() .getIndices() .stream() .anyMatch(index -> indexMetadata.getIndex().getName().equals(index.getName()))) @@ -2518,7 +2518,7 @@ private static void collectDataStreams( indexToDataStreamLookup.put(i.getName(), dataStream); } if (DataStream.isFailureStoreFeatureFlagEnabled()) { - for (Index i : dataStream.getFailureIndices().getIndices()) { + for (Index i : dataStream.getFailureIndices()) { indexToDataStreamLookup.put(i.getName(), dataStream); } } @@ -2534,7 +2534,8 @@ private static IndexAbstraction.Alias makeDsAliasAbstraction(Map dataStreams.get(name).getIndices().stream()).toList(), - writeIndexOfWriteDataStream + writeIndexOfWriteDataStream, + alias.getDataStreams() ); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java index 254646f8e71a9..d4be8b2ddf5b6 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamService.java @@ -111,8 +111,8 @@ public ClusterState execute(ClusterState currentState) throws Exception { ClusterState clusterState = createDataStream(request, currentState, delegate.reroute(), false); DataStream createdDataStream = clusterState.metadata().dataStreams().get(request.name); firstBackingIndexRef.set(createdDataStream.getIndices().get(0).getName()); - if (createdDataStream.getFailureIndices().getIndices().isEmpty() == false) { - firstFailureStoreRef.set(createdDataStream.getFailureIndices().getIndices().get(0).getName()); + if (createdDataStream.getFailureIndices().isEmpty() == false) { + firstFailureStoreRef.set(createdDataStream.getFailureIndices().get(0).getName()); } return clusterState; } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java index db3973c1a15a8..71be34db9626f 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsService.java @@ -366,7 +366,7 @@ private static void removeBackingIndex( ) { boolean indexNotRemoved = true; DataStream dataStream = validateDataStream(metadata, dataStreamName); - List targetIndices = failureStore ? dataStream.getFailureIndices().getIndices() : dataStream.getIndices(); + List targetIndices = failureStore ? dataStream.getFailureIndices() : dataStream.getIndices(); for (Index backingIndex : targetIndices) { if (backingIndex.getName().equals(indexName)) { if (failureStore) { diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDeleteIndexService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDeleteIndexService.java index 5d1a037d6bc3e..df9fc71dd1b76 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDeleteIndexService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataDeleteIndexService.java @@ -152,7 +152,7 @@ public static ClusterState deleteIndices(ClusterState currentState, Set i IndexMetadata im = meta.getIndexSafe(index); DataStream parent = meta.getIndicesLookup().get(im.getIndex().getName()).getParentDataStream(); if (parent != null) { - boolean isFailureStoreWriteIndex = im.getIndex().equals(parent.getFailureStoreWriteIndex()); + boolean isFailureStoreWriteIndex = im.getIndex().equals(parent.getWriteFailureIndex()); if (isFailureStoreWriteIndex || im.getIndex().equals(parent.getWriteIndex())) { throw new IllegalArgumentException( "index [" diff --git a/server/src/main/java/org/elasticsearch/ingest/IngestService.java b/server/src/main/java/org/elasticsearch/ingest/IngestService.java index 1494d2a46f9d0..86522742a66c0 100644 --- a/server/src/main/java/org/elasticsearch/ingest/IngestService.java +++ b/server/src/main/java/org/elasticsearch/ingest/IngestService.java @@ -293,7 +293,7 @@ static boolean isRolloverOnWrite(Metadata metadata, IndexRequest indexRequest) { if (dataStream == null) { return false; } - return dataStream.getBackingIndices().isRolloverOnWrite(); + return dataStream.getDataComponent().isRolloverOnWrite(); } /** diff --git a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java index debe3d6e6bd92..0bdf39142f766 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java @@ -426,7 +426,7 @@ private void startRestore( if (DataStream.isFailureStoreFeatureFlagEnabled()) { failureIndices = dataStreamsToRestore.values() .stream() - .flatMap(ds -> ds.getFailureIndices().getIndices().stream().map(idx -> new Tuple<>(ds.isSystem(), idx.getName()))) + .flatMap(ds -> ds.getFailureIndices().stream().map(idx -> new Tuple<>(ds.isSystem(), idx.getName()))) .collect(Collectors.partitioningBy(Tuple::v1, Collectors.mapping(Tuple::v2, Collectors.toSet()))); } systemDataStreamIndices = Sets.union(backingIndices.getOrDefault(true, Set.of()), failureIndices.getOrDefault(true, Set.of())); @@ -773,7 +773,7 @@ static DataStream updateDataStream(DataStream dataStream, Metadata.Builder metad .map(i -> metadata.get(renameIndex(i.getName(), request, true, false)).getIndex()) .toList(); List updatedFailureIndices = DataStream.isFailureStoreFeatureFlagEnabled() - ? dataStream.getFailureIndices() + ? dataStream.getFailureComponent() .getIndices() .stream() .map(i -> metadata.get(renameIndex(i.getName(), request, false, true)).getIndex()) @@ -781,8 +781,8 @@ static DataStream updateDataStream(DataStream dataStream, Metadata.Builder metad : List.of(); return dataStream.copy() .setName(dataStreamName) - .setBackingIndices(dataStream.getBackingIndices().copy().setIndices(updatedIndices).build()) - .setFailureIndices(dataStream.getFailureIndices().copy().setIndices(updatedFailureIndices).build()) + .setBackingIndices(dataStream.getDataComponent().copy().setIndices(updatedIndices).build()) + .setFailureIndices(dataStream.getFailureComponent().copy().setIndices(updatedFailureIndices).build()) .build(); } diff --git a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java index 42984dc9da828..57750bb02bb14 100644 --- a/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java +++ b/server/src/test/java/org/elasticsearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java @@ -673,9 +673,7 @@ public void testRolloverClusterStateForDataStreamFailureStore() throws Exception Metadata.Builder builder = Metadata.builder(); builder.put("template", template); dataStream.getIndices().forEach(index -> builder.put(DataStreamTestHelper.getIndexMetadataBuilderForIndex(index))); - dataStream.getFailureIndices() - .getIndices() - .forEach(index -> builder.put(DataStreamTestHelper.getIndexMetadataBuilderForIndex(index))); + dataStream.getFailureIndices().forEach(index -> builder.put(DataStreamTestHelper.getIndexMetadataBuilderForIndex(index))); builder.put(dataStream); final ClusterState clusterState = ClusterState.builder(new ClusterName("test")).metadata(builder).build(); final TestTelemetryPlugin telemetryPlugin = new TestTelemetryPlugin(); @@ -716,19 +714,16 @@ public void testRolloverClusterStateForDataStreamFailureStore() throws Exception assertEquals(sourceIndexName, rolloverResult.sourceIndexName()); assertEquals(newIndexName, rolloverResult.rolloverIndexName()); Metadata rolloverMetadata = rolloverResult.clusterState().metadata(); - assertEquals( - dataStream.getIndices().size() + dataStream.getFailureIndices().getIndices().size() + 1, - rolloverMetadata.indices().size() - ); + assertEquals(dataStream.getIndices().size() + dataStream.getFailureIndices().size() + 1, rolloverMetadata.indices().size()); IndexMetadata rolloverIndexMetadata = rolloverMetadata.index(newIndexName); var ds = (DataStream) rolloverMetadata.getIndicesLookup().get(dataStream.getName()); assertThat(ds.getType(), equalTo(IndexAbstraction.Type.DATA_STREAM)); assertThat(ds.getIndices(), hasSize(dataStream.getIndices().size())); - assertThat(ds.getFailureIndices().getIndices(), hasSize(dataStream.getFailureIndices().getIndices().size() + 1)); - assertThat(ds.getFailureIndices().getIndices(), hasItem(rolloverMetadata.index(sourceIndexName).getIndex())); - assertThat(ds.getFailureIndices().getIndices(), hasItem(rolloverIndexMetadata.getIndex())); - assertThat(ds.getFailureStoreWriteIndex(), equalTo(rolloverIndexMetadata.getIndex())); + assertThat(ds.getFailureIndices(), hasSize(dataStream.getFailureIndices().size() + 1)); + assertThat(ds.getFailureIndices(), hasItem(rolloverMetadata.index(sourceIndexName).getIndex())); + assertThat(ds.getFailureIndices(), hasItem(rolloverIndexMetadata.getIndex())); + assertThat(ds.getWriteFailureIndex(), equalTo(rolloverIndexMetadata.getIndex())); RolloverInfo info = rolloverMetadata.index(sourceIndexName).getRolloverInfos().get(dataStream.getName()); assertThat(info.getTime(), lessThanOrEqualTo(after)); @@ -753,7 +748,7 @@ public void testValidation() throws Exception { .promoteDataStream(); rolloverTarget = dataStream.getName(); if (dataStream.isFailureStoreExplicitlyEnabled() && randomBoolean()) { - sourceIndexName = dataStream.getFailureStoreWriteIndex().getName(); + sourceIndexName = dataStream.getWriteFailureIndex().getName(); isFailureStoreRollover = true; defaultRolloverIndexName = DataStream.getDefaultFailureStoreName( dataStream.getName(), diff --git a/server/src/test/java/org/elasticsearch/action/bulk/BulkOperationTests.java b/server/src/test/java/org/elasticsearch/action/bulk/BulkOperationTests.java index 9360ce1719634..6155e11a127ec 100644 --- a/server/src/test/java/org/elasticsearch/action/bulk/BulkOperationTests.java +++ b/server/src/test/java/org/elasticsearch/action/bulk/BulkOperationTests.java @@ -852,7 +852,10 @@ public void testLazilyRollingOverFailureStore() throws Exception { DataStream rolledOverDataStream = dataStream3.copy() .setFailureIndices( - dataStream3.getFailureIndices().copy().setIndices(List.of(ds3FailureStore1.getIndex(), ds3FailureStore2.getIndex())).build() + dataStream3.getFailureComponent() + .copy() + .setIndices(List.of(ds3FailureStore1.getIndex(), ds3FailureStore2.getIndex())) + .build() ) .build(); Metadata metadata = Metadata.builder(DEFAULT_STATE.metadata()) @@ -897,7 +900,10 @@ public void testFailureWhileRollingOverFailureStore() throws Exception { DataStream rolledOverDataStream = dataStream3.copy() .setFailureIndices( - dataStream3.getFailureIndices().copy().setIndices(List.of(ds3FailureStore1.getIndex(), ds3FailureStore2.getIndex())).build() + dataStream3.getFailureComponent() + .copy() + .setIndices(List.of(ds3FailureStore1.getIndex(), ds3FailureStore2.getIndex())) + .build() ) .build(); Metadata metadata = Metadata.builder(DEFAULT_STATE.metadata()) diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java index f7f299683c3fc..0b3ad270a9d2b 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/DataStreamTests.java @@ -101,11 +101,11 @@ protected DataStream mutateInstance(DataStream instance) { var indexMode = instance.getIndexMode(); var lifecycle = instance.getLifecycle(); var dataStreamOptions = instance.getDataStreamOptions(); - var failureIndices = instance.getFailureIndices().getIndices(); + var failureIndices = instance.getFailureIndices(); var rolloverOnWrite = instance.rolloverOnWrite(); var autoShardingEvent = instance.getAutoShardingEvent(); - var failureRolloverOnWrite = instance.getFailureIndices().isRolloverOnWrite(); - var failureAutoShardingEvent = instance.getBackingIndices().getAutoShardingEvent(); + var failureRolloverOnWrite = instance.getFailureComponent().isRolloverOnWrite(); + var failureAutoShardingEvent = instance.getDataComponent().getAutoShardingEvent(); switch (between(0, 15)) { case 0 -> name = randomAlphaOfLength(10); case 1 -> indices = randomNonEmptyIndexInstances(); @@ -197,7 +197,7 @@ protected DataStream mutateInstance(DataStream instance) { public void testRollover() { DataStream ds = DataStreamTestHelper.randomInstance().promoteDataStream(); - Tuple newCoordinates = ds.nextWriteIndexAndGeneration(Metadata.EMPTY_METADATA, ds.getBackingIndices()); + Tuple newCoordinates = ds.nextWriteIndexAndGeneration(Metadata.EMPTY_METADATA, ds.getDataComponent()); final DataStream rolledDs = ds.rollover(new Index(newCoordinates.v1(), UUIDs.randomBase64UUID()), newCoordinates.v2(), null, null); assertThat(rolledDs.getName(), equalTo(ds.getName())); assertThat(rolledDs.getGeneration(), equalTo(ds.getGeneration() + 1)); @@ -224,7 +224,7 @@ public void testRolloverWithConflictingBackingIndexName() { builder.put(im, false); } - final Tuple newCoordinates = ds.nextWriteIndexAndGeneration(builder.build(), ds.getBackingIndices()); + final Tuple newCoordinates = ds.nextWriteIndexAndGeneration(builder.build(), ds.getDataComponent()); final DataStream rolledDs = ds.rollover(new Index(newCoordinates.v1(), UUIDs.randomBase64UUID()), newCoordinates.v2(), null, null); assertThat(rolledDs.getName(), equalTo(ds.getName())); assertThat(rolledDs.getGeneration(), equalTo(ds.getGeneration() + numConflictingIndices + 1)); @@ -240,7 +240,7 @@ public void testRolloverUpgradeToTsdbDataStream() { .setReplicated(false) .setIndexMode(randomBoolean() ? IndexMode.STANDARD : null) .build(); - var newCoordinates = ds.nextWriteIndexAndGeneration(Metadata.EMPTY_METADATA, ds.getBackingIndices()); + var newCoordinates = ds.nextWriteIndexAndGeneration(Metadata.EMPTY_METADATA, ds.getDataComponent()); var rolledDs = ds.rollover( new Index(newCoordinates.v1(), UUIDs.randomBase64UUID()), @@ -262,7 +262,7 @@ public void testRolloverUpgradeToLogsdbDataStream() { .setReplicated(false) .setIndexMode(randomBoolean() ? IndexMode.STANDARD : null) .build(); - var newCoordinates = ds.nextWriteIndexAndGeneration(Metadata.EMPTY_METADATA, ds.getBackingIndices()); + var newCoordinates = ds.nextWriteIndexAndGeneration(Metadata.EMPTY_METADATA, ds.getDataComponent()); var rolledDs = ds.rollover(new Index(newCoordinates.v1(), UUIDs.randomBase64UUID()), newCoordinates.v2(), IndexMode.LOGSDB, null); assertThat(rolledDs.getName(), equalTo(ds.getName())); @@ -275,7 +275,7 @@ public void testRolloverUpgradeToLogsdbDataStream() { public void testRolloverDowngradeFromTsdbToRegularDataStream() { DataStream ds = DataStreamTestHelper.randomInstance().copy().setReplicated(false).setIndexMode(IndexMode.TIME_SERIES).build(); - var newCoordinates = ds.nextWriteIndexAndGeneration(Metadata.EMPTY_METADATA, ds.getBackingIndices()); + var newCoordinates = ds.nextWriteIndexAndGeneration(Metadata.EMPTY_METADATA, ds.getDataComponent()); var rolledDs = ds.rollover(new Index(newCoordinates.v1(), UUIDs.randomBase64UUID()), newCoordinates.v2(), null, null); assertThat(rolledDs.getName(), equalTo(ds.getName())); @@ -288,7 +288,7 @@ public void testRolloverDowngradeFromTsdbToRegularDataStream() { public void testRolloverDowngradeFromLogsdbToRegularDataStream() { DataStream ds = DataStreamTestHelper.randomInstance().copy().setReplicated(false).setIndexMode(IndexMode.LOGSDB).build(); - var newCoordinates = ds.nextWriteIndexAndGeneration(Metadata.EMPTY_METADATA, ds.getBackingIndices()); + var newCoordinates = ds.nextWriteIndexAndGeneration(Metadata.EMPTY_METADATA, ds.getDataComponent()); var rolledDs = ds.rollover(new Index(newCoordinates.v1(), UUIDs.randomBase64UUID()), newCoordinates.v2(), null, null); assertThat(rolledDs.getName(), equalTo(ds.getName())); @@ -301,18 +301,18 @@ public void testRolloverDowngradeFromLogsdbToRegularDataStream() { public void testRolloverFailureStore() { DataStream ds = DataStreamTestHelper.randomInstance(true).promoteDataStream(); - Tuple newCoordinates = ds.nextWriteIndexAndGeneration(Metadata.EMPTY_METADATA, ds.getFailureIndices()); + Tuple newCoordinates = ds.nextWriteIndexAndGeneration(Metadata.EMPTY_METADATA, ds.getFailureComponent()); final DataStream rolledDs = ds.rolloverFailureStore(new Index(newCoordinates.v1(), UUIDs.randomBase64UUID()), newCoordinates.v2()); assertThat(rolledDs.getName(), equalTo(ds.getName())); assertThat(rolledDs.getGeneration(), equalTo(ds.getGeneration() + 1)); assertThat(rolledDs.getIndices().size(), equalTo(ds.getIndices().size())); // Ensure that the rolloverOnWrite flag hasn't changed when rolling over a failure store. assertThat(rolledDs.rolloverOnWrite(), equalTo(ds.rolloverOnWrite())); - assertThat(rolledDs.getFailureIndices().getIndices().size(), equalTo(ds.getFailureIndices().getIndices().size() + 1)); + assertThat(rolledDs.getFailureIndices().size(), equalTo(ds.getFailureIndices().size() + 1)); assertTrue(rolledDs.getIndices().containsAll(ds.getIndices())); assertTrue(rolledDs.getIndices().contains(rolledDs.getWriteIndex())); - assertTrue(rolledDs.getFailureIndices().getIndices().containsAll(ds.getFailureIndices().getIndices())); - assertTrue(rolledDs.getFailureIndices().getIndices().contains(rolledDs.getFailureStoreWriteIndex())); + assertTrue(rolledDs.getFailureIndices().containsAll(ds.getFailureIndices())); + assertTrue(rolledDs.getFailureIndices().contains(rolledDs.getWriteFailureIndex())); } public void testRemoveBackingIndex() { @@ -361,18 +361,15 @@ public void testRemoveBackingWriteIndex() { public void testRemoveFailureStoreIndex() { DataStream original = createRandomDataStream(); - int indexToRemove = randomIntBetween(1, original.getFailureIndices().getIndices().size() - 1); + int indexToRemove = randomIntBetween(1, original.getFailureIndices().size() - 1); - DataStream updated = original.removeFailureStoreIndex(original.getFailureIndices().getIndices().get(indexToRemove - 1)); + DataStream updated = original.removeFailureStoreIndex(original.getFailureIndices().get(indexToRemove - 1)); assertThat(updated.getName(), equalTo(original.getName())); assertThat(updated.getGeneration(), equalTo(original.getGeneration() + 1)); assertThat(updated.getIndices().size(), equalTo(original.getIndices().size())); - assertThat(updated.getFailureIndices().getIndices().size(), equalTo(original.getFailureIndices().getIndices().size() - 1)); - for (int k = 0; k < (original.getFailureIndices().getIndices().size() - 1); k++) { - assertThat( - updated.getFailureIndices().getIndices().get(k), - equalTo(original.getFailureIndices().getIndices().get(k < (indexToRemove - 1) ? k : k + 1)) - ); + assertThat(updated.getFailureIndices().size(), equalTo(original.getFailureIndices().size() - 1)); + for (int k = 0; k < (original.getFailureIndices().size() - 1); k++) { + assertThat(updated.getFailureIndices().get(k), equalTo(original.getFailureIndices().get(k < (indexToRemove - 1) ? k : k + 1))); } } @@ -389,16 +386,16 @@ public void testRemoveFailureStoreIndexThatDoesNotExist() { public void testRemoveFailureStoreWriteIndex() { DataStream original = createRandomDataStream(); - int indexToRemove = original.getFailureIndices().getIndices().size() - 1; + int indexToRemove = original.getFailureIndices().size() - 1; - DataStream updated = original.removeFailureStoreIndex(original.getFailureIndices().getIndices().get(indexToRemove)); + DataStream updated = original.removeFailureStoreIndex(original.getFailureIndices().get(indexToRemove)); assertThat(updated.getName(), equalTo(original.getName())); assertThat(updated.getGeneration(), equalTo(original.getGeneration() + 1)); assertThat(updated.getIndices().size(), equalTo(original.getIndices().size())); - assertThat(updated.getFailureIndices().getIndices().size(), equalTo(original.getFailureIndices().getIndices().size() - 1)); - assertThat(updated.getFailureIndices().isRolloverOnWrite(), equalTo(true)); - for (int k = 0; k < (original.getFailureIndices().getIndices().size() - 1); k++) { - assertThat(updated.getFailureIndices().getIndices().get(k), equalTo(original.getFailureIndices().getIndices().get(k))); + assertThat(updated.getFailureIndices().size(), equalTo(original.getFailureIndices().size() - 1)); + assertThat(updated.getFailureComponent().isRolloverOnWrite(), equalTo(true)); + for (int k = 0; k < (original.getFailureIndices().size() - 1); k++) { + assertThat(updated.getFailureIndices().get(k), equalTo(original.getFailureIndices().get(k))); } } @@ -440,9 +437,9 @@ public void testAddBackingIndexThatIsPartOfAnotherDataStream() { builder.put(ds2); createMetadataForIndices(builder, ds1.getIndices()); - createMetadataForIndices(builder, ds1.getFailureIndices().getIndices()); + createMetadataForIndices(builder, ds1.getFailureIndices()); createMetadataForIndices(builder, ds2.getIndices()); - createMetadataForIndices(builder, ds2.getFailureIndices().getIndices()); + createMetadataForIndices(builder, ds2.getFailureIndices()); Index indexToAdd = randomFrom(ds2.getIndices().toArray(Index.EMPTY_ARRAY)); @@ -470,11 +467,11 @@ public void testAddBackingIndexThatIsPartOfDataStreamFailureStore() { builder.put(ds2); createMetadataForIndices(builder, ds1.getIndices()); - createMetadataForIndices(builder, ds1.getFailureIndices().getIndices()); + createMetadataForIndices(builder, ds1.getFailureIndices()); createMetadataForIndices(builder, ds2.getIndices()); - createMetadataForIndices(builder, ds2.getFailureIndices().getIndices()); + createMetadataForIndices(builder, ds2.getFailureIndices()); - Index indexToAdd = randomFrom(ds2.getFailureIndices().getIndices().toArray(Index.EMPTY_ARRAY)); + Index indexToAdd = randomFrom(ds2.getFailureIndices().toArray(Index.EMPTY_ARRAY)); IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> ds1.addBackingIndex(builder.build(), indexToAdd)); assertThat( @@ -559,7 +556,7 @@ public void testAddFailureStoreIndex() { builder.put(original); createMetadataForIndices(builder, original.getIndices()); - createMetadataForIndices(builder, original.getFailureIndices().getIndices()); + createMetadataForIndices(builder, original.getFailureIndices()); Index indexToAdd = new Index(randomAlphaOfLength(4), UUIDs.randomBase64UUID(random())); builder.put( @@ -575,11 +572,11 @@ public void testAddFailureStoreIndex() { assertThat(updated.getName(), equalTo(original.getName())); assertThat(updated.getGeneration(), equalTo(original.getGeneration() + 1)); assertThat(updated.getIndices().size(), equalTo(original.getIndices().size())); - assertThat(updated.getFailureIndices().getIndices().size(), equalTo(original.getFailureIndices().getIndices().size() + 1)); - for (int k = 1; k <= original.getFailureIndices().getIndices().size(); k++) { - assertThat(updated.getFailureIndices().getIndices().get(k), equalTo(original.getFailureIndices().getIndices().get(k - 1))); + assertThat(updated.getFailureIndices().size(), equalTo(original.getFailureIndices().size() + 1)); + for (int k = 1; k <= original.getFailureIndices().size(); k++) { + assertThat(updated.getFailureIndices().get(k), equalTo(original.getFailureIndices().get(k - 1))); } - assertThat(updated.getFailureIndices().getIndices().get(0), equalTo(indexToAdd)); + assertThat(updated.getFailureIndices().get(0), equalTo(indexToAdd)); } public void testAddFailureStoreIndexThatIsPartOfAnotherDataStream() { @@ -591,11 +588,11 @@ public void testAddFailureStoreIndexThatIsPartOfAnotherDataStream() { builder.put(ds2); createMetadataForIndices(builder, ds1.getIndices()); - createMetadataForIndices(builder, ds1.getFailureIndices().getIndices()); + createMetadataForIndices(builder, ds1.getFailureIndices()); createMetadataForIndices(builder, ds2.getIndices()); - createMetadataForIndices(builder, ds2.getFailureIndices().getIndices()); + createMetadataForIndices(builder, ds2.getFailureIndices()); - Index indexToAdd = randomFrom(ds2.getFailureIndices().getIndices().toArray(Index.EMPTY_ARRAY)); + Index indexToAdd = randomFrom(ds2.getFailureIndices().toArray(Index.EMPTY_ARRAY)); IllegalArgumentException e = expectThrows( IllegalArgumentException.class, @@ -624,9 +621,9 @@ public void testAddFailureStoreIndexThatIsPartOfDataStreamBackingIndices() { builder.put(ds2); createMetadataForIndices(builder, ds1.getIndices()); - createMetadataForIndices(builder, ds1.getFailureIndices().getIndices()); + createMetadataForIndices(builder, ds1.getFailureIndices()); createMetadataForIndices(builder, ds2.getIndices()); - createMetadataForIndices(builder, ds2.getFailureIndices().getIndices()); + createMetadataForIndices(builder, ds2.getFailureIndices()); Index indexToAdd = randomFrom(ds2.getIndices().toArray(Index.EMPTY_ARRAY)); @@ -655,16 +652,16 @@ public void testAddExistingFailureStoreIndex() { builder.put(original); createMetadataForIndices(builder, original.getIndices()); - createMetadataForIndices(builder, original.getFailureIndices().getIndices()); + createMetadataForIndices(builder, original.getFailureIndices()); - Index indexToAdd = randomFrom(original.getFailureIndices().getIndices().toArray(Index.EMPTY_ARRAY)); + Index indexToAdd = randomFrom(original.getFailureIndices().toArray(Index.EMPTY_ARRAY)); DataStream updated = original.addFailureStoreIndex(builder.build(), indexToAdd); assertThat(updated.getName(), equalTo(original.getName())); assertThat(updated.getGeneration(), equalTo(original.getGeneration())); assertThat(updated.getIndices().size(), equalTo(original.getIndices().size())); - assertThat(updated.getFailureIndices().getIndices().size(), equalTo(original.getFailureIndices().getIndices().size())); - assertThat(updated.getFailureIndices().getIndices(), equalTo(original.getFailureIndices().getIndices())); + assertThat(updated.getFailureIndices().size(), equalTo(original.getFailureIndices().size())); + assertThat(updated.getFailureIndices(), equalTo(original.getFailureIndices())); } public void testAddFailureStoreIndexWithAliases() { @@ -674,7 +671,7 @@ public void testAddFailureStoreIndexWithAliases() { builder.put(original); createMetadataForIndices(builder, original.getIndices()); - createMetadataForIndices(builder, original.getFailureIndices().getIndices()); + createMetadataForIndices(builder, original.getFailureIndices()); Index indexToAdd = new Index(randomAlphaOfLength(4), UUIDs.randomBase64UUID(random())); IndexMetadata.Builder b = IndexMetadata.builder(indexToAdd.getName()) @@ -762,9 +759,9 @@ public void testReplaceBackingIndexThrowsExceptionIfIndexNotPartOfDataStream() { public void testReplaceBackingIndexThrowsExceptionIfIndexPartOfFailureStore() { DataStream original = createRandomDataStream(); - int indexToReplace = randomIntBetween(1, original.getFailureIndices().getIndices().size() - 1) - 1; + int indexToReplace = randomIntBetween(1, original.getFailureIndices().size() - 1) - 1; - Index failureIndex = original.getFailureIndices().getIndices().get(indexToReplace); + Index failureIndex = original.getFailureIndices().get(indexToReplace); Index newBackingIndex = new Index("replacement-index", UUIDs.randomBase64UUID(random())); expectThrows(IllegalArgumentException.class, () -> original.replaceBackingIndex(failureIndex, newBackingIndex)); } @@ -801,21 +798,18 @@ public void testReplaceBackingIndexThrowsExceptionIfReplacingWriteIndex() { public void testReplaceFailureIndex() { DataStream original = createRandomDataStream(); - int indexToReplace = randomIntBetween(1, original.getFailureIndices().getIndices().size() - 1) - 1; + int indexToReplace = randomIntBetween(1, original.getFailureIndices().size() - 1) - 1; Index newFailureIndex = new Index("replacement-index", UUIDs.randomBase64UUID(random())); - DataStream updated = original.replaceFailureStoreIndex( - original.getFailureIndices().getIndices().get(indexToReplace), - newFailureIndex - ); + DataStream updated = original.replaceFailureStoreIndex(original.getFailureIndices().get(indexToReplace), newFailureIndex); assertThat(updated.getName(), equalTo(original.getName())); assertThat(updated.getGeneration(), equalTo(original.getGeneration() + 1)); - assertThat(updated.getFailureIndices().getIndices().size(), equalTo(original.getFailureIndices().getIndices().size())); - assertThat(updated.getFailureIndices().getIndices().get(indexToReplace), equalTo(newFailureIndex)); + assertThat(updated.getFailureIndices().size(), equalTo(original.getFailureIndices().size())); + assertThat(updated.getFailureIndices().get(indexToReplace), equalTo(newFailureIndex)); - for (int i = 0; i < original.getFailureIndices().getIndices().size(); i++) { + for (int i = 0; i < original.getFailureIndices().size(); i++) { if (i != indexToReplace) { - assertThat(updated.getFailureIndices().getIndices().get(i), equalTo(original.getFailureIndices().getIndices().get(i))); + assertThat(updated.getFailureIndices().get(i), equalTo(original.getFailureIndices().get(i))); } } } @@ -886,23 +880,23 @@ public void testSnapshot() { postSnapshotBackingIndices.addAll(backingIndicesToAdd); // Mutate failure indices - var failureIndicesToRemove = randomSubsetOf(preSnapshotDataStream.getFailureIndices().getIndices()); + var failureIndicesToRemove = randomSubsetOf(preSnapshotDataStream.getFailureIndices()); var failureIndicesToAdd = randomIndexInstances(); - var postSnapshotFailureIndices = new ArrayList<>(preSnapshotDataStream.getFailureIndices().getIndices()); + var postSnapshotFailureIndices = new ArrayList<>(preSnapshotDataStream.getFailureIndices()); postSnapshotFailureIndices.removeAll(failureIndicesToRemove); postSnapshotFailureIndices.addAll(failureIndicesToAdd); var replicated = preSnapshotDataStream.isReplicated() && randomBoolean(); var postSnapshotDataStream = preSnapshotDataStream.copy() .setBackingIndices( - preSnapshotDataStream.getBackingIndices() + preSnapshotDataStream.getDataComponent() .copy() .setIndices(postSnapshotBackingIndices) .setRolloverOnWrite(replicated == false && preSnapshotDataStream.rolloverOnWrite()) .build() ) .setFailureIndices( - preSnapshotDataStream.getFailureIndices() + preSnapshotDataStream.getFailureComponent() .copy() .setIndices(postSnapshotFailureIndices) .setRolloverOnWrite(replicated == false && preSnapshotDataStream.rolloverOnWrite()) @@ -915,7 +909,7 @@ public void testSnapshot() { Set indicesInSnapshot = new HashSet<>(); preSnapshotDataStream.getIndices().forEach(index -> indicesInSnapshot.add(index.getName())); - preSnapshotDataStream.getFailureIndices().getIndices().forEach(index -> indicesInSnapshot.add(index.getName())); + preSnapshotDataStream.getFailureIndices().forEach(index -> indicesInSnapshot.add(index.getName())); var reconciledDataStream = postSnapshotDataStream.snapshot(indicesInSnapshot, Metadata.builder()); assertThat(reconciledDataStream.getName(), equalTo(postSnapshotDataStream.getName())); @@ -936,12 +930,12 @@ public void testSnapshot() { reconciledDataStream.getIndices().size(), equalTo(preSnapshotDataStream.getIndices().size() - backingIndicesToRemove.size()) ); - var reconciledFailureIndices = reconciledDataStream.getFailureIndices().getIndices(); + var reconciledFailureIndices = reconciledDataStream.getFailureIndices(); assertThat(reconciledFailureIndices, everyItem(not(in(failureIndicesToRemove)))); assertThat(reconciledFailureIndices, everyItem(not(in(failureIndicesToAdd)))); assertThat( reconciledFailureIndices.size(), - equalTo(preSnapshotDataStream.getFailureIndices().getIndices().size() - failureIndicesToRemove.size()) + equalTo(preSnapshotDataStream.getFailureIndices().size() - failureIndicesToRemove.size()) ); } @@ -950,7 +944,7 @@ public void testSnapshotWithAllBackingIndicesRemoved() { var indicesToAdd = randomNonEmptyIndexInstances(); var postSnapshotDataStream = preSnapshotDataStream.copy() - .setBackingIndices(preSnapshotDataStream.getBackingIndices().copy().setIndices(indicesToAdd).build()) + .setBackingIndices(preSnapshotDataStream.getDataComponent().copy().setIndices(indicesToAdd).build()) .build(); assertNull( @@ -2139,7 +2133,7 @@ public void testWriteFailureIndex() { replicated == false && randomBoolean(), null ); - assertThat(noFailureStoreDataStream.getFailureStoreWriteIndex(), nullValue()); + assertThat(noFailureStoreDataStream.getWriteFailureIndex(), nullValue()); DataStream failureStoreDataStreamWithEmptyFailureIndices = new DataStream( randomAlphaOfLength(10), @@ -2157,7 +2151,7 @@ public void testWriteFailureIndex() { replicated == false && randomBoolean(), null ); - assertThat(failureStoreDataStreamWithEmptyFailureIndices.getFailureStoreWriteIndex(), nullValue()); + assertThat(failureStoreDataStreamWithEmptyFailureIndices.getWriteFailureIndex(), nullValue()); List failureIndices = randomIndexInstances(); String dataStreamName = randomAlphaOfLength(10); @@ -2182,7 +2176,7 @@ public void testWriteFailureIndex() { replicated == false && randomBoolean(), null ); - assertThat(failureStoreDataStream.getFailureStoreWriteIndex(), is(writeFailureIndex)); + assertThat(failureStoreDataStream.getWriteFailureIndex(), is(writeFailureIndex)); } public void testIsFailureIndex() { diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexAbstractionTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexAbstractionTests.java new file mode 100644 index 0000000000000..abcd65c3d2d25 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexAbstractionTests.java @@ -0,0 +1,211 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.cluster.metadata; + +import org.elasticsearch.common.UUIDs; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.test.ESTestCase; + +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; + +import static org.hamcrest.CoreMatchers.nullValue; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.equalTo; + +public class IndexAbstractionTests extends ESTestCase { + + public void testIndexAbstractionsDoNotSupportFailureIndices() { + AliasMetadata aliasMetadata = AliasMetadata.builder("my-alias").build(); + IndexMetadata standaloneIndexMetadata = newIndexMetadata("my-index", aliasMetadata); + IndexMetadata backingIndexMetadata = newIndexMetadata(".ds-my-ds", null); + IndexMetadata failureIndexMetadata = newIndexMetadata(".fs-my-ds", null); + DataStream dataStreamWithFs = newDataStreamInstance( + List.of(backingIndexMetadata.getIndex()), + List.of(failureIndexMetadata.getIndex()) + ); + Metadata metadata = Metadata.builder() + .put(standaloneIndexMetadata, false) + .put(backingIndexMetadata, false) + .put(failureIndexMetadata, false) + .dataStreams(Map.of(dataStreamWithFs.getName(), dataStreamWithFs), Map.of()) + .build(); + + // Concrete indices do not support failure store + + IndexAbstraction standaloneIndex = new IndexAbstraction.ConcreteIndex(standaloneIndexMetadata); + assertThat(standaloneIndex.getWriteIndex(), equalTo(standaloneIndexMetadata.getIndex())); + assertThat(standaloneIndex.getWriteFailureIndex(metadata), nullValue()); + assertThat(standaloneIndex.getFailureIndices(metadata), empty()); + + // Even if they belong to a data stream + IndexAbstraction backingIndex = new IndexAbstraction.ConcreteIndex(backingIndexMetadata, dataStreamWithFs); + assertThat(backingIndex.getWriteIndex(), equalTo(backingIndexMetadata.getIndex())); + assertThat(backingIndex.getWriteFailureIndex(metadata), nullValue()); + assertThat(backingIndex.getFailureIndices(metadata), empty()); + + IndexAbstraction failureIndex = new IndexAbstraction.ConcreteIndex(failureIndexMetadata, dataStreamWithFs); + assertThat(failureIndex.getWriteIndex(), equalTo(failureIndexMetadata.getIndex())); + assertThat(failureIndex.getWriteFailureIndex(metadata), nullValue()); + assertThat(failureIndex.getFailureIndices(metadata), empty()); + + // Aliases of standalone indices also do not support the failure store + List referenceIndices = List.of(standaloneIndexMetadata); + IndexAbstraction alias = new IndexAbstraction.Alias(aliasMetadata, referenceIndices); + assertThat(alias.getIndices(), containsInAnyOrder(referenceIndices.stream().map(IndexMetadata::getIndex).toArray())); + assertThat(alias.getWriteFailureIndex(metadata), nullValue()); + assertThat(alias.getFailureIndices(metadata), empty()); + } + + public void testIndexAbstractionsWithFailureIndices() { + IndexMetadata backingIndexMetadata = newIndexMetadata(".ds-my-fs", null); + IndexMetadata failureIndexMetadata = newIndexMetadata(".fs-my-fs", null); + IndexMetadata otherBackingIndexMetadata = newIndexMetadata(".ds-my-ds", null); + DataStream dsWithFailureStore = newDataStreamInstance( + List.of(backingIndexMetadata.getIndex()), + List.of(failureIndexMetadata.getIndex()) + ); + DataStream dsWithoutFailureStore = newDataStreamInstance(List.of(otherBackingIndexMetadata.getIndex()), List.of()); + DataStreamAlias aliasWithoutFailureStore = new DataStreamAlias( + "no-fs-alias", + List.of(dsWithoutFailureStore.getName()), + dsWithoutFailureStore.getName(), + Map.of() + ); + DataStreamAlias aliasWithFailureStore = new DataStreamAlias( + "with-fs-alias", + List.of(dsWithoutFailureStore.getName(), dsWithFailureStore.getName()), + dsWithFailureStore.getName(), + Map.of() + ); + DataStreamAlias aliasWithoutWriteDataStream = new DataStreamAlias( + "no-write-alias", + List.of(dsWithoutFailureStore.getName(), dsWithFailureStore.getName()), + null, + Map.of() + ); + DataStreamAlias aliasWithoutWriteFailureStoreDataStream = new DataStreamAlias( + "no-write-failure-stote-alias", + List.of(dsWithoutFailureStore.getName(), dsWithFailureStore.getName()), + dsWithoutFailureStore.getName(), + Map.of() + ); + Metadata metadata = Metadata.builder() + .put(otherBackingIndexMetadata, false) + .put(backingIndexMetadata, false) + .put(failureIndexMetadata, false) + .dataStreams( + Map.of(dsWithFailureStore.getName(), dsWithFailureStore, dsWithoutFailureStore.getName(), dsWithoutFailureStore), + Map.of( + aliasWithoutFailureStore.getAlias(), + aliasWithoutFailureStore, + aliasWithFailureStore.getAlias(), + aliasWithFailureStore, + aliasWithoutWriteDataStream.getAlias(), + aliasWithoutWriteDataStream + ) + ) + .build(); + + // Data stream with no failure store + assertThat(dsWithoutFailureStore.getWriteIndex(), equalTo(otherBackingIndexMetadata.getIndex())); + assertThat(dsWithoutFailureStore.getIndices(), contains(otherBackingIndexMetadata.getIndex())); + assertThat(dsWithoutFailureStore.getWriteFailureIndex(), nullValue()); + assertThat(dsWithoutFailureStore.getFailureIndices(), empty()); + + // Data stream with failure store + assertThat(dsWithFailureStore.getWriteIndex(), equalTo(backingIndexMetadata.getIndex())); + assertThat(dsWithFailureStore.getIndices(), contains(backingIndexMetadata.getIndex())); + assertThat(dsWithFailureStore.getWriteFailureIndex(), equalTo(failureIndexMetadata.getIndex())); + assertThat(dsWithFailureStore.getFailureIndices(), contains(failureIndexMetadata.getIndex())); + + // Alias with no write data stream + List referenceIndices = Stream.concat(dsWithFailureStore.getIndices().stream(), dsWithoutFailureStore.getIndices().stream()) + .toList(); + IndexAbstraction aliasWithNoWriteDs = new IndexAbstraction.Alias( + aliasWithoutWriteDataStream, + referenceIndices, + null, + List.of(dsWithFailureStore.getName(), dsWithoutFailureStore.getName()) + ); + assertThat(aliasWithNoWriteDs.getWriteIndex(), nullValue()); + assertThat(aliasWithNoWriteDs.getIndices(), containsInAnyOrder(referenceIndices.toArray())); + assertThat(aliasWithNoWriteDs.getWriteFailureIndex(metadata), nullValue()); + assertThat(aliasWithNoWriteDs.getFailureIndices(metadata), contains(failureIndexMetadata.getIndex())); + + // Alias with no failure store + IndexAbstraction aliasWithNoFs = new IndexAbstraction.Alias( + aliasWithoutFailureStore, + dsWithoutFailureStore.getIndices(), + dsWithoutFailureStore.getWriteIndex(), + List.of(dsWithoutFailureStore.getName()) + ); + assertThat(aliasWithNoFs.getWriteIndex(), equalTo(dsWithoutFailureStore.getWriteIndex())); + assertThat(aliasWithNoFs.getIndices(), contains(otherBackingIndexMetadata.getIndex())); + assertThat(aliasWithNoFs.getWriteFailureIndex(metadata), nullValue()); + assertThat(aliasWithNoFs.getFailureIndices(metadata), empty()); + + // Alias with failure store and write ds with failure store + IndexAbstraction aliasWithWriteFs = new IndexAbstraction.Alias( + aliasWithoutWriteDataStream, + referenceIndices, + dsWithFailureStore.getWriteIndex(), + List.of(dsWithFailureStore.getName(), dsWithoutFailureStore.getName()) + ); + assertThat(aliasWithWriteFs.getWriteIndex(), equalTo(backingIndexMetadata.getIndex())); + assertThat(aliasWithWriteFs.getIndices(), containsInAnyOrder(referenceIndices.toArray())); + assertThat(aliasWithWriteFs.getWriteFailureIndex(metadata), equalTo(failureIndexMetadata.getIndex())); + assertThat(aliasWithWriteFs.getFailureIndices(metadata), contains(failureIndexMetadata.getIndex())); + + // Alias with failure store and write ds without failure store + IndexAbstraction aliasWithWithoutWriteFs = new IndexAbstraction.Alias( + aliasWithoutWriteDataStream, + referenceIndices, + dsWithoutFailureStore.getWriteIndex(), + List.of(dsWithFailureStore.getName(), dsWithoutFailureStore.getName()) + ); + assertThat(aliasWithWithoutWriteFs.getWriteIndex(), equalTo(otherBackingIndexMetadata.getIndex())); + assertThat(aliasWithWithoutWriteFs.getIndices(), containsInAnyOrder(referenceIndices.toArray())); + assertThat(aliasWithWithoutWriteFs.getWriteFailureIndex(metadata), nullValue()); + assertThat(aliasWithWithoutWriteFs.getFailureIndices(metadata), contains(failureIndexMetadata.getIndex())); + } + + private IndexMetadata newIndexMetadata(String indexName, AliasMetadata aliasMetadata) { + Settings dummyIndexSettings = Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()) + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID()) + .build(); + + IndexMetadata.Builder builder = new IndexMetadata.Builder(indexName).settings(dummyIndexSettings); + if (aliasMetadata != null) { + builder.putAlias(aliasMetadata); + } + return builder.build(); + } + + private static DataStream newDataStreamInstance(List backingIndices, List failureStoreIndices) { + boolean isSystem = randomBoolean(); + return DataStream.builder(randomAlphaOfLength(50), backingIndices) + .setFailureIndices(DataStream.DataStreamIndices.failureIndicesBuilder(failureStoreIndices).build()) + .setGeneration(randomLongBetween(1, 1000)) + .setMetadata(Map.of()) + .setSystem(isSystem) + .setHidden(isSystem || randomBoolean()) + .setReplicated(randomBoolean()) + .build(); + } +} diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java index bb4c843bb7bdf..7cb13027a064d 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataCreateDataStreamServiceTests.java @@ -333,7 +333,7 @@ public void testCreateDataStreamWithFailureStoreUninitialized() throws Exception assertThat(newState.metadata().dataStreams().get(dataStreamName).isSystem(), is(false)); assertThat(newState.metadata().dataStreams().get(dataStreamName).isHidden(), is(false)); assertThat(newState.metadata().dataStreams().get(dataStreamName).isReplicated(), is(false)); - assertThat(newState.metadata().dataStreams().get(dataStreamName).getFailureIndices().getIndices(), empty()); + assertThat(newState.metadata().dataStreams().get(dataStreamName).getFailureIndices(), empty()); assertThat(newState.metadata().index(backingIndexName), notNullValue()); assertThat(newState.metadata().index(backingIndexName).getSettings().get("index.hidden"), equalTo("true")); assertThat(newState.metadata().index(backingIndexName).isSystem(), is(false)); diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsServiceTests.java index 276c20d2d1322..3c8b6ebbfb271 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDataStreamsServiceTests.java @@ -360,7 +360,7 @@ public void testRemoveBrokenBackingIndexReference() { var original = state.getMetadata().dataStreams().get(dataStreamName); var broken = original.copy() .setBackingIndices( - original.getBackingIndices() + original.getDataComponent() .copy() .setIndices(List.of(new Index(original.getIndices().get(0).getName(), "broken"), original.getIndices().get(1))) .build() diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDeleteIndexServiceTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDeleteIndexServiceTests.java index 3ada92dbe7ae5..9abbc59d672e2 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDeleteIndexServiceTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataDeleteIndexServiceTests.java @@ -304,10 +304,10 @@ public void testDeleteMultipleFailureIndexForDataStream() { DataStream dataStream = after.metadata().dataStreams().get(dataStreamName); assertThat(dataStream, notNullValue()); - assertThat(dataStream.getFailureIndices().getIndices().size(), equalTo(numBackingIndices - indexNumbersToDelete.size())); + assertThat(dataStream.getFailureIndices().size(), equalTo(numBackingIndices - indexNumbersToDelete.size())); for (Index i : indicesToDelete) { assertThat(after.metadata().getIndices().get(i.getName()), nullValue()); - assertFalse(dataStream.getFailureIndices().getIndices().contains(i)); + assertFalse(dataStream.getFailureIndices().contains(i)); } assertThat(after.metadata().getIndices().size(), equalTo((2 * numBackingIndices) - indexNumbersToDelete.size())); } diff --git a/server/src/test/java/org/elasticsearch/snapshots/RestoreServiceTests.java b/server/src/test/java/org/elasticsearch/snapshots/RestoreServiceTests.java index 1060654da14da..0213ee9046462 100644 --- a/server/src/test/java/org/elasticsearch/snapshots/RestoreServiceTests.java +++ b/server/src/test/java/org/elasticsearch/snapshots/RestoreServiceTests.java @@ -76,7 +76,7 @@ public void testUpdateDataStream() { assertEquals(dataStreamName, updateDataStream.getName()); assertEquals(List.of(updatedBackingIndex), updateDataStream.getIndices()); - assertEquals(List.of(updatedFailureIndex), updateDataStream.getFailureIndices().getIndices()); + assertEquals(List.of(updatedFailureIndex), updateDataStream.getFailureIndices()); } public void testUpdateDataStreamRename() { @@ -112,7 +112,7 @@ public void testUpdateDataStreamRename() { assertEquals(renamedDataStreamName, renamedDataStream.getName()); assertEquals(List.of(renamedBackingIndex), renamedDataStream.getIndices()); - assertEquals(List.of(renamedFailureIndex), renamedDataStream.getFailureIndices().getIndices()); + assertEquals(List.of(renamedFailureIndex), renamedDataStream.getFailureIndices()); } public void testPrefixNotChanged() { @@ -147,7 +147,7 @@ public void testPrefixNotChanged() { assertEquals(renamedDataStreamName, renamedDataStream.getName()); assertEquals(List.of(renamedIndex), renamedDataStream.getIndices()); - assertEquals(List.of(renamedFailureIndex), renamedDataStream.getFailureIndices().getIndices()); + assertEquals(List.of(renamedFailureIndex), renamedDataStream.getFailureIndices()); request = new RestoreSnapshotRequest(TEST_REQUEST_TIMEOUT).renamePattern("ds-000001").renameReplacement("ds2-000001"); @@ -155,7 +155,7 @@ public void testPrefixNotChanged() { assertEquals(renamedDataStreamName, renamedDataStream.getName()); assertEquals(List.of(renamedIndex), renamedDataStream.getIndices()); - assertEquals(List.of(renamedFailureIndex), renamedDataStream.getFailureIndices().getIndices()); + assertEquals(List.of(renamedFailureIndex), renamedDataStream.getFailureIndices()); } public void testRefreshRepositoryUuidsDoesNothingIfDisabled() { diff --git a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderService.java b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderService.java index 4c612d5e04886..42caf79fe9a39 100644 --- a/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderService.java +++ b/x-pack/plugin/autoscaling/src/main/java/org/elasticsearch/xpack/autoscaling/storage/ReactiveStorageDeciderService.java @@ -818,7 +818,7 @@ private SingleForecast forecast(Metadata metadata, DataStream stream, long forec final String uuid = UUIDs.randomBase64UUID(); final Tuple rolledDataStreamInfo = stream.unsafeNextWriteIndexAndGeneration( state.metadata(), - stream.getBackingIndices() + stream.getDataComponent() ); stream = stream.unsafeRollover( new Index(rolledDataStreamInfo.v1(), uuid), diff --git a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java index d8e634a297bfa..32ea3bc1caaa6 100644 --- a/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java +++ b/x-pack/plugin/ccr/src/main/java/org/elasticsearch/xpack/ccr/action/TransportPutFollowAction.java @@ -337,11 +337,11 @@ static DataStream updateLocalDataStream( .setBackingIndices( // Replicated data streams can't be rolled over, so having the `rolloverOnWrite` flag set to `true` wouldn't make sense // (and potentially even break things). - remoteDataStream.getBackingIndices().copy().setIndices(List.of(backingIndexToFollow)).setRolloverOnWrite(false).build() + remoteDataStream.getDataComponent().copy().setIndices(List.of(backingIndexToFollow)).setRolloverOnWrite(false).build() ) // Replicated data streams should not have the failure store marked for lazy rollover (which they do by default for lazy // failure store creation). - .setFailureIndices(remoteDataStream.getFailureIndices().copy().setRolloverOnWrite(false).build()) + .setFailureIndices(remoteDataStream.getFailureComponent().copy().setRolloverOnWrite(false).build()) .setReplicated(true) .build(); } else { @@ -384,7 +384,7 @@ static DataStream updateLocalDataStream( } return localDataStream.copy() - .setBackingIndices(localDataStream.getBackingIndices().copy().setIndices(backingIndices).build()) + .setBackingIndices(localDataStream.getDataComponent().copy().setIndices(backingIndices).build()) .setGeneration(remoteDataStream.getGeneration()) .setMetadata(remoteDataStream.getMetadata()) .build(); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/DataStreamUsageTransportAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/DataStreamUsageTransportAction.java index 7eece9177cf2b..f1b3fec97ad3c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/DataStreamUsageTransportAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/DataStreamUsageTransportAction.java @@ -68,8 +68,8 @@ protected void masterOperation( if (ds.isFailureStoreEffectivelyEnabled(dataStreamFailureStoreSettings)) { failureStoreEffectivelyEnabledCounter++; } - if (ds.getFailureIndices().getIndices().isEmpty() == false) { - failureIndicesCounter += ds.getFailureIndices().getIndices().size(); + if (ds.getFailureIndices().isEmpty() == false) { + failureIndicesCounter += ds.getFailureIndices().size(); } } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/CheckNotDataStreamWriteIndexStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/CheckNotDataStreamWriteIndexStep.java index fcb9c78ebefd7..9973f85bc64fa 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/CheckNotDataStreamWriteIndexStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/CheckNotDataStreamWriteIndexStep.java @@ -60,7 +60,7 @@ public Result isConditionMet(Index index, ClusterState clusterState) { assert indexAbstraction != null : "invalid cluster metadata. index [" + indexName + "] was not found"; DataStream dataStream = indexAbstraction.getParentDataStream(); if (dataStream != null) { - boolean isFailureStoreWriteIndex = index.equals(dataStream.getFailureStoreWriteIndex()); + boolean isFailureStoreWriteIndex = index.equals(dataStream.getWriteFailureIndex()); if (isFailureStoreWriteIndex || dataStream.getWriteIndex().equals(index)) { String errorMessage = Strings.format( "index [%s] is the%s write index for data stream [%s], pausing " diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/DeleteStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/DeleteStep.java index 7fb350f13a850..c178b28f08ed6 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/DeleteStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/DeleteStep.java @@ -41,7 +41,7 @@ public void performDuringNoSnapshot(IndexMetadata indexMetadata, ClusterState cu DataStream dataStream = indexAbstraction.getParentDataStream(); if (dataStream != null) { - Index failureStoreWriteIndex = dataStream.getFailureStoreWriteIndex(); + Index failureStoreWriteIndex = dataStream.getWriteFailureIndex(); boolean isFailureStoreWriteIndex = failureStoreWriteIndex != null && indexName.equals(failureStoreWriteIndex.getName()); // using index name equality across this if/else branch as the UUID of the index might change via restoring a data stream diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ReplaceDataStreamBackingIndexStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ReplaceDataStreamBackingIndexStep.java index 8bbe68513f425..68143899c0349 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ReplaceDataStreamBackingIndexStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/ReplaceDataStreamBackingIndexStep.java @@ -85,7 +85,7 @@ public ClusterState performAction(Index index, ClusterState clusterState) { throw new IllegalStateException(errorMessage); } - boolean isFailureStoreWriteIndex = index.equals(dataStream.getFailureStoreWriteIndex()); + boolean isFailureStoreWriteIndex = index.equals(dataStream.getWriteFailureIndex()); if (isFailureStoreWriteIndex || dataStream.getWriteIndex().equals(index)) { String errorMessage = Strings.format( "index [%s] is the%s write index for data stream [%s], pausing " diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RolloverStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RolloverStep.java index 4a8788a849b90..dce6704c21eda 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RolloverStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/RolloverStep.java @@ -61,7 +61,7 @@ public void performAction( final boolean targetFailureStore; DataStream dataStream = indexAbstraction.getParentDataStream(); if (dataStream != null) { - boolean isFailureStoreWriteIndex = indexMetadata.getIndex().equals(dataStream.getFailureStoreWriteIndex()); + boolean isFailureStoreWriteIndex = indexMetadata.getIndex().equals(dataStream.getWriteFailureIndex()); targetFailureStore = dataStream.isFailureStoreIndex(indexMetadata.getIndex().getName()); if (isFailureStoreWriteIndex == false && dataStream.getWriteIndex().equals(indexMetadata.getIndex()) == false) { logger.warn( diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForActiveShardsStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForActiveShardsStep.java index 989223ef48da7..08709f950f562 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForActiveShardsStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForActiveShardsStep.java @@ -84,7 +84,7 @@ public Result isConditionMet(Index index, ClusterState clusterState) { // Determine which write index we care about right now: final Index rolledIndex; if (dataStream.isFailureStoreIndex(index.getName())) { - rolledIndex = dataStream.getFailureStoreWriteIndex(); + rolledIndex = dataStream.getWriteFailureIndex(); } else { rolledIndex = dataStream.getWriteIndex(); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForRolloverReadyStep.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForRolloverReadyStep.java index be1a9d14cfdd1..7fd702bebc9c6 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForRolloverReadyStep.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ilm/WaitForRolloverReadyStep.java @@ -88,7 +88,7 @@ public void evaluateCondition(Metadata metadata, Index index, Listener listener, DataStream dataStream = indexAbstraction.getParentDataStream(); if (dataStream != null) { targetFailureStore = dataStream.isFailureStoreIndex(index.getName()); - boolean isFailureStoreWriteIndex = index.equals(dataStream.getFailureStoreWriteIndex()); + boolean isFailureStoreWriteIndex = index.equals(dataStream.getWriteFailureIndex()); if (isFailureStoreWriteIndex == false && dataStream.getWriteIndex().equals(index) == false) { logger.warn( "index [{}] is not the {}write index for data stream [{}]. skipping rollover for policy [{}]", diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/AuthorizationEngine.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/AuthorizationEngine.java index 80716c9f7c9df..9d102e6954d04 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/AuthorizationEngine.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/AuthorizationEngine.java @@ -12,6 +12,7 @@ import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.IndicesRequest; import org.elasticsearch.cluster.metadata.IndexAbstraction; +import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.BytesStreamOutput; import org.elasticsearch.common.io.stream.StreamInput; @@ -74,7 +75,7 @@ * can actually impersonate the user running the request. *

  • {@link #authorizeClusterAction(RequestInfo, AuthorizationInfo, ActionListener)} if the * request is a cluster level operation.
    • - *
  • {@link #authorizeIndexAction(RequestInfo, AuthorizationInfo, AsyncSupplier, Map, ActionListener)} if + *
  • {@link #authorizeIndexAction(RequestInfo, AuthorizationInfo, AsyncSupplier, Metadata, ActionListener)} if * the request is a an index action. This method may be called multiple times for a single * request as the request may be made up of sub-requests that also need to be authorized. The async supplier * for resolved indices will invoke the @@ -83,7 +84,7 @@ * *

    * NOTE: the {@link #loadAuthorizedIndices(RequestInfo, AuthorizationInfo, Map, ActionListener)} - * method may be called prior to {@link #authorizeIndexAction(RequestInfo, AuthorizationInfo, AsyncSupplier, Map, ActionListener)} + * method may be called prior to {@link #authorizeIndexAction(RequestInfo, AuthorizationInfo, AsyncSupplier, Metadata, ActionListener)} * in cases where wildcards need to be expanded. *


    * Authorization engines can be called from various threads including network threads that should @@ -157,7 +158,7 @@ public interface AuthorizationEngine { * from {@link #resolveAuthorizationInfo(RequestInfo, ActionListener)} * @param indicesAsyncSupplier the asynchronous supplier for the indices that this request is * attempting to operate on - * @param aliasOrIndexLookup a map of a string name to the cluster metadata specific to that + * @param metadata a map of a string name to the cluster metadata specific to that * alias or index * @param listener the listener to be notified of the authorization result */ @@ -165,7 +166,7 @@ void authorizeIndexAction( RequestInfo requestInfo, AuthorizationInfo authorizationInfo, AsyncSupplier indicesAsyncSupplier, - Map aliasOrIndexLookup, + Metadata metadata, ActionListener listener ); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/IndicesPermission.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/IndicesPermission.java index 4ea590afff864..b91db5ca34366 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/IndicesPermission.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/IndicesPermission.java @@ -14,6 +14,7 @@ import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexAbstraction; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.logging.DeprecationCategory; @@ -437,13 +438,13 @@ public int size(Map lookup) { for (Index index : indexAbstraction.getIndices()) { DataStream parentDataStream = lookup.get(index.getName()).getParentDataStream(); if (parentDataStream != null && aliasDataStreams.add(parentDataStream)) { - failureIndices += parentDataStream.getFailureIndices().getIndices().size(); + failureIndices += parentDataStream.getFailureIndices().size(); } } size += failureIndices; } else { DataStream parentDataStream = (DataStream) indexAbstraction; - size += parentDataStream.getFailureIndices().getIndices().size(); + size += parentDataStream.getFailureIndices().size(); } } return size; @@ -452,37 +453,18 @@ public int size(Map lookup) { } } - public Collection resolveConcreteIndices(Map lookup) { + public Collection resolveConcreteIndices(Metadata metadata) { if (indexAbstraction == null) { return List.of(); } else if (indexAbstraction.getType() == IndexAbstraction.Type.CONCRETE_INDEX) { return List.of(indexAbstraction.getName()); } else if (IndexComponentSelector.FAILURES.equals(selector)) { - if (IndexAbstraction.Type.ALIAS.equals(indexAbstraction.getType())) { - Set aliasDataStreams = new HashSet<>(); - for (Index index : indexAbstraction.getIndices()) { - DataStream parentDataStream = lookup.get(index.getName()).getParentDataStream(); - if (parentDataStream != null) { - aliasDataStreams.add(parentDataStream); - } - } - List concreteIndexNames = new ArrayList<>(aliasDataStreams.size()); - for (DataStream aliasDataStream : aliasDataStreams) { - DataStream.DataStreamIndices failureIndices = aliasDataStream.getFailureIndices(); - for (Index index : failureIndices.getIndices()) { - concreteIndexNames.add(index.getName()); - } - } - return concreteIndexNames; - } else { - DataStream parentDataStream = (DataStream) indexAbstraction; - DataStream.DataStreamIndices failureIndices = parentDataStream.getFailureIndices(); - List concreteIndexNames = new ArrayList<>(failureIndices.getIndices().size()); - for (Index index : failureIndices.getIndices()) { - concreteIndexNames.add(index.getName()); - } - return concreteIndexNames; + final List failureIndices = indexAbstraction.getFailureIndices(metadata); + final List concreteIndexNames = new ArrayList<>(failureIndices.size()); + for (var idx : failureIndices) { + concreteIndexNames.add(idx.getName()); } + return concreteIndexNames; } else { final List indices = indexAbstraction.getIndices(); final List concreteIndexNames = new ArrayList<>(indices.size()); @@ -504,7 +486,7 @@ public boolean canHaveBackingIndices() { public IndicesAccessControl authorize( String action, Set requestedIndicesOrAliases, - Map lookup, + Metadata metadata, FieldPermissionsCache fieldPermissionsCache ) { // Short circuit if the indicesPermission allows all access to every index @@ -516,7 +498,7 @@ public IndicesAccessControl authorize( final Map resources = Maps.newMapWithExpectedSize(requestedIndicesOrAliases.size()); int totalResourceCount = 0; - + Map lookup = metadata.getIndicesLookup(); for (String indexOrAlias : requestedIndicesOrAliases) { // Remove any selectors from abstraction name. Discard them for this check as we do not have access control for them (yet) Tuple expressionAndSelector = IndexNameExpressionResolver.splitSelectorExpression(indexOrAlias); @@ -536,7 +518,7 @@ public IndicesAccessControl authorize( resources, finalTotalResourceCount, fieldPermissionsCache, - lookup + metadata ); return new IndicesAccessControl(overallGranted, indexPermissions); @@ -547,11 +529,11 @@ private Map buildIndicesAccessC final Map requestedResources, final int totalResourceCount, final FieldPermissionsCache fieldPermissionsCache, - final Map lookup + final Metadata metadata ) { // now... every index that is associated with the request, must be granted - // by at least one indices permission group + // by at least one index permission group final Map> fieldPermissionsByIndex = Maps.newMapWithExpectedSize(totalResourceCount); final Map roleQueriesByIndex = Maps.newMapWithExpectedSize(totalResourceCount); final Set grantedResources = Sets.newHashSetWithExpectedSize(totalResourceCount); @@ -562,7 +544,7 @@ private Map buildIndicesAccessC // true if ANY group covers the given index AND the given action boolean granted = false; - final Collection concreteIndices = resource.resolveConcreteIndices(lookup); + final Collection concreteIndices = resource.resolveConcreteIndices(metadata); for (Group group : groups) { // the group covers the given index OR the given index is a backing index and the group covers the parent data stream if (resource.checkIndex(group)) { diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/LimitedRole.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/LimitedRole.java index ea32ba13ae576..010e08b0d4db6 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/LimitedRole.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/LimitedRole.java @@ -11,7 +11,7 @@ import org.apache.logging.log4j.Logger; import org.apache.lucene.util.automaton.Automaton; import org.elasticsearch.TransportVersion; -import org.elasticsearch.cluster.metadata.IndexAbstraction; +import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.common.Strings; import org.elasticsearch.core.Nullable; import org.elasticsearch.transport.TransportRequest; @@ -28,13 +28,12 @@ import java.util.Collection; import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.Set; /** * A {@link Role} limited by another role.
    - * The effective permissions returned on {@link #authorize(String, Set, Map, FieldPermissionsCache)} call would be limited by the + * The effective permissions returned on {@link #authorize(String, Set, Metadata, FieldPermissionsCache)} call would be limited by the * provided role. */ public final class LimitedRole implements Role { @@ -139,19 +138,14 @@ public int hashCode() { public IndicesAccessControl authorize( String action, Set requestedIndicesOrAliases, - Map aliasAndIndexLookup, + Metadata metadata, FieldPermissionsCache fieldPermissionsCache ) { - IndicesAccessControl indicesAccessControl = baseRole.authorize( - action, - requestedIndicesOrAliases, - aliasAndIndexLookup, - fieldPermissionsCache - ); + IndicesAccessControl indicesAccessControl = baseRole.authorize(action, requestedIndicesOrAliases, metadata, fieldPermissionsCache); IndicesAccessControl limitedByIndicesAccessControl = limitedByRole.authorize( action, requestedIndicesOrAliases, - aliasAndIndexLookup, + metadata, fieldPermissionsCache ); return indicesAccessControl.limitIndicesAccessControl(limitedByIndicesAccessControl); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/Role.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/Role.java index f52f8f85f006d..fe97b152a2ee7 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/Role.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/Role.java @@ -9,7 +9,7 @@ import org.apache.lucene.util.automaton.Automaton; import org.elasticsearch.TransportVersion; -import org.elasticsearch.cluster.metadata.IndexAbstraction; +import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.util.set.Sets; import org.elasticsearch.core.Nullable; @@ -180,7 +180,7 @@ boolean checkApplicationResourcePrivileges( IndicesAccessControl authorize( String action, Set requestedIndicesOrAliases, - Map aliasAndIndexLookup, + Metadata metadata, FieldPermissionsCache fieldPermissionsCache ); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/SimpleRole.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/SimpleRole.java index 0ec9d2a48316a..9b63b73d7801b 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/SimpleRole.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/permission/SimpleRole.java @@ -8,7 +8,7 @@ import org.apache.lucene.util.automaton.Automaton; import org.elasticsearch.TransportVersion; -import org.elasticsearch.cluster.metadata.IndexAbstraction; +import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.cache.Cache; import org.elasticsearch.common.cache.CacheBuilder; @@ -32,7 +32,6 @@ import java.util.Collection; import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.concurrent.ExecutionException; @@ -196,10 +195,10 @@ public boolean checkApplicationResourcePrivileges( public IndicesAccessControl authorize( String action, Set requestedIndicesOrAliases, - Map aliasAndIndexLookup, + Metadata metadata, FieldPermissionsCache fieldPermissionsCache ) { - return indices.authorize(action, requestedIndicesOrAliases, aliasAndIndexLookup, fieldPermissionsCache); + return indices.authorize(action, requestedIndicesOrAliases, metadata, fieldPermissionsCache); } @Override diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ReplaceDataStreamBackingIndexStepTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ReplaceDataStreamBackingIndexStepTests.java index a3318e68305c6..9050154ee6e50 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ReplaceDataStreamBackingIndexStepTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ilm/ReplaceDataStreamBackingIndexStepTests.java @@ -222,9 +222,7 @@ public void testPerformAction() { IndexMetadata indexToOperateOn = useFailureStore ? failureSourceIndexMetadata : sourceIndexMetadata; ClusterState newState = replaceSourceIndexStep.performAction(indexToOperateOn.getIndex(), clusterState); DataStream updatedDataStream = newState.metadata().dataStreams().get(dataStreamName); - DataStream.DataStreamIndices resultIndices = useFailureStore - ? updatedDataStream.getFailureIndices() - : updatedDataStream.getBackingIndices(); + DataStream.DataStreamIndices resultIndices = updatedDataStream.getDataStreamIndices(useFailureStore); assertThat(resultIndices.getIndices().size(), is(2)); assertThat(resultIndices.getIndices().get(0), is(targetIndexMetadata.getIndex())); } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/permission/LimitedRoleTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/permission/LimitedRoleTests.java index feea49430cfc3..a4646c0d736c5 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/permission/LimitedRoleTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/permission/LimitedRoleTests.java @@ -353,7 +353,7 @@ public void testAuthorize() { IndicesAccessControl iac = fromRole.authorize( TransportSearchAction.TYPE.name(), Sets.newHashSet("_index", "_alias1"), - md.getIndicesLookup(), + md, fieldPermissionsCache ); assertThat(iac.isGranted(), is(false)); @@ -361,12 +361,7 @@ public void testAuthorize() { assertThat(iac.hasIndexPermissions("_index"), is(true)); assertThat(iac.getIndexPermissions("_index1"), is(nullValue())); assertThat(iac.hasIndexPermissions("_index1"), is(false)); - iac = fromRole.authorize( - TransportCreateIndexAction.TYPE.name(), - Sets.newHashSet("_index", "_index1"), - md.getIndicesLookup(), - fieldPermissionsCache - ); + iac = fromRole.authorize(TransportCreateIndexAction.TYPE.name(), Sets.newHashSet("_index", "_index1"), md, fieldPermissionsCache); assertThat(iac.isGranted(), is(true)); assertThat(iac.getIndexPermissions("_index"), is(notNullValue())); assertThat(iac.hasIndexPermissions("_index"), is(true)); @@ -382,7 +377,7 @@ public void testAuthorize() { iac = limitedByRole.authorize( TransportSearchAction.TYPE.name(), Sets.newHashSet("_index", "_alias1"), - md.getIndicesLookup(), + md, fieldPermissionsCache ); assertThat(iac.isGranted(), is(false)); @@ -393,7 +388,7 @@ public void testAuthorize() { iac = limitedByRole.authorize( TransportDeleteIndexAction.TYPE.name(), Sets.newHashSet("_index", "_alias1"), - md.getIndicesLookup(), + md, fieldPermissionsCache ); assertThat(iac.isGranted(), is(false)); @@ -404,7 +399,7 @@ public void testAuthorize() { iac = limitedByRole.authorize( TransportCreateIndexAction.TYPE.name(), Sets.newHashSet("_index", "_alias1"), - md.getIndicesLookup(), + md, fieldPermissionsCache ); assertThat(iac.isGranted(), is(false)); @@ -419,34 +414,19 @@ public void testAuthorize() { } else { role = fromRole.limitedBy(limitedByRole); } - iac = role.authorize( - TransportSearchAction.TYPE.name(), - Sets.newHashSet("_index", "_alias1"), - md.getIndicesLookup(), - fieldPermissionsCache - ); + iac = role.authorize(TransportSearchAction.TYPE.name(), Sets.newHashSet("_index", "_alias1"), md, fieldPermissionsCache); assertThat(iac.isGranted(), is(false)); assertThat(iac.getIndexPermissions("_index"), is(notNullValue())); assertThat(iac.hasIndexPermissions("_index"), is(true)); assertThat(iac.getIndexPermissions("_index1"), is(nullValue())); assertThat(iac.hasIndexPermissions("_index1"), is(false)); - iac = role.authorize( - TransportDeleteIndexAction.TYPE.name(), - Sets.newHashSet("_index", "_alias1"), - md.getIndicesLookup(), - fieldPermissionsCache - ); + iac = role.authorize(TransportDeleteIndexAction.TYPE.name(), Sets.newHashSet("_index", "_alias1"), md, fieldPermissionsCache); assertThat(iac.isGranted(), is(false)); assertThat(iac.getIndexPermissions("_index"), is(nullValue())); assertThat(iac.hasIndexPermissions("_index"), is(false)); assertThat(iac.getIndexPermissions("_index1"), is(nullValue())); assertThat(iac.hasIndexPermissions("_index1"), is(false)); - iac = role.authorize( - TransportCreateIndexAction.TYPE.name(), - Sets.newHashSet("_index", "_index1"), - md.getIndicesLookup(), - fieldPermissionsCache - ); + iac = role.authorize(TransportCreateIndexAction.TYPE.name(), Sets.newHashSet("_index", "_index1"), md, fieldPermissionsCache); assertThat(iac.isGranted(), is(false)); assertThat(iac.getIndexPermissions("_index"), is(nullValue())); assertThat(iac.hasIndexPermissions("_index"), is(false)); diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java index b08dd90ae9065..107953557f3ea 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStoreTests.java @@ -223,7 +223,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.SortedMap; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -2634,7 +2633,7 @@ private void assertMonitoringOnRestrictedIndices(Role role) { .authorize( indexMonitoringActionName, Sets.newHashSet(internalSecurityIndex, TestRestrictedIndices.SECURITY_MAIN_ALIAS, asyncSearchIndex), - metadata.getIndicesLookup(), + metadata, fieldPermissionsCache ); assertThat(iac.hasIndexPermissions(internalSecurityIndex), is(true)); @@ -2857,20 +2856,19 @@ public void testSuperuserRole() { .build(); FieldPermissionsCache fieldPermissionsCache = new FieldPermissionsCache(Settings.EMPTY); - SortedMap lookup = metadata.getIndicesLookup(); IndicesAccessControl iac = superuserRole.indices() - .authorize(TransportSearchAction.TYPE.name(), Sets.newHashSet("a1", "ba"), lookup, fieldPermissionsCache); + .authorize(TransportSearchAction.TYPE.name(), Sets.newHashSet("a1", "ba"), metadata, fieldPermissionsCache); assertThat(iac.hasIndexPermissions("a1"), is(true)); assertThat(iac.hasIndexPermissions("b"), is(true)); iac = superuserRole.indices() - .authorize(TransportDeleteIndexAction.TYPE.name(), Sets.newHashSet("a1", "ba"), lookup, fieldPermissionsCache); + .authorize(TransportDeleteIndexAction.TYPE.name(), Sets.newHashSet("a1", "ba"), metadata, fieldPermissionsCache); assertThat(iac.hasIndexPermissions("a1"), is(true)); assertThat(iac.hasIndexPermissions("b"), is(true)); - iac = superuserRole.indices().authorize(TransportIndexAction.NAME, Sets.newHashSet("a2", "ba"), lookup, fieldPermissionsCache); + iac = superuserRole.indices().authorize(TransportIndexAction.NAME, Sets.newHashSet("a2", "ba"), metadata, fieldPermissionsCache); assertThat(iac.hasIndexPermissions("a2"), is(true)); assertThat(iac.hasIndexPermissions("b"), is(true)); iac = superuserRole.indices() - .authorize(TransportUpdateSettingsAction.TYPE.name(), Sets.newHashSet("aaaaaa", "ba"), lookup, fieldPermissionsCache); + .authorize(TransportUpdateSettingsAction.TYPE.name(), Sets.newHashSet("aaaaaa", "ba"), metadata, fieldPermissionsCache); assertThat(iac.hasIndexPermissions("aaaaaa"), is(true)); assertThat(iac.hasIndexPermissions("b"), is(true)); @@ -2879,7 +2877,7 @@ public void testSuperuserRole() { .authorize( randomFrom(TransportSearchAction.TYPE.name(), GetIndexAction.NAME), Sets.newHashSet(TestRestrictedIndices.SECURITY_MAIN_ALIAS), - lookup, + metadata, fieldPermissionsCache ); assertThat("For " + iac, iac.hasIndexPermissions(TestRestrictedIndices.SECURITY_MAIN_ALIAS), is(true)); @@ -2890,7 +2888,7 @@ public void testSuperuserRole() { .authorize( randomFrom(TransportIndexAction.NAME, TransportDeleteIndexAction.TYPE.name()), Sets.newHashSet(TestRestrictedIndices.SECURITY_MAIN_ALIAS), - lookup, + metadata, fieldPermissionsCache ); assertThat("For " + iac, iac.hasIndexPermissions(TestRestrictedIndices.SECURITY_MAIN_ALIAS), is(false)); diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamSecurityIT.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamSecurityIT.java index 2039b72c4f49f..550578161302b 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamSecurityIT.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/integration/DataStreamSecurityIT.java @@ -91,7 +91,7 @@ public ClusterState execute(ClusterState currentState) throws Exception { : original.getIndices().get(0).getName(); DataStream broken = original.copy() .setBackingIndices( - original.getBackingIndices() + original.getDataComponent() .copy() .setIndices(List.of(new Index(brokenIndexName, "broken"), original.getIndices().get(1))) .build() diff --git a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/profile/ProfileCancellationIntegTests.java b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/profile/ProfileCancellationIntegTests.java index 87a5146113f72..d0aa95e37aeee 100644 --- a/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/profile/ProfileCancellationIntegTests.java +++ b/x-pack/plugin/security/src/internalClusterTest/java/org/elasticsearch/xpack/security/profile/ProfileCancellationIntegTests.java @@ -18,6 +18,7 @@ import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.ResponseListener; import org.elasticsearch.cluster.metadata.IndexAbstraction; +import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexModule; @@ -409,7 +410,7 @@ public void authorizeIndexAction( RequestInfo requestInfo, AuthorizationInfo authorizationInfo, AsyncSupplier indicesAsyncSupplier, - Map aliasOrIndexLookup, + Metadata metadata, ActionListener listener ) { listener.onResponse(IndexAuthorizationResult.ALLOW_NO_INDICES); diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java index 0dfdf0861e321..e36c96ab02562 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/AuthorizationService.java @@ -511,7 +511,7 @@ private void authorizeAction( requestInfo, authzInfo, resolvedIndicesAsyncSupplier, - metadata.getIndicesLookup(), + metadata, wrapPreservingContext( new AuthorizationResultListener<>( result -> handleIndexActionAuthorizationResult( @@ -586,7 +586,7 @@ private void handleIndexActionAuthorizationResult( ResolvedIndices withAliases = new ResolvedIndices(aliasesAndIndices, Collections.emptyList()); l.onResponse(withAliases); })), - metadata.getIndicesLookup(), + metadata, wrapPreservingContext( new AuthorizationResultListener<>( authorizationResult -> runRequestInterceptors(requestInfo, authzInfo, authorizationEngine, listener), @@ -867,7 +867,7 @@ private void authorizeBulkItems( bulkItemInfo, authzInfo, ril -> ril.onResponse(new ResolvedIndices(new ArrayList<>(indices), Collections.emptyList())), - metadata.getIndicesLookup(), + metadata, groupedActionListener.delegateFailureAndWrap( (l, indexAuthorizationResult) -> l.onResponse(new Tuple<>(bulkItemAction, indexAuthorizationResult)) ) diff --git a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java index c5f4af93ab4b8..614401770cfb7 100644 --- a/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java +++ b/x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authz/RBACEngine.java @@ -34,6 +34,7 @@ import org.elasticsearch.action.termvectors.MultiTermVectorsAction; import org.elasticsearch.cluster.metadata.DataStream; import org.elasticsearch.cluster.metadata.IndexAbstraction; +import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.regex.Regex; @@ -317,7 +318,7 @@ public void authorizeIndexAction( RequestInfo requestInfo, AuthorizationInfo authorizationInfo, AsyncSupplier indicesAsyncSupplier, - Map aliasOrIndexLookup, + Metadata metadata, ActionListener listener ) { final String action = requestInfo.getAction(); @@ -422,7 +423,7 @@ public void authorizeIndexAction( .allMatch(IndicesAliasesRequest.AliasActions::expandAliasesWildcards)) : "expanded wildcards for local indices OR the request should not expand wildcards at all"; - IndexAuthorizationResult result = buildIndicesAccessControl(action, role, resolvedIndices, aliasOrIndexLookup); + IndexAuthorizationResult result = buildIndicesAccessControl(action, role, resolvedIndices, metadata); if (requestInfo.getAuthentication().isCrossClusterAccess() && request instanceof IndicesRequest.RemoteClusterShardRequest shardsRequest && shardsRequest.shards() != null) { @@ -882,7 +883,7 @@ static AuthorizedIndices resolveAuthorizedIndicesFromRole( indicesAndAliases.add(index.getName()); } // TODO: We need to limit if a data stream's failure indices should return here. - for (Index index : ((DataStream) indexAbstraction).getFailureIndices().getIndices()) { + for (Index index : ((DataStream) indexAbstraction).getFailureIndices()) { indicesAndAliases.add(index.getName()); } } @@ -917,12 +918,12 @@ private IndexAuthorizationResult buildIndicesAccessControl( String action, Role role, ResolvedIndices resolvedIndices, - Map aliasAndIndexLookup + Metadata metadata ) { final IndicesAccessControl accessControl = role.authorize( action, Sets.newHashSet(resolvedIndices.getLocal()), - aliasAndIndexLookup, + metadata, fieldPermissionsCache ); return new IndexAuthorizationResult(accessControl); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java index d3b0d3b2e1faa..4f0e15eec7708 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/AuthorizationServiceTests.java @@ -3416,7 +3416,7 @@ public void authorizeIndexAction( RequestInfo requestInfo, AuthorizationInfo authorizationInfo, AsyncSupplier indicesAsyncSupplier, - Map aliasOrIndexLookup, + Metadata metadata, ActionListener listener ) { throw new UnsupportedOperationException("not implemented"); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java index a0d63d26d475f..f7dc725c3f07d 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/IndicesAndAliasesResolverTests.java @@ -2387,7 +2387,7 @@ public void testBackingIndicesAreVisibleWhenIncludedByRequestWithWildcard() { assertThat(authorizedIndices.all().get(), hasItem(i.getName())); assertThat(authorizedIndices.check(i.getName()), is(true)); } - for (Index i : dataStream.getFailureIndices().getIndices()) { + for (Index i : dataStream.getFailureIndices()) { assertThat(authorizedIndices.all().get(), hasItem(i.getName())); assertThat(authorizedIndices.check(i.getName()), is(true)); } @@ -2427,7 +2427,7 @@ public void testBackingIndicesAreNotVisibleWhenNotIncludedByRequestWithoutWildca assertThat(authorizedIndices.all().get(), hasItem(i.getName())); assertThat(authorizedIndices.check(i.getName()), is(true)); } - for (Index i : dataStream.getFailureIndices().getIndices()) { + for (Index i : dataStream.getFailureIndices()) { assertThat(authorizedIndices.all().get(), hasItem(i.getName())); assertThat(authorizedIndices.check(i.getName()), is(true)); } @@ -2444,7 +2444,7 @@ public void testBackingIndicesAreNotVisibleWhenNotIncludedByRequestWithoutWildca for (Index i : dataStream.getIndices()) { assertThat(resolvedIndices.getLocal(), hasItem(i.getName())); } - for (Index i : dataStream.getFailureIndices().getIndices()) { + for (Index i : dataStream.getFailureIndices()) { assertThat(resolvedIndices.getLocal(), hasItem(i.getName())); } } @@ -2470,7 +2470,7 @@ public void testDataStreamNotAuthorizedWhenBackingIndicesAreAuthorizedViaWildcar assertThat(authorizedIndices.all().get(), hasItem(i.getName())); assertThat(authorizedIndices.check(i.getName()), is(true)); } - for (Index i : dataStream.getFailureIndices().getIndices()) { + for (Index i : dataStream.getFailureIndices()) { assertThat(authorizedIndices.all().get(), hasItem(i.getName())); assertThat(authorizedIndices.check(i.getName()), is(true)); } @@ -2542,7 +2542,7 @@ public void testDataStreamNotAuthorizedWhenBackingIndicesAreAuthorizedViaWildcar assertThat(authorizedIndices.all().get(), hasItem(i.getName())); assertThat(authorizedIndices.check(i.getName()), is(true)); } - for (Index i : dataStream.getFailureIndices().getIndices()) { + for (Index i : dataStream.getFailureIndices()) { assertThat(authorizedIndices.all().get(), hasItem(i.getName())); assertThat(authorizedIndices.check(i.getName()), is(true)); } diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/RBACEngineTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/RBACEngineTests.java index a41c54ada781a..482715bb74c83 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/RBACEngineTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/RBACEngineTests.java @@ -26,6 +26,7 @@ import org.elasticsearch.cluster.metadata.DataStreamTestHelper; import org.elasticsearch.cluster.metadata.IndexAbstraction; import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; import org.elasticsearch.common.bytes.BytesReference; @@ -1971,17 +1972,16 @@ private void authorizeIndicesAction( final RequestInfo requestInfo = createRequestInfo(searchRequest, action, parentAuthorization); final AsyncSupplier indicesAsyncSupplier = s -> s.onResponse(resolvedIndices); - final Map aliasOrIndexLookup = Stream.of(indices) - .collect( - Collectors.toMap( - i -> i, - v -> new IndexAbstraction.ConcreteIndex( - IndexMetadata.builder(v).settings(indexSettings(IndexVersion.current(), 1, 0)).build() - ) + Metadata.Builder metadata = Metadata.builder(); + Stream.of(indices) + .forEach( + indexName -> metadata.put( + IndexMetadata.builder(indexName).settings(indexSettings(IndexVersion.current(), 1, 0)).build(), + false ) ); - engine.authorizeIndexAction(requestInfo, authzInfo, indicesAsyncSupplier, aliasOrIndexLookup, listener); + engine.authorizeIndexAction(requestInfo, authzInfo, indicesAsyncSupplier, metadata.build(), listener); } private static RequestInfo createRequestInfo(TransportRequest request, String action, ParentActionAuthorization parentAuthorization) { diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/accesscontrol/IndicesPermissionTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/accesscontrol/IndicesPermissionTests.java index ac834911fc4e6..4488c28750dc0 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/accesscontrol/IndicesPermissionTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/accesscontrol/IndicesPermissionTests.java @@ -46,7 +46,6 @@ import java.util.Collections; import java.util.List; import java.util.Set; -import java.util.SortedMap; import java.util.stream.Collectors; import static org.elasticsearch.common.settings.Settings.builder; @@ -66,7 +65,6 @@ public void testAuthorize() { .putAlias(AliasMetadata.builder("_alias")); Metadata md = Metadata.builder().put(imbBuilder).build(); FieldPermissionsCache fieldPermissionsCache = new FieldPermissionsCache(Settings.EMPTY); - SortedMap lookup = md.getIndicesLookup(); // basics: Set query = Collections.singleton(new BytesArray("{}")); @@ -77,7 +75,7 @@ public void testAuthorize() { IndicesAccessControl permissions = role.authorize( TransportSearchAction.TYPE.name(), Sets.newHashSet("_index"), - lookup, + md, fieldPermissionsCache ); assertThat(permissions.getIndexPermissions("_index"), notNullValue()); @@ -91,7 +89,7 @@ public void testAuthorize() { role = Role.builder(RESTRICTED_INDICES, "_role") .add(new FieldPermissions(fieldPermissionDef(fields, null)), null, IndexPrivilege.ALL, randomBoolean(), "_index") .build(); - permissions = role.authorize(TransportSearchAction.TYPE.name(), Sets.newHashSet("_index"), lookup, fieldPermissionsCache); + permissions = role.authorize(TransportSearchAction.TYPE.name(), Sets.newHashSet("_index"), md, fieldPermissionsCache); assertThat(permissions.getIndexPermissions("_index"), notNullValue()); assertTrue(permissions.getIndexPermissions("_index").getFieldPermissions().grantsAccessTo("_field")); assertTrue(permissions.getIndexPermissions("_index").getFieldPermissions().hasFieldLevelSecurity()); @@ -102,7 +100,7 @@ public void testAuthorize() { role = Role.builder(RESTRICTED_INDICES, "_role") .add(FieldPermissions.DEFAULT, query, IndexPrivilege.ALL, randomBoolean(), "_index") .build(); - permissions = role.authorize(TransportSearchAction.TYPE.name(), Sets.newHashSet("_index"), lookup, fieldPermissionsCache); + permissions = role.authorize(TransportSearchAction.TYPE.name(), Sets.newHashSet("_index"), md, fieldPermissionsCache); assertThat(permissions.getIndexPermissions("_index"), notNullValue()); assertFalse(permissions.getIndexPermissions("_index").getFieldPermissions().hasFieldLevelSecurity()); assertThat(permissions.getIndexPermissions("_index").getDocumentPermissions().hasDocumentLevelPermissions(), is(true)); @@ -113,7 +111,7 @@ public void testAuthorize() { role = Role.builder(RESTRICTED_INDICES, "_role") .add(new FieldPermissions(fieldPermissionDef(fields, null)), query, IndexPrivilege.ALL, randomBoolean(), "_alias") .build(); - permissions = role.authorize(TransportSearchAction.TYPE.name(), Sets.newHashSet("_alias"), lookup, fieldPermissionsCache); + permissions = role.authorize(TransportSearchAction.TYPE.name(), Sets.newHashSet("_alias"), md, fieldPermissionsCache); assertThat(permissions.getIndexPermissions("_index"), notNullValue()); assertTrue(permissions.getIndexPermissions("_index").getFieldPermissions().grantsAccessTo("_field")); assertTrue(permissions.getIndexPermissions("_index").getFieldPermissions().hasFieldLevelSecurity()); @@ -137,7 +135,7 @@ public void testAuthorize() { role = Role.builder(RESTRICTED_INDICES, "_role") .add(new FieldPermissions(fieldPermissionDef(allFields, null)), query, IndexPrivilege.ALL, randomBoolean(), "_alias") .build(); - permissions = role.authorize(TransportSearchAction.TYPE.name(), Sets.newHashSet("_alias"), lookup, fieldPermissionsCache); + permissions = role.authorize(TransportSearchAction.TYPE.name(), Sets.newHashSet("_alias"), md, fieldPermissionsCache); assertThat(permissions.getIndexPermissions("_index"), notNullValue()); assertFalse(permissions.getIndexPermissions("_index").getFieldPermissions().hasFieldLevelSecurity()); assertThat(permissions.getIndexPermissions("_index").getDocumentPermissions().hasDocumentLevelPermissions(), is(true)); @@ -154,7 +152,6 @@ public void testAuthorize() { .settings(indexSettings(IndexVersion.current(), 1, 1)) .putAlias(AliasMetadata.builder("_alias")); md = Metadata.builder(md).put(imbBuilder1).build(); - lookup = md.getIndicesLookup(); // match all fields with more than one permission Set fooQuery = Collections.singleton(new BytesArray("{foo}")); @@ -163,7 +160,7 @@ public void testAuthorize() { .add(new FieldPermissions(fieldPermissionDef(allFields, null)), fooQuery, IndexPrivilege.ALL, randomBoolean(), "_alias") .add(new FieldPermissions(fieldPermissionDef(allFields, null)), query, IndexPrivilege.ALL, randomBoolean(), "_alias") .build(); - permissions = role.authorize(TransportSearchAction.TYPE.name(), Sets.newHashSet("_alias"), lookup, fieldPermissionsCache); + permissions = role.authorize(TransportSearchAction.TYPE.name(), Sets.newHashSet("_alias"), md, fieldPermissionsCache); Set bothQueries = Sets.union(fooQuery, query); assertThat(permissions.getIndexPermissions("_index"), notNullValue()); assertFalse(permissions.getIndexPermissions("_index").getFieldPermissions().hasFieldLevelSecurity()); @@ -191,7 +188,6 @@ public void testAuthorizeMultipleGroupsMixedDls() { .putAlias(AliasMetadata.builder("_alias")); Metadata md = Metadata.builder().put(imbBuilder).build(); FieldPermissionsCache fieldPermissionsCache = new FieldPermissionsCache(Settings.EMPTY); - SortedMap lookup = md.getIndicesLookup(); Set query = Collections.singleton(new BytesArray("{}")); String[] fields = new String[] { "_field" }; @@ -202,7 +198,7 @@ public void testAuthorizeMultipleGroupsMixedDls() { IndicesAccessControl permissions = role.authorize( TransportSearchAction.TYPE.name(), Sets.newHashSet("_index"), - lookup, + md, fieldPermissionsCache ); assertThat(permissions.getIndexPermissions("_index"), notNullValue()); @@ -253,7 +249,6 @@ public void testCorePermissionAuthorize() { new IndexMetadata.Builder("a1").settings(indexSettings).numberOfShards(1).numberOfReplicas(0).build(), true ).put(new IndexMetadata.Builder("a2").settings(indexSettings).numberOfShards(1).numberOfReplicas(0).build(), true).build(); - SortedMap lookup = metadata.getIndicesLookup(); FieldPermissionsCache fieldPermissionsCache = new FieldPermissionsCache(Settings.EMPTY); IndicesPermission core = new IndicesPermission.Builder(RESTRICTED_INDICES).addGroup( @@ -274,7 +269,7 @@ public void testCorePermissionAuthorize() { IndicesAccessControl iac = core.authorize( TransportSearchAction.TYPE.name(), Sets.newHashSet("a1", "ba"), - lookup, + metadata, fieldPermissionsCache ); assertTrue(iac.getIndexPermissions("a1").getFieldPermissions().grantsAccessTo("denied_field")); @@ -317,7 +312,7 @@ public void testCorePermissionAuthorize() { "a2" ) .build(); - iac = core.authorize(TransportSearchAction.TYPE.name(), Sets.newHashSet("a1", "a2"), lookup, fieldPermissionsCache); + iac = core.authorize(TransportSearchAction.TYPE.name(), Sets.newHashSet("a1", "a2"), metadata, fieldPermissionsCache); assertFalse(iac.getIndexPermissions("a1").getFieldPermissions().hasFieldLevelSecurity()); assertFalse(iac.getIndexPermissions("a2").getFieldPermissions().grantsAccessTo("denied_field2")); assertFalse(iac.getIndexPermissions("a2").getFieldPermissions().grantsAccessTo("denied_field")); @@ -368,7 +363,6 @@ public void testSecurityIndicesPermissions() { true ).build(); FieldPermissionsCache fieldPermissionsCache = new FieldPermissionsCache(Settings.EMPTY); - SortedMap lookup = metadata.getIndicesLookup(); // allow_restricted_indices: false IndicesPermission indicesPermission = new IndicesPermission.Builder(RESTRICTED_INDICES).addGroup( @@ -381,7 +375,7 @@ public void testSecurityIndicesPermissions() { IndicesAccessControl iac = indicesPermission.authorize( TransportSearchAction.TYPE.name(), Sets.newHashSet(internalSecurityIndex, SecuritySystemIndices.SECURITY_MAIN_ALIAS), - lookup, + metadata, fieldPermissionsCache ); assertThat(iac.isGranted(), is(false)); @@ -401,7 +395,7 @@ public void testSecurityIndicesPermissions() { iac = indicesPermission.authorize( TransportSearchAction.TYPE.name(), Sets.newHashSet(internalSecurityIndex, SecuritySystemIndices.SECURITY_MAIN_ALIAS), - lookup, + metadata, fieldPermissionsCache ); assertThat(iac.isGranted(), is(true)); @@ -419,7 +413,6 @@ public void testAsyncSearchIndicesPermissions() { true ).build(); FieldPermissionsCache fieldPermissionsCache = new FieldPermissionsCache(Settings.EMPTY); - SortedMap lookup = metadata.getIndicesLookup(); // allow_restricted_indices: false IndicesPermission indicesPermission = new IndicesPermission.Builder(RESTRICTED_INDICES).addGroup( @@ -432,7 +425,7 @@ public void testAsyncSearchIndicesPermissions() { IndicesAccessControl iac = indicesPermission.authorize( TransportSearchAction.TYPE.name(), Sets.newHashSet(asyncSearchIndex), - lookup, + metadata, fieldPermissionsCache ); assertThat(iac.isGranted(), is(false)); @@ -450,7 +443,7 @@ public void testAsyncSearchIndicesPermissions() { iac = indicesPermission.authorize( TransportSearchAction.TYPE.name(), Sets.newHashSet(asyncSearchIndex), - lookup, + metadata, fieldPermissionsCache ); assertThat(iac.isGranted(), is(true)); @@ -477,7 +470,6 @@ public void testAuthorizationForBackingIndices() { Metadata metadata = builder.build(); FieldPermissionsCache fieldPermissionsCache = new FieldPermissionsCache(Settings.EMPTY); - SortedMap lookup = metadata.getIndicesLookup(); IndicesPermission indicesPermission = new IndicesPermission.Builder(RESTRICTED_INDICES).addGroup( IndexPrivilege.READ, FieldPermissions.DEFAULT, @@ -488,7 +480,7 @@ public void testAuthorizationForBackingIndices() { IndicesAccessControl iac = indicesPermission.authorize( TransportSearchAction.TYPE.name(), Sets.newHashSet(backingIndices.stream().map(im -> im.getIndex().getName()).collect(Collectors.toList())), - lookup, + metadata, fieldPermissionsCache ); @@ -508,7 +500,7 @@ public void testAuthorizationForBackingIndices() { iac = indicesPermission.authorize( randomFrom(TransportPutMappingAction.TYPE.name(), TransportAutoPutMappingAction.TYPE.name()), Sets.newHashSet(backingIndices.stream().map(im -> im.getIndex().getName()).collect(Collectors.toList())), - lookup, + metadata, fieldPermissionsCache ); @@ -521,7 +513,7 @@ public void testAuthorizationForBackingIndices() { public void testAuthorizationForMappingUpdates() { final Settings indexSettings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()).build(); - final Metadata.Builder metadata = new Metadata.Builder().put( + final Metadata.Builder metadataBuilder = new Metadata.Builder().put( new IndexMetadata.Builder("test1").settings(indexSettings).numberOfShards(1).numberOfReplicas(0).build(), true ).put(new IndexMetadata.Builder("test_write1").settings(indexSettings).numberOfShards(1).numberOfReplicas(0).build(), true); @@ -535,12 +527,12 @@ public void testAuthorizationForMappingUpdates() { "test_write2", backingIndices.stream().map(IndexMetadata::getIndex).collect(Collectors.toList()) ); - metadata.put(ds); + metadataBuilder.put(ds); for (IndexMetadata index : backingIndices) { - metadata.put(index, false); + metadataBuilder.put(index, false); } - SortedMap lookup = metadata.build().getIndicesLookup(); + Metadata metadata = metadataBuilder.build(); FieldPermissionsCache fieldPermissionsCache = new FieldPermissionsCache(Settings.EMPTY); IndicesPermission core = new IndicesPermission.Builder(RESTRICTED_INDICES).addGroup( @@ -561,7 +553,7 @@ public void testAuthorizationForMappingUpdates() { IndicesAccessControl iac = core.authorize( TransportPutMappingAction.TYPE.name(), Sets.newHashSet("test1", "test_write1"), - lookup, + metadata, fieldPermissionsCache ); assertThat(iac.isGranted(), is(true)); @@ -589,7 +581,7 @@ public void testAuthorizationForMappingUpdates() { iac = core.authorize( TransportAutoPutMappingAction.TYPE.name(), Sets.newHashSet("test1", "test_write1"), - lookup, + metadata, fieldPermissionsCache ); assertThat(iac.isGranted(), is(true)); @@ -605,17 +597,17 @@ public void testAuthorizationForMappingUpdates() { + "users who require access to update mappings must be granted explicit privileges" ); - iac = core.authorize(TransportAutoPutMappingAction.TYPE.name(), Sets.newHashSet("test_write2"), lookup, fieldPermissionsCache); + iac = core.authorize(TransportAutoPutMappingAction.TYPE.name(), Sets.newHashSet("test_write2"), metadata, fieldPermissionsCache); assertThat(iac.isGranted(), is(true)); assertThat(iac.getIndexPermissions("test_write2"), is(notNullValue())); assertThat(iac.hasIndexPermissions("test_write2"), is(true)); - iac = core.authorize(TransportPutMappingAction.TYPE.name(), Sets.newHashSet("test_write2"), lookup, fieldPermissionsCache); + iac = core.authorize(TransportPutMappingAction.TYPE.name(), Sets.newHashSet("test_write2"), metadata, fieldPermissionsCache); assertThat(iac.getIndexPermissions("test_write2"), is(nullValue())); assertThat(iac.hasIndexPermissions("test_write2"), is(false)); iac = core.authorize( TransportAutoPutMappingAction.TYPE.name(), Sets.newHashSet(backingIndices.stream().map(im -> im.getIndex().getName()).collect(Collectors.toList())), - lookup, + metadata, fieldPermissionsCache ); assertThat(iac.isGranted(), is(true)); @@ -626,7 +618,7 @@ public void testAuthorizationForMappingUpdates() { iac = core.authorize( TransportPutMappingAction.TYPE.name(), Sets.newHashSet(backingIndices.stream().map(im -> im.getIndex().getName()).collect(Collectors.toList())), - lookup, + metadata, fieldPermissionsCache ); assertThat(iac.isGranted(), is(false)); diff --git a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java index cef3572ee3ac4..ed173d8e2b127 100644 --- a/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java +++ b/x-pack/plugin/security/src/test/java/org/elasticsearch/xpack/security/authz/store/CompositeRolesStoreTests.java @@ -991,8 +991,7 @@ public void testMergingRolesWithFls() { true ) .build(); - IndicesAccessControl iac = role.indices() - .authorize("indices:data/read/search", Collections.singleton("test"), metadata.getIndicesLookup(), cache); + IndicesAccessControl iac = role.indices().authorize("indices:data/read/search", Collections.singleton("test"), metadata, cache); assertTrue(iac.getIndexPermissions("test").getFieldPermissions().grantsAccessTo("L1.foo")); assertFalse(iac.getIndexPermissions("test").getFieldPermissions().grantsAccessTo("L2.foo")); assertTrue(iac.getIndexPermissions("test").getFieldPermissions().grantsAccessTo("L3.foo")); @@ -2306,18 +2305,15 @@ public void testGetRoleForCrossClusterAccessAuthentication() throws Exception { .build(); final var emptyCache = new FieldPermissionsCache(Settings.EMPTY); assertThat( - role.authorize(TransportSearchAction.TYPE.name(), Sets.newHashSet("index1"), indexMetadata.getIndicesLookup(), emptyCache) - .isGranted(), + role.authorize(TransportSearchAction.TYPE.name(), Sets.newHashSet("index1"), indexMetadata, emptyCache).isGranted(), is(false == emptyRemoteRole) ); assertThat( - role.authorize(TransportCreateIndexAction.TYPE.name(), Sets.newHashSet("index1"), indexMetadata.getIndicesLookup(), emptyCache) - .isGranted(), + role.authorize(TransportCreateIndexAction.TYPE.name(), Sets.newHashSet("index1"), indexMetadata, emptyCache).isGranted(), is(false) ); assertThat( - role.authorize(TransportSearchAction.TYPE.name(), Sets.newHashSet("index2"), indexMetadata.getIndicesLookup(), emptyCache) - .isGranted(), + role.authorize(TransportSearchAction.TYPE.name(), Sets.newHashSet("index2"), indexMetadata, emptyCache).isGranted(), is(false) ); }